Function bpe_tokenize_text

ds4.c:22153–22221

* DeepSeek V4 Flash declares tokenizer.ggml.pre = "joyai-llm". The split * below mirrors the JoyAI BPE pre-tokenizer for the cases this model * uses in normal text and source-code prompts: * * \p{N}{1,3} * [CJK/Hiragana/Katakana]+ * [P/S][A-Za-z]+ * [^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+ * ?[\p{P}\p{S}]+[\r\n]* * \s*[\r\n]+ * \s+(?!\S) * \s+ * * The punctuation rule in

        }
    } else if (ok) {
        ok = metal_graph_decode_hc_pre(metal_graph_attn_cur(g),
                                       metal_graph_hc_split(g),
                                       metal_graph_hc_mix(g),
                                       metal_graph_cur_hc(g),
                                       model,
                                       layer->hc_attn_scale->abs_offset,
                                       layer->hc_attn_base->abs_offset);
    }
    DS4_METAL_PROFILE_DECODE_STAGE("attn_hc_pre");
    if (ok) {
        metal_graph_debug_dump_tensor("hc_attn_pre_mixes", metal_graph_hc_mix(g), mix_hc, il, pos);
        metal_graph_debug_dump_tensor("hc_attn_pre_weights", metal_graph_hc_pre(g), DS4_N_HC, il, pos);
        metal_graph_debug_dump_tensor("hc_attn_pre_post_weights", metal_graph_hc_post(g), DS4_N_HC, il, pos);
        metal_graph_debug_dump_tensor("hc_attn_pre_comb", metal_graph_hc_comb(g), (uint64_t)DS4_N_HC * DS4_N_HC, il, pos);
    }
    if (ok) {
        metal_graph_debug_dump_tensor("hc_attn_pre", metal_graph_attn_cur(g), DS4_N_EMBD, il, pos);
    }
    if (ok && !fuse_hc_norm) ok = ds4_gpu_rms_norm_weight_tensor(metal_graph_attn_norm(g), metal_graph_attn_cur(g),
                                                                   model->map, model->size,
                                                                   layer->attn_norm->abs_offset,
                                                                   DS4_N_EMBD, DS4_RMS_EPS) != 0;
    DS4_METAL_PROFILE_DECODE_STAGE("attn_norm");
    if (ok) {
        metal_graph_debug_dump_tensor("attn_norm", metal_graph_attn_norm(g), DS4_N_EMBD, il, pos);
    }
    if (phase == METAL_DECODE_LAYER_TO_QKV) return ok;
    }
    bool kv_norm_store_fused = false;
    if (!resume_after_attn) {
    /* Fuse the KV RoPE tail into the FP8/raw finalizer: both were single
     * 64-thread threadgroups on the same 2 KB row, so the pair paid two full
     * dispatch launches. Deferring the RoPE and running it inside the
     * finalizer removes one dispatch per layer. */
    bool fuse_kv_rope_store = false;
    bool qkv_pair_quad_fused = false;
    if (!resume_after_qkv) {
    bool qkv_pair_projected = resume_after_qa_kv_raw;
    /* M1-M5 decode fusion: the q_a/kv Q8 pair and the four F16 compressor
     * projections all read the same normalized attention input and write
     * disjoint outputs, so one dispatch covers both stages with unchanged
     * per-row reduction trees (see the kernel comment).  Restricted to the
     * resident FULL phase so split-phase / CUDA / SSD flows keep their
     * original ordering. */
    if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && compressed &&
        phase == METAL_DECODE_LAYER_FULL &&
        !g->ssd_streaming && !g->ssd_streaming_cold &&
        ds4_layer_compress_ratio(il) == 4u &&
        layer->attn_q_a->type == DS4_TENSOR_Q8_0 &&
        layer->attn_kv->type == DS4_TENSOR_Q8_0 &&
        g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj() &&
        !metal_graph_use_reference_compressor_pair_proj() &&
        layer->attn_compressor_kv && layer->attn_compressor_gate &&
        layer->attn_compressor_ape &&
        layer->attn_compressor_kv->type == DS4_TENSOR_F16 &&
        layer->attn_compressor_gate->type == DS4_TENSOR_F16 &&
        layer->attn_compressor_kv->dim[0] == DS4_N_EMBD &&
        layer->attn_compressor_gate->dim[0] == DS4_N_EMBD &&