| 12362 | static bool g_test_tokenizer_loaded = false; |
| 12363 | |
| 12364 | bool sam3_test_load_tokenizer(const std::string& model_path) { |
| 12365 | std::ifstream fin(model_path, std::ios::binary); |
| 12366 | if (!fin) return false; |
| 12367 | |
| 12368 | // Read header |
| 12369 | uint32_t magic; |
| 12370 | int32_t version, ftype, n_tensors; |
| 12371 | fin.read(reinterpret_cast<char*>(&magic), 4); |
| 12372 | fin.read(reinterpret_cast<char*>(&version), 4); |
| 12373 | fin.read(reinterpret_cast<char*>(&ftype), 4); |
| 12374 | fin.read(reinterpret_cast<char*>(&n_tensors), 4); |
| 12375 | if (magic != SAM3_MAGIC || version != SAM3_FILE_VERSION) return false; |
| 12376 | |
| 12377 | // Skip hparams |
| 12378 | sam3_hparams hp; |
| 12379 | if (!sam3_load_hparams(fin, hp)) return false; |
| 12380 | if (hp.visual_only) return false; |
| 12381 | |
| 12382 | // Skip tensors |
| 12383 | for (int t = 0; t < n_tensors; ++t) { |
| 12384 | int32_t n_dims, name_len, dtype; |
| 12385 | fin.read(reinterpret_cast<char*>(&n_dims), 4); |
| 12386 | fin.read(reinterpret_cast<char*>(&name_len), 4); |
| 12387 | fin.read(reinterpret_cast<char*>(&dtype), 4); |
| 12388 | if (fin.fail()) return false; |
| 12389 | |
| 12390 | // Read shape to compute data size |
| 12391 | int64_t n_el = 1; |
| 12392 | std::vector<int64_t> shape(n_dims); |
| 12393 | for (int i = 0; i < n_dims; ++i) { |
| 12394 | int32_t d; |
| 12395 | fin.read(reinterpret_cast<char*>(&d), 4); |
| 12396 | shape[i] = d; |
| 12397 | n_el *= d; |
| 12398 | } |
| 12399 | |
| 12400 | // Skip name |
| 12401 | fin.seekg(name_len, std::ios::cur); |
| 12402 | |
| 12403 | // Skip padding to 32-byte alignment |
| 12404 | size_t pos = fin.tellg(); |
| 12405 | size_t pad = (32 - pos % 32) % 32; |
| 12406 | if (pad > 0) fin.seekg(pad, std::ios::cur); |
| 12407 | |
| 12408 | // Compute data size and skip |
| 12409 | const ggml_type file_type = static_cast<ggml_type>(dtype); |
| 12410 | size_t bytes; |
| 12411 | if (ggml_is_quantized(file_type)) { |
| 12412 | const int64_t n_rows = n_el / shape[0]; |
| 12413 | bytes = ggml_row_size(file_type, shape[0]) * n_rows; |
| 12414 | } else { |
| 12415 | const size_t elem_size = (file_type == GGML_TYPE_F16) ? 2 : 4; |
| 12416 | bytes = n_el * elem_size; |
| 12417 | } |
| 12418 | fin.seekg(bytes, std::ios::cur); |
| 12419 | if (fin.fail()) return false; |
| 12420 | } |
| 12421 | |