Tiny hand-rolled parser for the fixed safetensors JSON schema. Safetensors JSON is always a single object: { "name": {"dtype":"BF16","shape":[a,b],"data_offsets":[s,e]}, ..., "__metadata__":{...} } We parse one top-level entry at a time, tracking brace depth to find each tensor's object boundary cleanly.
| 75 | // { "name": {"dtype":"BF16","shape":[a,b],"data_offsets":[s,e]}, ..., "__metadata__":{...} } |
| 76 | // We parse one top-level entry at a time, tracking brace depth to find each |
| 77 | // tensor's object boundary cleanly. |
| 78 | bool parse_st_header(const char * h, size_t hlen, StMap & out) { |
| 79 | auto skip_ws = [&](size_t & i) { |
| 80 | while (i < hlen && (h[i] == ' ' || h[i] == '\t' || h[i] == '\n' || h[i] == '\r')) i++; |
| 81 | }; |
| 82 | size_t i = 0; |
| 83 | skip_ws(i); |
| 84 | if (i >= hlen || h[i] != '{') return false; |
| 85 | i++; |
| 86 | while (i < hlen) { |
| 87 | skip_ws(i); |
| 88 | if (i >= hlen) return false; |
| 89 | if (h[i] == '}') { i++; break; } |
| 90 | if (h[i] == ',') { i++; skip_ws(i); } |
| 91 | if (i >= hlen || h[i] != '"') return false; |
| 92 | i++; |
| 93 | size_t name_start = i; |
| 94 | while (i < hlen && h[i] != '"') i++; |
| 95 | if (i >= hlen) return false; |
| 96 | std::string name(h + name_start, i - name_start); |
| 97 | i++; |
| 98 | skip_ws(i); |
| 99 | if (i >= hlen || h[i] != ':') return false; |
| 100 | i++; |
| 101 | skip_ws(i); |
| 102 | if (i >= hlen || h[i] != '{') return false; |
| 103 | size_t obj_start = i; |
| 104 | int depth = 0; |
| 105 | size_t obj_end = i; |
| 106 | for (; obj_end < hlen; obj_end++) { |
| 107 | if (h[obj_end] == '{') depth++; |
| 108 | else if (h[obj_end] == '}') { |
| 109 | depth--; |
| 110 | if (depth == 0) { obj_end++; break; } |
| 111 | } |
| 112 | } |
| 113 | if (depth != 0) return false; |
| 114 | |
| 115 | if (name == "__metadata__") { |
| 116 | i = obj_end; |
| 117 | continue; |
| 118 | } |
| 119 | |
| 120 | std::string obj(h + obj_start, obj_end - obj_start); |
| 121 | |
| 122 | StEntry e; |
| 123 | { |
| 124 | auto k = obj.find("\"dtype\":\""); |
| 125 | if (k == std::string::npos) return false; |
| 126 | auto vs = k + 9; |
| 127 | auto ve = obj.find('"', vs); |
| 128 | if (ve == std::string::npos) return false; |
| 129 | e.dtype = obj.substr(vs, ve - vs); |
| 130 | } |
| 131 | { |
| 132 | auto k = obj.find("\"shape\":["); |
| 133 | if (k == std::string::npos) return false; |
| 134 | auto vs = k + 9; |
no outgoing calls
no test coverage detected