| 139 | } |
| 140 | |
| 141 | Status ReadVariantTensor(io::InputBuffer* buffered_file, Tensor* ret, |
| 142 | size_t offset, size_t size, uint32* actual_crc32c) { |
| 143 | // On-disk format: |
| 144 | // [varint64 len1][bytes variant1][4 byte checksum] |
| 145 | // .. |
| 146 | // [varint64 lenN][bytes variantN][4 byte checksum] |
| 147 | // Var "crc32c" checksums all the lens, variant bytes, individual variant |
| 148 | // checksums (as uint32, not varint32 bytes). |
| 149 | if (size == 0) return Status::OK(); |
| 150 | size_t num_elements = ret->NumElements(); |
| 151 | |
| 152 | // Reads the actual string bytes. |
| 153 | TF_RETURN_IF_ERROR(buffered_file->Seek(offset)); |
| 154 | for (size_t i = 0; i < num_elements; ++i) { |
| 155 | // Read the serialized variant length. |
| 156 | uint64 string_length = 0; |
| 157 | TF_RETURN_IF_ERROR(buffered_file->ReadVarint64(&string_length)); |
| 158 | *actual_crc32c = crc32c::Extend( |
| 159 | *actual_crc32c, reinterpret_cast<const char*>(&string_length), |
| 160 | sizeof(uint64)); |
| 161 | // Read the actual serialized variant. |
| 162 | string buffer; |
| 163 | buffer.resize(string_length); |
| 164 | size_t bytes_read = 0; |
| 165 | TF_RETURN_IF_ERROR( |
| 166 | buffered_file->ReadNBytes(string_length, &buffer[0], &bytes_read)); |
| 167 | *actual_crc32c = crc32c::Extend(*actual_crc32c, buffer.data(), bytes_read); |
| 168 | VariantTensorDataProto proto; |
| 169 | if (!proto.ParseFromString(buffer)) { |
| 170 | return errors::DataLoss("Unable to parse VariantTensorDataProto from ", |
| 171 | "buffer of size ", string_length, ". ", |
| 172 | "Bundle entry offset: ", offset, " size: ", size); |
| 173 | } |
| 174 | Variant v = proto; |
| 175 | if (!DecodeUnaryVariant(&v)) { |
| 176 | return errors::Internal("Could not decode variant with type_name: \"", |
| 177 | v.TypeName(), "\". Perhaps you forgot to ", |
| 178 | "register a decoder via ", |
| 179 | "REGISTER_UNARY_VARIANT_DECODE_FUNCTION?"); |
| 180 | } |
| 181 | |
| 182 | // Read the checksum. |
| 183 | uint32 checksum = 0; |
| 184 | size_t unused_bytes_read = 0; |
| 185 | TF_RETURN_IF_ERROR(buffered_file->ReadNBytes( |
| 186 | sizeof(uint32), reinterpret_cast<char*>(&checksum), |
| 187 | &unused_bytes_read)); |
| 188 | if (crc32c::Unmask(checksum) != *actual_crc32c) { |
| 189 | return errors::DataLoss( |
| 190 | "The checksum after Variant ", i, " does not match.", |
| 191 | " Expected: ", strings::Printf("%08u", crc32c::Unmask(checksum)), |
| 192 | " Actual: ", strings::Printf("%08u", *actual_crc32c)); |
| 193 | } |
| 194 | *actual_crc32c = crc32c::Extend( |
| 195 | *actual_crc32c, reinterpret_cast<char*>(&checksum), sizeof(uint32)); |
| 196 | |
| 197 | ret->flat<Variant>()(i) = std::move(v); |
| 198 | } |
no test coverage detected