Reads "num_elements" string elements from file[offset, offset+size) into the length-N "destination". Discards the original content of "destination". Checksums the string lengths (as restored uint32 or uint64, not varint64 bytes) and string bytes, and stores it into "actual_crc32c".
| 69 | // Checksums the string lengths (as restored uint32 or uint64, not varint64 |
| 70 | // bytes) and string bytes, and stores it into "actual_crc32c". |
| 71 | Status ReadStringTensor(io::InputBuffer* buffered_file, size_t num_elements, |
| 72 | size_t offset, size_t size, tstring* destination, |
| 73 | uint32* actual_crc32c, bool need_to_swap_bytes) { |
| 74 | if (size == 0) return Status::OK(); |
| 75 | CHECK_GT(size, 0); |
| 76 | |
| 77 | // Reads "num_elements" varint64's from "buffered_file". |
| 78 | TF_RETURN_IF_ERROR(buffered_file->Seek(offset)); |
| 79 | std::vector<uint64> string_lengths(num_elements); |
| 80 | for (size_t i = 0; i < num_elements; ++i) { |
| 81 | TF_RETURN_IF_ERROR(buffered_file->ReadVarint64(&string_lengths[i])); |
| 82 | if (string_lengths[i] <= UINT32_MAX) { |
| 83 | // We need to do this because older checkpoints only used uint32s and we |
| 84 | // should still support them. |
| 85 | uint32 elem_size_uint32 = static_cast<uint32>(string_lengths[i]); |
| 86 | if (need_to_swap_bytes) { |
| 87 | // Checksum would have been computed on the source machine's byte order |
| 88 | elem_size_uint32 = BYTE_SWAP_32(elem_size_uint32); |
| 89 | } |
| 90 | *actual_crc32c = crc32c::Extend( |
| 91 | *actual_crc32c, reinterpret_cast<const char*>(&elem_size_uint32), |
| 92 | sizeof(uint32)); |
| 93 | } else { |
| 94 | uint64 length = string_lengths[i]; |
| 95 | if (need_to_swap_bytes) { |
| 96 | length = BYTE_SWAP_64(length); |
| 97 | } |
| 98 | *actual_crc32c = |
| 99 | crc32c::Extend(*actual_crc32c, reinterpret_cast<const char*>(&length), |
| 100 | sizeof(uint64)); |
| 101 | } |
| 102 | } |
| 103 | if (offset + size < buffered_file->Tell()) { |
| 104 | return errors::DataLoss("String lengths longer than expected offset ", |
| 105 | offset + size); |
| 106 | } |
| 107 | |
| 108 | // Reads the length-checksum. |
| 109 | uint32 raw_length_checksum = 0; // Bytes in file |
| 110 | uint32 length_checksum = 0; // In-memory representation |
| 111 | size_t unused_bytes_read = 0; |
| 112 | TF_RETURN_IF_ERROR(buffered_file->ReadNBytes( |
| 113 | sizeof(uint32), reinterpret_cast<char*>(&raw_length_checksum), |
| 114 | &unused_bytes_read)); |
| 115 | length_checksum = need_to_swap_bytes ? BYTE_SWAP_32(raw_length_checksum) |
| 116 | : raw_length_checksum; |
| 117 | if (crc32c::Unmask(length_checksum) != *actual_crc32c) { |
| 118 | return errors::DataLoss( |
| 119 | "The length checksum does not match: expected ", |
| 120 | strings::Printf("%08u", crc32c::Unmask(length_checksum)), |
| 121 | " but actual is ", strings::Printf("%08u", *actual_crc32c)); |
| 122 | } |
| 123 | *actual_crc32c = crc32c::Extend(*actual_crc32c, |
| 124 | reinterpret_cast<char*>(&raw_length_checksum), |
| 125 | sizeof(uint32)); |
| 126 | |
| 127 | // Reads the actual string bytes. |
| 128 | for (size_t i = 0; i < num_elements; ++i) { |
no test coverage detected