| 1077 | } |
| 1078 | |
| 1079 | Status BundleReader::LookupHeader(StringPiece tensor_key, int64 total_bytes) { |
| 1080 | BundleEntryProto entry; |
| 1081 | TF_RETURN_IF_ERROR(GetBundleEntryProto(tensor_key, &entry)); |
| 1082 | if (entry.size() != total_bytes) { |
| 1083 | return errors::DataLoss("Invalid size in bundle entry: key ", key(), |
| 1084 | "; stored size ", entry.size(), |
| 1085 | "; expected size ", total_bytes); |
| 1086 | } |
| 1087 | io::InputBuffer* buffered_file = data_[entry.shard_id()]; |
| 1088 | if (buffered_file == nullptr) { |
| 1089 | std::unique_ptr<RandomAccessFile> file = nullptr; |
| 1090 | TF_RETURN_IF_ERROR(env_->NewRandomAccessFile( |
| 1091 | DataFilename(prefix_, entry.shard_id(), num_shards_), &file)); |
| 1092 | buffered_file = |
| 1093 | new io::InputBuffer(file.release(), 256 << 10 /* 256KB buffer */); |
| 1094 | // The InputBuffer and RandomAccessFile objects are both released in dtor. |
| 1095 | data_[entry.shard_id()] = buffered_file; |
| 1096 | } |
| 1097 | CHECK(buffered_file != nullptr); |
| 1098 | |
| 1099 | TF_RETURN_IF_ERROR(buffered_file->Seek(entry.offset())); |
| 1100 | if (!DataTypeCanUseMemcpy(entry.dtype())) { |
| 1101 | return errors::DataLoss("segment lookup not support string"); |
| 1102 | } |
| 1103 | LookupSegItem seg_item; |
| 1104 | seg_item.entry = entry; |
| 1105 | seg_item.total_size = entry.size(); |
| 1106 | seg_item.bytes_read = 0; |
| 1107 | |
| 1108 | tmp_lookupseg_items_[string(tensor_key)] = seg_item; |
| 1109 | return Status::OK(); |
| 1110 | |
| 1111 | } |
| 1112 | |
| 1113 | Status BundleReader::LookupSegment(StringPiece key, size_t buffer_size, char* destination, size_t& real_bytes_read) { |
| 1114 | LookupSegItem& seg_item = tmp_lookupseg_items_[string(key)]; |
no test coverage detected