MCPcopy Create free account
hub / github.com/DeepRec-AI/DeepRec / LookupHeader

Method LookupHeader

tensorflow/core/util/tensor_bundle/tensor_bundle.cc:1079–1111  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1077}
1078
1079Status BundleReader::LookupHeader(StringPiece tensor_key, int64 total_bytes) {
1080 BundleEntryProto entry;
1081 TF_RETURN_IF_ERROR(GetBundleEntryProto(tensor_key, &entry));
1082 if (entry.size() != total_bytes) {
1083 return errors::DataLoss("Invalid size in bundle entry: key ", key(),
1084 "; stored size ", entry.size(),
1085 "; expected size ", total_bytes);
1086 }
1087 io::InputBuffer* buffered_file = data_[entry.shard_id()];
1088 if (buffered_file == nullptr) {
1089 std::unique_ptr<RandomAccessFile> file = nullptr;
1090 TF_RETURN_IF_ERROR(env_->NewRandomAccessFile(
1091 DataFilename(prefix_, entry.shard_id(), num_shards_), &file));
1092 buffered_file =
1093 new io::InputBuffer(file.release(), 256 << 10 /* 256KB buffer */);
1094 // The InputBuffer and RandomAccessFile objects are both released in dtor.
1095 data_[entry.shard_id()] = buffered_file;
1096 }
1097 CHECK(buffered_file != nullptr);
1098
1099 TF_RETURN_IF_ERROR(buffered_file->Seek(entry.offset()));
1100 if (!DataTypeCanUseMemcpy(entry.dtype())) {
1101 return errors::DataLoss("segment lookup not support string");
1102 }
1103 LookupSegItem seg_item;
1104 seg_item.entry = entry;
1105 seg_item.total_size = entry.size();
1106 seg_item.bytes_read = 0;
1107
1108 tmp_lookupseg_items_[string(tensor_key)] = seg_item;
1109 return Status::OK();
1110
1111}
1112
1113Status BundleReader::LookupSegment(StringPiece key, size_t buffer_size, char* destination, size_t& real_bytes_read) {
1114 LookupSegItem& seg_item = tmp_lookupseg_items_[string(key)];

Callers 3

GetTotalKeysNumFunction · 0.80
ReadRecordFunction · 0.80

Calls 8

DataFilenameFunction · 0.85
DataTypeCanUseMemcpyFunction · 0.85
sizeMethod · 0.45
NewRandomAccessFileMethod · 0.45
releaseMethod · 0.45
SeekMethod · 0.45
offsetMethod · 0.45
dtypeMethod · 0.45

Tested by

no test coverage detected