| 81 | } |
| 82 | |
| 83 | void BlockBuilder::Add(const StringPiece& key, const StringPiece& value) { |
| 84 | StringPiece last_key_piece(last_key_); |
| 85 | assert(!finished_); |
| 86 | assert(counter_ <= options_->block_restart_interval); |
| 87 | assert(buffer_.empty() // No values yet? |
| 88 | || key.compare(last_key_piece) > 0); |
| 89 | size_t shared = 0; |
| 90 | if (counter_ < options_->block_restart_interval) { |
| 91 | // See how much sharing to do with previous string |
| 92 | const size_t min_length = std::min(last_key_piece.size(), key.size()); |
| 93 | while ((shared < min_length) && (last_key_piece[shared] == key[shared])) { |
| 94 | shared++; |
| 95 | } |
| 96 | } else { |
| 97 | // Restart compression |
| 98 | CHECK_LE(buffer_.size(), std::numeric_limits<uint32_t>::max()); |
| 99 | restarts_.push_back(static_cast<uint32_t>(buffer_.size())); |
| 100 | counter_ = 0; |
| 101 | } |
| 102 | const size_t non_shared = key.size() - shared; |
| 103 | |
| 104 | CHECK_LE(shared, std::numeric_limits<uint32_t>::max()); |
| 105 | CHECK_LE(non_shared, std::numeric_limits<uint32_t>::max()); |
| 106 | CHECK_LE(value.size(), std::numeric_limits<uint32_t>::max()); |
| 107 | |
| 108 | // Add "<shared><non_shared><value_size>" to buffer_ |
| 109 | core::PutVarint32(&buffer_, static_cast<uint32_t>(shared)); |
| 110 | core::PutVarint32(&buffer_, static_cast<uint32_t>(non_shared)); |
| 111 | core::PutVarint32(&buffer_, static_cast<uint32_t>(value.size())); |
| 112 | |
| 113 | // Add string delta to buffer_ followed by value |
| 114 | buffer_.append(key.data() + shared, non_shared); |
| 115 | buffer_.append(value.data(), static_cast<uint32_t>(value.size())); |
| 116 | |
| 117 | // Update state |
| 118 | last_key_.resize(shared); |
| 119 | last_key_.append(key.data() + shared, non_shared); |
| 120 | assert(StringPiece(last_key_) == key); |
| 121 | counter_++; |
| 122 | } |
| 123 | |
| 124 | } // namespace table |
| 125 | } // namespace tensorflow |