MCPcopy Create free account
hub / github.com/bytedance/bolt / setupSpillState

Method setupSpillState

bolt/exec/tests/SpillTest.cpp:110–275  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

108 }
109
110 void setupSpillState(
111 int64_t targetFileSize,
112 uint64_t writeBufferSize,
113 int numPartitions,
114 int numBatches,
115 int numRowsPerBatch = 1000,
116 int numDuplicates = 1,
117 const std::vector<CompareFlags>& compareFlags = {}) {
118 ASSERT_TRUE(compareFlags.empty() || compareFlags.size() == 1);
119 ASSERT_EQ(numBatches % 2, 0);
120
121 state_.reset();
122 batchesByPartition_.clear();
123 values_.clear();
124 runtimeStats_.clear();
125 stats_.wlock()->reset();
126
127 fileNamePrefix_ = "test";
128 values_.resize(numBatches * numRowsPerBatch);
129 // Create a sequence of sorted 'values' in ascending order starting at -10.
130 // Each distinct value occurs 'numDuplicates' times. The sequence total has
131 // numBatches * kNumRowsPerBatch item. Each batch created in the test below,
132 // contains a subsequence with index mod being equal to its batch number.
133 const int kNumNulls = numBatches;
134 for (int i = 0, value = -10; i < numRowsPerBatch * numBatches;) {
135 while (i < kNumNulls) {
136 values_[i++] = std::nullopt;
137 }
138 for (int j = 0; j < numDuplicates; ++j) {
139 values_[i++] = value++;
140 }
141 }
142 const bool nullsFirst =
143 compareFlags.empty() ? true : compareFlags[0].nullsFirst;
144 const bool ascending =
145 compareFlags.empty() ? true : compareFlags[0].ascending;
146
147 if (!ascending) {
148 if (nullsFirst) {
149 std::reverse(values_.begin() + kNumNulls, values_.end());
150 } else {
151 std::reverse(values_.begin(), values_.end());
152 ASSERT_FALSE(values_.back().has_value());
153 }
154 } else {
155 if (!nullsFirst) {
156 for (int i = 0; i < values_.size(); ++i) {
157 if (i < values_.size() - kNumNulls) {
158 values_[i] = values_[i + kNumNulls];
159 } else {
160 values_[i] = std::nullopt;
161 }
162 }
163 }
164 }
165 batchesByPartition_.resize(numPartitions);
166
167 // Setup state that has 'numPartitions' partitions, each with its own

Callers

nothing calls this directly

Calls 15

reverseFunction · 0.85
has_valueMethod · 0.80
backMethod · 0.80
targetFileSizeMethod · 0.80
maxPartitionsMethod · 0.80
spilledPartitionSetMethod · 0.80
isPartitionSpilledMethod · 0.80
appendToPartitionMethod · 0.80
setPartitionSpilledMethod · 0.80
poolFunction · 0.50
emptyMethod · 0.45

Tested by

no test coverage detected