Shuffles the DataFrame according to its new divisions. Simplifies the Expression to blockwise pre-processing, shuffle and blockwise post-processing expressions. Parameters ---------- frame: Expr Frame-like expression where the index is set. _other: Expr | Scalar
| 1130 | |
| 1131 | |
| 1132 | class SetPartition(SetIndex): |
| 1133 | """Shuffles the DataFrame according to its new divisions. |
| 1134 | |
| 1135 | Simplifies the Expression to blockwise pre-processing, shuffle and |
| 1136 | blockwise post-processing expressions. |
| 1137 | |
| 1138 | Parameters |
| 1139 | ---------- |
| 1140 | frame: Expr |
| 1141 | Frame-like expression where the index is set. |
| 1142 | _other: Expr | Scalar |
| 1143 | Either a Series-like expression to use as Index or a scalar defining the column. |
| 1144 | drop: bool |
| 1145 | Whether to drop the old column. |
| 1146 | new_divisions: int |
| 1147 | Divisions of the resulting expression. |
| 1148 | """ |
| 1149 | |
| 1150 | _parameters = [ |
| 1151 | "frame", |
| 1152 | "_other", |
| 1153 | "drop", |
| 1154 | "npartitions", |
| 1155 | "ascending", |
| 1156 | "upsample", |
| 1157 | "user_divisions", |
| 1158 | "shuffle_method", |
| 1159 | "options", # Shuffle method options |
| 1160 | ] |
| 1161 | |
| 1162 | def _lower(self): |
| 1163 | divisions = self.other._meta._constructor(self._divisions()) |
| 1164 | partitions = _SetPartitionsPreSetIndex(self.other, divisions) |
| 1165 | assigned = Assign(self.frame, "_partitions", partitions) |
| 1166 | if isinstance(self._other, Expr): |
| 1167 | assigned = Assign(assigned, "_index", self._other) |
| 1168 | shuffled = Shuffle( |
| 1169 | assigned, |
| 1170 | "_partitions", |
| 1171 | npartitions_out=len(self._divisions()) - 1, |
| 1172 | ignore_index=True, |
| 1173 | method=self.shuffle_method, |
| 1174 | options=self.options, |
| 1175 | ) |
| 1176 | shuffled = Projection( |
| 1177 | shuffled, [c for c in assigned.columns if c != "_partitions"] |
| 1178 | ) |
| 1179 | |
| 1180 | if isinstance(self._other, Expr): |
| 1181 | drop, set_name = True, "_index" |
| 1182 | else: |
| 1183 | drop, set_name = self.drop, self.other._meta.name |
| 1184 | lru_key = ( |
| 1185 | self.other._name, |
| 1186 | self._npartitions_input, |
| 1187 | self.ascending, |
| 1188 | 128e6, |
| 1189 | self.upsample, |