Compute an initial indices arrays for data to be indexed.
(
self, xarr: list[np.ndarray], nrow: int, reduction: int
)
| 650 | self.create_temp() |
| 651 | |
| 652 | def initial_append( |
| 653 | self, xarr: list[np.ndarray], nrow: int, reduction: int |
| 654 | ) -> tuple[int, np.ndarray, np.ndarray]: |
| 655 | """Compute an initial indices arrays for data to be indexed.""" |
| 656 | if profile: |
| 657 | tref = clock() |
| 658 | if profile: |
| 659 | show_stats("Entering initial_append", tref) |
| 660 | arr = xarr.pop() |
| 661 | indsize = self.indsize |
| 662 | slicesize = self.slicesize |
| 663 | nelements_ilr = self.nelementsILR |
| 664 | if profile: |
| 665 | show_stats("Before creating idx", tref) |
| 666 | if indsize == 8: |
| 667 | # Casting to int "nrow * slicesize" fixes #1185 |
| 668 | idx = np.arange(0, len(arr), dtype="uint64") + int( |
| 669 | nrow * slicesize |
| 670 | ) |
| 671 | elif indsize == 4: |
| 672 | # For medium (32-bit) all the rows in tables should be |
| 673 | # directly reachable. But as len(arr) < 2**31, we can |
| 674 | # choose uint32 for representing indices. In this way, we |
| 675 | # consume far less memory during the keysort process. The |
| 676 | # offset will be added in self.final_idx32() later on. |
| 677 | # |
| 678 | # This optimization also prevents the values in LR to |
| 679 | # participate in the ``swap_chunks`` process, and this is |
| 680 | # the main reason to not allow the medium indexes to create |
| 681 | # completely sorted indexes. However, I don't find this to |
| 682 | # be a big limitation, as probably fully indexes are much |
| 683 | # more suitable for producing completely sorted indexes |
| 684 | # because in this case the indices part is usable for |
| 685 | # getting the reverse indices of the index, and I forsee |
| 686 | # this to be a common requirement in many operations (for |
| 687 | # example, in table sorts). |
| 688 | # |
| 689 | # F. Alted 2008-09-15 |
| 690 | idx = np.arange(0, len(arr), dtype="uint32") |
| 691 | else: |
| 692 | idx = np.empty(len(arr), "uint%d" % (indsize * 8)) |
| 693 | lbucket = self.lbucket |
| 694 | # Fill the idx with the bucket indices |
| 695 | offset = int(lbucket - ((nrow * (slicesize % lbucket)) % lbucket)) |
| 696 | idx[0:offset] = 0 |
| 697 | for i in range(offset, slicesize, lbucket): |
| 698 | idx[i : i + lbucket] = (i + lbucket - 1) // lbucket |
| 699 | if indsize == 2: |
| 700 | # Add a second offset in this case |
| 701 | # First normalize the number of rows |
| 702 | offset2 = (nrow % self.nslicesblock) * slicesize // lbucket |
| 703 | assert offset2 < 2 ** (indsize * 8) |
| 704 | idx += np.asarray(offset2).astype(idx.dtype) |
| 705 | # Add the last row at the beginning of arr & idx (if needed) |
| 706 | if indsize == 8 and nelements_ilr > 0: |
| 707 | # It is possible that the values in LR are already sorted. |
| 708 | # Fetch them and override existing values in arr and idx. |
| 709 | assert len(arr) > nelements_ilr |
no test coverage detected