Produce a counter series for each memory allocator.
(self)
| 785 | input_name) |
| 786 | |
| 787 | def _show_memory_counters(self): |
| 788 | """Produce a counter series for each memory allocator.""" |
| 789 | # Iterate over all tensor trackers to build a list of allocations and |
| 790 | # frees for each allocator. Then sort the lists and emit a cumulative |
| 791 | # counter series for each allocator. |
| 792 | allocations = {} |
| 793 | for name in self._tensors: |
| 794 | tensor = self._tensors[name] |
| 795 | self._chrome_trace.emit_obj_delete('Tensor', name, tensor.last_unref, |
| 796 | tensor.pid, 0, tensor.object_id) |
| 797 | allocator = tensor.allocator |
| 798 | if allocator not in allocations: |
| 799 | allocations[allocator] = [] |
| 800 | num_bytes = tensor.num_bytes |
| 801 | allocations[allocator].append((tensor.create_time, num_bytes, name)) |
| 802 | allocations[allocator].append((tensor.last_unref, -num_bytes, name)) |
| 803 | |
| 804 | alloc_maxes = {} |
| 805 | |
| 806 | # Generate a counter series showing total allocations for each allocator. |
| 807 | for allocator in allocations: |
| 808 | alloc_list = allocations[allocator] |
| 809 | alloc_list.sort() |
| 810 | total_bytes = 0 |
| 811 | alloc_tensor_set = set() |
| 812 | alloc_maxes[allocator] = AllocationMaximum( |
| 813 | timestamp=0, num_bytes=0, tensors=set()) |
| 814 | for time, num_bytes, name in sorted( |
| 815 | alloc_list, key=lambda allocation: allocation[0]): |
| 816 | total_bytes += num_bytes |
| 817 | if num_bytes < 0: |
| 818 | alloc_tensor_set.discard(name) |
| 819 | else: |
| 820 | alloc_tensor_set.add(name) |
| 821 | |
| 822 | if total_bytes > alloc_maxes[allocator].num_bytes: |
| 823 | alloc_maxes[allocator] = AllocationMaximum( |
| 824 | timestamp=time, |
| 825 | num_bytes=total_bytes, |
| 826 | tensors=copy.deepcopy(alloc_tensor_set)) |
| 827 | |
| 828 | self._chrome_trace.emit_counter('Memory', allocator, |
| 829 | self._allocators_pid, time, allocator, |
| 830 | total_bytes) |
| 831 | self._allocator_maximums = alloc_maxes |
| 832 | |
| 833 | def analyze_step_stats(self, show_dataflow=True, show_memory=True, use_real_thread_id=True): |
| 834 | self._allocate_pids() |
no test coverage detected