Track errors and aggregates their counts by similarity.
| 21 | |
| 22 | |
| 23 | class ErrorTracker: |
| 24 | """Track errors and aggregates their counts by similarity.""" |
| 25 | |
| 26 | def __init__( |
| 27 | self, |
| 28 | *, |
| 29 | snapshot_kvs_name: str | None = None, |
| 30 | show_error_name: bool = True, |
| 31 | show_file_and_line_number: bool = True, |
| 32 | show_error_message: bool = True, |
| 33 | show_full_message: bool = False, |
| 34 | save_error_snapshots: bool = False, |
| 35 | ) -> None: |
| 36 | self.error_snapshotter = ErrorSnapshotter(snapshot_kvs_name=snapshot_kvs_name) if save_error_snapshots else None |
| 37 | self.show_error_name = show_error_name |
| 38 | self.show_file_and_line_number = show_file_and_line_number |
| 39 | self.show_error_message = show_error_message |
| 40 | if show_full_message and not show_error_message: |
| 41 | raise ValueError('`show_error_message` must be `True` if `show_full_message` is set to `True`') |
| 42 | self.show_full_message = show_full_message |
| 43 | self._errors: ErrorFilenameGroups = defaultdict(lambda: defaultdict(Counter)) |
| 44 | |
| 45 | async def add( |
| 46 | self, |
| 47 | error: Exception, |
| 48 | *, |
| 49 | context: BasicCrawlingContext | None = None, |
| 50 | ) -> None: |
| 51 | """Add an error in the statistics. |
| 52 | |
| 53 | Args: |
| 54 | error: Error to be added to statistics. |
| 55 | context: Context used to collect error snapshot. |
| 56 | """ |
| 57 | error_group_name = error.__class__.__name__ if self.show_error_name else None |
| 58 | error_group_message = self._get_error_message(error) |
| 59 | new_error_group_message = '' # In case of wildcard similarity match |
| 60 | error_group_file_and_line = self._get_file_and_line(error) |
| 61 | |
| 62 | # First two levels are grouped only in case of exact match. |
| 63 | specific_groups = self._errors[error_group_file_and_line][error_group_name] |
| 64 | |
| 65 | # Lowest level group is matched by similarity. |
| 66 | if error_group_message in specific_groups: |
| 67 | # Exact match. |
| 68 | specific_groups.update([error_group_message]) |
| 69 | else: |
| 70 | for existing_error_group_message in specific_groups: |
| 71 | # Add to first group with similar text. Modify text with wildcard characters if necessary. |
| 72 | if new_error_group_message := self._create_generic_message( |
| 73 | existing_error_group_message, error_group_message |
| 74 | ): |
| 75 | # Replace old name. |
| 76 | specific_groups[new_error_group_message] = specific_groups.pop(existing_error_group_message) |
| 77 | # Increment. |
| 78 | specific_groups.update([new_error_group_message]) |
| 79 | break |
| 80 | else: |
no outgoing calls