Applies sentence level edits to a source document.
(
source,
edit,
delimiter_range_pair,
task = None,
)
| 699 | |
| 700 | |
| 701 | def normalize( |
| 702 | source, |
| 703 | edit, |
| 704 | delimiter_range_pair, |
| 705 | task = None, |
| 706 | ): |
| 707 | """Applies sentence level edits to a source document.""" |
| 708 | |
| 709 | evidence_delimiter_range = delimiter_range_pair.evidence_delimiter_range |
| 710 | sentence_delimiter_range = delimiter_range_pair.sentence_delimiter_range |
| 711 | |
| 712 | # Remove context from clean source. |
| 713 | source, _ = source.split(CONTEXT_MARKER) |
| 714 | source = source.strip() |
| 715 | |
| 716 | # In controllable setting we need to remove all of the additional markers from |
| 717 | # the input in order to make it similar to the diff format. |
| 718 | if task == Task.controllable: |
| 719 | source = source.replace(ADDITION_MARKER, "") |
| 720 | source = source.replace(DELETION_MARKER, "") |
| 721 | source = source.replace(EDIT_MARKER, "") |
| 722 | source = evidence_delimiter_range.remove_delims(source) |
| 723 | |
| 724 | # Split source into sentences using sentence delimiters, and recover clean |
| 725 | # source by joining delimiter-less sentences. |
| 726 | clean_source = sentence_delimiter_range.remove_delims(source) |
| 727 | |
| 728 | # Remove evidence delimiters from edit if present. |
| 729 | clean_edit = evidence_delimiter_range.remove_delims(edit) |
| 730 | |
| 731 | # Iterate over copy delimiters in edit to identify which source sentences |
| 732 | # to keep vs edit and join them together. |
| 733 | # NOTE: Source UIDs are zero-delimited, however there will be at least one |
| 734 | # (possibly empty) string before the first delimiter, hence we always reject |
| 735 | # the first "sentence" in the split. |
| 736 | source_sentences = sentence_delimiter_range.split(source)[1:] |
| 737 | clean_edited = sentence_delimiter_range.copy_replace( |
| 738 | clean_edit, src=source_sentences) |
| 739 | |
| 740 | # # Lastly remove any double whitespace. |
| 741 | clean_source = re.sub(r"\s{2,}", " ", clean_source) |
| 742 | clean_edited = re.sub(r"\s{2,}", " ", clean_edited) |
| 743 | |
| 744 | return clean_source, clean_edited |
| 745 | |
| 746 | |
| 747 | def extract_additions(source, target): |
nothing calls this directly
no test coverage detected
searching dependent graphs…