Extracts tokens from text and applies tokenization.
(examples)
| 71 | |
| 72 | # Tokenize dataset for token classification |
| 73 | def preprocess(examples): |
| 74 | """Extracts tokens from text and applies tokenization.""" |
| 75 | |
| 76 | doc = examples['document'] |
| 77 | |
| 78 | tokenized = tokenizer(doc, padding="max_length", truncation=True, return_offsets_mapping=True) |
| 79 | |
| 80 | # Generate labels (1 for credentials, 0 otherwise) |
| 81 | labels = [0] * len(tokenized["input_ids"]) |
| 82 | |
| 83 | credentials = examples["cred"] |
| 84 | if credentials != []: |
| 85 | for credential in credentials: |
| 86 | start_idx = doc.find(credential) |
| 87 | |
| 88 | end_idx = start_idx + len(credential) |
| 89 | |
| 90 | # Label tokens within credential span |
| 91 | for i, (tok_start, tok_end) in enumerate(tokenized["offset_mapping"]): |
| 92 | if tok_start >= start_idx and tok_end <= end_idx: |
| 93 | labels[i] = 1 # Mark as credential token |
| 94 | |
| 95 | tokenized["labels"] = labels |
| 96 | return tokenized |
| 97 | |
| 98 | |
| 99 | # Apply preprocessing to dataset |
nothing calls this directly
no outgoing calls
no test coverage detected