(a, tokenizer=None)
| 153 | |
| 154 | |
| 155 | def process(a, tokenizer=None): |
| 156 | if tokenizer is not None: |
| 157 | a = tokenizer(a)[0].tolist() |
| 158 | return set(a) |
| 159 | |
| 160 | |
| 161 | def get_idf_dict(arr, tokenizer, nthreads=4): |
nothing calls this directly
no outgoing calls
no test coverage detected