| 50 | self.args = args |
| 51 | |
| 52 | def initializer(self): |
| 53 | # Use Encoder class as a container for global data |
| 54 | Encoder.tokenizer = build_tokenizer(self.args) |
| 55 | if self.args.split_sentences: |
| 56 | if not nltk_available: |
| 57 | print("NLTK is not available to split sentences.") |
| 58 | exit() |
| 59 | if os.environ.get("NLTK_DATA"): |
| 60 | library = os.path.join(os.environ.get("NLTK_DATA"), "tokenizers", "punkt", f"{self.args.lang}.pickle") |
| 61 | url = f"file:{library}" |
| 62 | else: |
| 63 | library = os.path.join("tokenizers", "punkt", f"{self.args.lang}.pickle") |
| 64 | url = f"nltk:{library}" |
| 65 | splitter = nltk.load(url) |
| 66 | if self.args.keep_newlines: |
| 67 | # this prevents punkt from eating newlines after sentences |
| 68 | Encoder.splitter = nltk.tokenize.punkt.PunktSentenceTokenizer( |
| 69 | train_text = splitter._params, |
| 70 | lang_vars = CustomLanguageVars()) |
| 71 | else: |
| 72 | Encoder.splitter = splitter |
| 73 | |
| 74 | else: |
| 75 | Encoder.splitter = IdentitySplitter() |
| 76 | |
| 77 | def split(self, json_line): |
| 78 | data = json.loads(json_line) |