Preprocess the fever data to the TFDS Fever data. Args: common_config: Common configuration from config.Config scrape_type: Which scrape to use, drqa/lucene/ukp, in training data_dir: Where to write data to download_dir: Where to download data to, unused but required by TFDS API
(
*,
common_config,
scrape_type,
data_dir,
download_dir)
| 152 | |
| 153 | |
| 154 | def preprocess( |
| 155 | *, |
| 156 | common_config, |
| 157 | scrape_type, |
| 158 | data_dir, |
| 159 | download_dir): |
| 160 | """Preprocess the fever data to the TFDS Fever data. |
| 161 | |
| 162 | Args: |
| 163 | common_config: Common configuration from config.Config |
| 164 | scrape_type: Which scrape to use, drqa/lucene/ukp, in training |
| 165 | data_dir: Where to write data to |
| 166 | download_dir: Where to download data to, unused but required by TFDS API |
| 167 | """ |
| 168 | logging.info('Creating fever dataset builder') |
| 169 | text_matcher_params_path = common_config.text_matcher_params |
| 170 | fever_train_path = common_config.fever_train |
| 171 | fever_dev_path = common_config.fever_dev |
| 172 | fever_test_path = common_config.fever_test |
| 173 | |
| 174 | ukp_docs_train = common_config.ukp_docs_train |
| 175 | ukp_docs_dev = common_config.ukp_docs_dev |
| 176 | ukp_docs_test = common_config.ukp_docs_test |
| 177 | builder = fever_tfds.FeverEvidence( |
| 178 | wiki_db_path=common_config.wikipedia_db_path, |
| 179 | text_matcher_params_path=text_matcher_params_path, |
| 180 | fever_train_path=fever_train_path, |
| 181 | fever_dev_path=fever_dev_path, |
| 182 | fever_test_path=fever_test_path, |
| 183 | drqa_db_path=common_config.drqa_scrape_db_path, |
| 184 | lucene_db_path=common_config.lucene_scrape_db_path, |
| 185 | data_dir=data_dir, |
| 186 | n_similar_negatives=common_config.n_similar_negatives, |
| 187 | n_background_negatives=common_config.n_background_negatives, |
| 188 | ukp_docs_train=ukp_docs_train, |
| 189 | ukp_docs_dev=ukp_docs_dev, |
| 190 | ukp_docs_test=ukp_docs_test, |
| 191 | train_scrape_type=scrape_type, |
| 192 | title_in_scoring=common_config.title_in_scoring, |
| 193 | n_inference_candidates=common_config.n_inference_candidates, |
| 194 | include_not_enough_info=common_config.include_not_enough_info, |
| 195 | n_inference_documents=common_config.n_inference_documents, |
| 196 | max_inference_sentence_id=common_config.max_inference_sentence_id, |
| 197 | ) |
| 198 | |
| 199 | logging.info('Preparing fever evidence dataset') |
| 200 | beam_runner = direct_runner.DirectRunner() |
| 201 | download_config = tfds.download.DownloadConfig(beam_runner=beam_runner,) |
| 202 | builder.download_and_prepare( |
| 203 | download_dir=download_dir, download_config=download_config) |
| 204 | |
| 205 | |
| 206 | def wiki_preprocess(*, |
no outgoing calls
no test coverage detected
searching dependent graphs…