| 9 | _KAGGLE_INPUT_DIR = '/kaggle/input' |
| 10 | |
| 11 | class KaggleDatasets: |
| 12 | GET_GCS_PATH_ENDPOINT = '/requests/CopyDatasetVersionToKnownGcsBucketRequest' |
| 13 | TIMEOUT_SECS = 600 |
| 14 | |
| 15 | # Integration types for GCS |
| 16 | AUTO_ML = 1 |
| 17 | TPU = 2 |
| 18 | |
| 19 | def __init__(self): |
| 20 | self.web_client = KaggleWebClient() |
| 21 | self.has_tpu = os.getenv(_KAGGLE_TPU_NAME_ENV_VAR_NAME) is not None |
| 22 | self.has_tpuvm = os.getenv(_KAGGLE_TPUVM_NAME_ENV_VAR_NAME) is not None |
| 23 | |
| 24 | def get_gcs_path(self, dataset_dir: str = None) -> str: |
| 25 | if self.has_tpuvm: |
| 26 | if dataset_dir is None: |
| 27 | onlydirs = [f for f in listdir(_KAGGLE_INPUT_DIR) if isdir(join(_KAGGLE_INPUT_DIR, f))] |
| 28 | if len(onlydirs) == 1: |
| 29 | dataset_dir = onlydirs[0] |
| 30 | else: |
| 31 | raise Exception("Could not infer dataset_dir. dataset_dir can only be inferred if there is exactly 1 Kaggle dataset attached.") |
| 32 | dataset = join(_KAGGLE_INPUT_DIR, dataset_dir) |
| 33 | print("get_gcs_path is not required on TPU VMs which can directly use Kaggle datasets, using path: " + dataset, file=sys.stderr) |
| 34 | return dataset |
| 35 | |
| 36 | integration_type = self.TPU if self.has_tpu else self.AUTO_ML |
| 37 | data = { |
| 38 | 'MountSlug': dataset_dir, |
| 39 | 'IntegrationType': integration_type, |
| 40 | } |
| 41 | result = self.web_client.make_post_request(data, self.GET_GCS_PATH_ENDPOINT, self.TIMEOUT_SECS) |
| 42 | return result['destinationBucket'] |
no outgoing calls