(data_source, repo_path)
| 982 | |
| 983 | |
| 984 | def _read_datasource(data_source, repo_path) -> dd.DataFrame: |
| 985 | storage_options = ( |
| 986 | { |
| 987 | "client_kwargs": { |
| 988 | "endpoint_url": data_source.file_options.s3_endpoint_override |
| 989 | } |
| 990 | } |
| 991 | if data_source.file_options.s3_endpoint_override |
| 992 | else None |
| 993 | ) |
| 994 | |
| 995 | path = FileSource.get_uri_for_file_path( |
| 996 | repo_path=repo_path, |
| 997 | uri=data_source.file_options.uri, |
| 998 | ) |
| 999 | |
| 1000 | return dd.read_parquet( |
| 1001 | path, |
| 1002 | storage_options=storage_options, |
| 1003 | ) |
| 1004 | |
| 1005 | |
| 1006 | def _run_dask_field_mapping( |
no test coverage detected