Generate only the loader needed for this source.
(source: DataSourceConfig, protectedFile: boolean)
| 4 | |
| 5 | /** Generate only the loader needed for this source. */ |
| 6 | function sourceCode(source: DataSourceConfig, protectedFile: boolean): string { |
| 7 | const file = protectedFile ? '_file' : "options['path']"; |
| 8 | const storage = protectedFile ? '' : ", storage_options=options.get('headers') or None"; |
| 9 | switch (source.type) { |
| 10 | case 'json': |
| 11 | return "tables = {'data': pd.DataFrame(options['data'])}"; |
| 12 | case 'json-file': |
| 13 | return `tables = {'data': pd.read_json(${file}, orient='records'${ |
| 14 | protectedFile ? ', compression=_compression' : storage |
| 15 | })}`; |
| 16 | case 'excel': { |
| 17 | const selected = source.options.sheets?.map(({ sheet, range, header }) => { |
| 18 | const bounds = range === undefined ? undefined : excelRange(range); |
| 19 | return { |
| 20 | sheet, |
| 21 | read: { |
| 22 | header: header === false ? null : 0, |
| 23 | ...(bounds && { |
| 24 | usecols: `${bounds.firstColumn}:${bounds.lastColumn}`, |
| 25 | skiprows: bounds.firstRow - 1, |
| 26 | nrows: bounds.lastRow - bounds.firstRow + (header === false ? 1 : 0), |
| 27 | }), |
| 28 | }, |
| 29 | }; |
| 30 | }); |
| 31 | return `with pd.ExcelFile(${file}${storage}) as workbook: |
| 32 | selected = json.loads(${JSON.stringify(JSON.stringify(selected ?? null))}) |
| 33 | if selected is None: |
| 34 | selected = [{'sheet': sheet, 'read': {'header': 0}} for sheet in workbook.sheet_names] |
| 35 | tables = {sheet['sheet']: workbook.parse(sheet_name=sheet['sheet'], **sheet['read']) for sheet in selected}`; |
| 36 | } |
| 37 | case 'parquet': |
| 38 | return `tables = {'data': pd.read_parquet(${file}${storage})}`; |
| 39 | case 'csv': |
| 40 | case 'csv-file': { |
| 41 | const read = source.options.options ?? {}; |
| 42 | const csv = { |
| 43 | sep: read.delim, |
| 44 | quotechar: read.quote, |
| 45 | comment: read.comment, |
| 46 | skiprows: read.skip, |
| 47 | na_values: read.nullstr, |
| 48 | decimal: read.decimal_separator, |
| 49 | thousands: read.thousands, |
| 50 | encoding: read.encoding, |
| 51 | names: read.names, |
| 52 | header: read.header === false || (read.names !== undefined && read.header !== true) ? null : 0, |
| 53 | keep_default_na: read.nullstr === undefined ? undefined : false, |
| 54 | dtype: read.all_varchar ? 'str' : undefined, |
| 55 | compression: protectedFile ? undefined : read.compression === 'none' ? null : read.compression, |
| 56 | doublequote: read.escape === undefined ? undefined : read.escape === (read.quote ?? '"'), |
| 57 | escapechar: read.escape === (read.quote ?? '"') ? undefined : read.escape, |
| 58 | storage_options: source.type === 'csv-file' && !protectedFile ? source.options.headers : undefined, |
| 59 | }; |
| 60 | const input = source.type === 'csv' ? "io.StringIO(options['csv'])" : file; |
| 61 | return `tables = {'data': pd.read_csv(${input}${ |
| 62 | protectedFile ? ', compression=_compression' : '' |
| 63 | }, **json.loads(${JSON.stringify(JSON.stringify(csv))}))}`; |
no test coverage detected