MCPcopy Create free account
hub / github.com/antvis/AVA / sourceCode

Function sourceCode

src/interpreter/python/util/code.ts:6–68  ·  view source on GitHub ↗

Generate only the loader needed for this source.

(source: DataSourceConfig, protectedFile: boolean)

Source from the content-addressed store, hash-verified

4
5/** Generate only the loader needed for this source. */
6function sourceCode(source: DataSourceConfig, protectedFile: boolean): string {
7 const file = protectedFile ? '_file' : "options['path']";
8 const storage = protectedFile ? '' : ", storage_options=options.get('headers') or None";
9 switch (source.type) {
10 case 'json':
11 return "tables = {'data': pd.DataFrame(options['data'])}";
12 case 'json-file':
13 return `tables = {'data': pd.read_json(${file}, orient='records'${
14 protectedFile ? ', compression=_compression' : storage
15 })}`;
16 case 'excel': {
17 const selected = source.options.sheets?.map(({ sheet, range, header }) => {
18 const bounds = range === undefined ? undefined : excelRange(range);
19 return {
20 sheet,
21 read: {
22 header: header === false ? null : 0,
23 ...(bounds && {
24 usecols: `${bounds.firstColumn}:${bounds.lastColumn}`,
25 skiprows: bounds.firstRow - 1,
26 nrows: bounds.lastRow - bounds.firstRow + (header === false ? 1 : 0),
27 }),
28 },
29 };
30 });
31 return `with pd.ExcelFile(${file}${storage}) as workbook:
32 selected = json.loads(${JSON.stringify(JSON.stringify(selected ?? null))})
33 if selected is None:
34 selected = [{'sheet': sheet, 'read': {'header': 0}} for sheet in workbook.sheet_names]
35 tables = {sheet['sheet']: workbook.parse(sheet_name=sheet['sheet'], **sheet['read']) for sheet in selected}`;
36 }
37 case 'parquet':
38 return `tables = {'data': pd.read_parquet(${file}${storage})}`;
39 case 'csv':
40 case 'csv-file': {
41 const read = source.options.options ?? {};
42 const csv = {
43 sep: read.delim,
44 quotechar: read.quote,
45 comment: read.comment,
46 skiprows: read.skip,
47 na_values: read.nullstr,
48 decimal: read.decimal_separator,
49 thousands: read.thousands,
50 encoding: read.encoding,
51 names: read.names,
52 header: read.header === false || (read.names !== undefined && read.header !== true) ? null : 0,
53 keep_default_na: read.nullstr === undefined ? undefined : false,
54 dtype: read.all_varchar ? 'str' : undefined,
55 compression: protectedFile ? undefined : read.compression === 'none' ? null : read.compression,
56 doublequote: read.escape === undefined ? undefined : read.escape === (read.quote ?? '"'),
57 escapechar: read.escape === (read.quote ?? '"') ? undefined : read.escape,
58 storage_options: source.type === 'csv-file' && !protectedFile ? source.options.headers : undefined,
59 };
60 const input = source.type === 'csv' ? "io.StringIO(options['csv'])" : file;
61 return `tables = {'data': pd.read_csv(${input}${
62 protectedFile ? ', compression=_compression' : ''
63 }, **json.loads(${JSON.stringify(JSON.stringify(csv))}))}`;

Callers 1

executionCodeFunction · 0.85

Calls 1

excelRangeFunction · 0.90

Tested by

no test coverage detected