(datasetId: string, csvString: string, firstRowHeaders: boolean)
| 132 | } |
| 133 | |
| 134 | const _csvToDatasetRows = async (datasetId: string, csvString: string, firstRowHeaders: boolean) => { |
| 135 | try { |
| 136 | const appServer = getRunningExpressApp() |
| 137 | // get the max value first |
| 138 | const maxValueEntity = await appServer.AppDataSource.getRepository(DatasetRow).find({ |
| 139 | order: { |
| 140 | sequenceNo: 'DESC' |
| 141 | }, |
| 142 | take: 1 |
| 143 | }) |
| 144 | let sequenceNo = 0 |
| 145 | if (maxValueEntity && maxValueEntity.length > 0) { |
| 146 | sequenceNo = maxValueEntity[0].sequenceNo |
| 147 | } |
| 148 | sequenceNo++ |
| 149 | // Array to hold parsed records |
| 150 | const results: any[] = [] |
| 151 | let files: string[] = [] |
| 152 | |
| 153 | if (csvString.startsWith('[') && csvString.endsWith(']')) { |
| 154 | files = JSON.parse(csvString) |
| 155 | } else { |
| 156 | files = [csvString] |
| 157 | } |
| 158 | |
| 159 | for (const file of files) { |
| 160 | const splitDataURI = file.split(',') |
| 161 | splitDataURI.pop() |
| 162 | const bf = Buffer.from(splitDataURI.pop() || '', 'base64') |
| 163 | const csvString = bf.toString('utf8') |
| 164 | |
| 165 | // Convert CSV string to a Readable stream |
| 166 | const stream = Readable.from(csvString) |
| 167 | const rows: any[] = [] |
| 168 | await _readCSV(stream, rows) |
| 169 | results.push(...rows) |
| 170 | } |
| 171 | if (results && results?.length > 0) { |
| 172 | for (let r = 0; r < results.length; r++) { |
| 173 | const row = results[r] |
| 174 | let input = '' |
| 175 | let output = '' |
| 176 | if (firstRowHeaders && r === 0) { |
| 177 | continue |
| 178 | } |
| 179 | input = row['0'] |
| 180 | output = row['1'] |
| 181 | const newRow = appServer.AppDataSource.getRepository(DatasetRow).create(new DatasetRow()) |
| 182 | newRow.datasetId = datasetId |
| 183 | newRow.input = input |
| 184 | newRow.output = output |
| 185 | newRow.sequenceNo = sequenceNo |
| 186 | await appServer.AppDataSource.getRepository(DatasetRow).save(newRow) |
| 187 | sequenceNo++ |
| 188 | } |
| 189 | } |
| 190 | } catch (error) { |
| 191 | throw new InternalFlowiseError( |
no test coverage detected