| 41 | let args = {} as any; |
| 42 | |
| 43 | async function readData (dataFile: string) { |
| 44 | // Vectorize the data. |
| 45 | const inputTexts: string[] = []; |
| 46 | const targetTexts: string[] = []; |
| 47 | |
| 48 | const inputCharacters = new Set<string>(); |
| 49 | const targetCharacters = new Set<string>(); |
| 50 | |
| 51 | const fileStream = fs.createReadStream(dataFile); |
| 52 | const rl = readline.createInterface({ |
| 53 | input: fileStream, |
| 54 | output: process.stdout, |
| 55 | terminal: false, |
| 56 | }); |
| 57 | |
| 58 | let lineNumber = 0; |
| 59 | rl.on('line', line => { |
| 60 | if (++lineNumber > args.num_samples) { |
| 61 | rl.close(); |
| 62 | return; |
| 63 | } |
| 64 | |
| 65 | let [inputText, targetText] = line.split('\t'); |
| 66 | // We use "tab" as the "start sequence" character for the targets, and "\n" |
| 67 | // as "end sequence" character. |
| 68 | targetText = '\t' + targetText + '\n'; |
| 69 | |
| 70 | inputTexts.push(inputText); |
| 71 | targetTexts.push(targetText); |
| 72 | |
| 73 | for (const char of inputText) { |
| 74 | if (!inputCharacters.has(char)) { |
| 75 | inputCharacters.add(char); |
| 76 | } |
| 77 | } |
| 78 | for (const char of targetText) { |
| 79 | if (!targetCharacters.has(char)) { |
| 80 | targetCharacters.add(char); |
| 81 | } |
| 82 | } |
| 83 | }) |
| 84 | |
| 85 | await new Promise(r => rl.on('close', r)); |
| 86 | |
| 87 | const inputCharacterList = [...inputCharacters].sort(); |
| 88 | const targetCharacterList = [...targetCharacters].sort(); |
| 89 | |
| 90 | const numEncoderTokens = inputCharacterList.length; |
| 91 | const numDecoderTokens = targetCharacterList.length; |
| 92 | |
| 93 | // Math.max() does not work with very large arrays because of the stack limitation |
| 94 | const maxEncoderSeqLength = inputTexts.map(text => text.length) |
| 95 | .reduceRight((prev, curr) => curr > prev ? curr : prev, 0); |
| 96 | const maxDecoderSeqLength = targetTexts.map(text => text.length) |
| 97 | .reduceRight((prev, curr) => curr > prev ? curr : prev, 0); |
| 98 | |
| 99 | console.log('Number of samples:', inputTexts.length); |
| 100 | console.log('Number of unique input tokens:', numEncoderTokens); |