Function uploadToIndex

src/modules/localVectorStore.ts:234–324

(
  file: File,
  progressCallback: (progress: number) => Promise<void> | void
)

const models = useCachedModels();

async function uploadToIndex(
  file: File,
  progressCallback: (progress: number) => Promise<void> | void
) {
  const txt = await loadFile(file);
  const txthash = hashCode(txt || '');
  let maxsteps = 0;
  let steps = 0;
  //console.log(txt)
  console.log(`processing ${file.name}`);
  if (txt && documentStore) {
    let output: Document[] = [];
    if (file.type == 'text/csv') {
      output = splitCSVIntoLines(txt).map(
        (txtLine, index) =>
          new Document({
            pageContent: txtLine,
            metadata: {
              filename: file.name,
              source: 'file upload',
              line: index,
            },
          })
      );
    } else {
      output = await splitter.splitDocuments([
        new Document({
          pageContent: txt,
          metadata: {
            filename: file.name,
            source: 'file upload',
          },
        }),
      ]);
    }

    // filter out empty lines etc...
    output = output.filter((doc) => doc.pageContent);

    // prepare data (vectorize) for ingestions
    maxsteps = output.length * 2 || 1;
    //const output = await splitter.createDocuments([txt], metadatas = [{ filename: file.name }]);
    const docvecs: idbDocument[] = [];
    for (let i = 0; i < output.length; i++) {
      const doc = output[i];
      //const uuids = uuidv4()
      //doc.metadata['uuid'] = uuid.to
      docvecs.push({
        document: doc,
        vector: (
          await models.vectorize(
            doc.pageContent,
            vecStoreUploaderConfigurationState.value.modelName
          )
        ).tolist()[0] as number[],
      });
      steps += 1;
      await progressCallback(steps / maxsteps);