Function processPostgresEmbeddings

cmd/repositoryembedder/main.go:241–304

(embClient *embedder.EmbeddingClient, tree *object.Tree, db *gorm.DB, url string)

}

func processPostgresEmbeddings(embClient *embedder.EmbeddingClient, tree *object.Tree, db *gorm.DB, url string) {
	// Check for existing processed hashes
	existingHashes := make(map[string]bool)
	// List all rows in the codeembedding table given the url
	var embeddings []database.CodeEmbedding
	if err := db.Where("url = ?", url).Find(&embeddings).Error; err != nil {
		log.Fatalf("failed to query existing embeddings: %v", err)
	}

	// Create a set of unique hashes
	for _, embedding := range embeddings {
		existingHashes[fmt.Sprintf("%s.%s", embedding.FileHash, embedding.FilePath)] = true
	}

	hashes := make(map[string]bool)
	filesBatch := []embedder.CodeEmbeddingRequest{}
	treeIter := tree.Files()
	batchSize := 10
	count := 0

	for {
		file, err := treeIter.Next()
		if err != nil {
			if err == io.EOF {
				break // No more files
			}
			log.Fatal(err)
		}

		// Skip unsupported file types
		ext := filepath.Ext(file.Name)
		if _, ok := supportedLanguages[ext]; !ok {
			fmt.Printf("Skipping file '%s' since it is not supported\n", file.Name)
			continue
		}

		hashes[fmt.Sprintf("%s.%s", file.Hash.String(), file.Name)] = true

		if !existingHashes[fmt.Sprintf("%s.%s", file.Hash.String(), file.Name)] {
			content, err := file.Contents()
			if err != nil {
				log.Fatal(err)
			}
			filesBatch = append(filesBatch, embedder.CodeEmbeddingRequest{
				Path:    file.Name,
				Content: content,
				Hash:    file.Hash.String(),
			})
			count++

			if count >= batchSize {
				processAndSaveEmbeddings(embClient, db, &filesBatch, url) // process embeddings
				filesBatch = []embedder.CodeEmbeddingRequest{}            // Reset the batch
				count = 0
			}
		} else {
			fmt.Printf("Skipping file '%s' since its hash is already processed\n", file.Name)
		}