Function processAndSaveEmbeddings

cmd/repositoryembedder/main.go:306–335

(embClient *embedder.EmbeddingClient, db *gorm.DB, filesBatch *[]embedder.CodeEmbeddingRequest, url string)

}

func processAndSaveEmbeddings(embClient *embedder.EmbeddingClient, db *gorm.DB, filesBatch *[]embedder.CodeEmbeddingRequest, url string) {
	embeddings, err := embClient.FetchEmbeddings(*filesBatch)
	if err != nil {
		log.Fatal(err)
	}
	for filePath, embs := range embeddings.Results {
		for _, emb := range embs.Embeddings {
			newEmb := database.CodeEmbedding{
				URL:        url,
				FileHash:   emb.FileHash,
				FilePath:   filePath,
				ChunkID:    emb.ChunkID,
				StartIndex: emb.StartIndex,
				EndIndex:   emb.EndIndex,
				Embedding:  pgvector.NewVector(emb.Embedding), // Assuming emb.Vector is the appropriate data structure
			}
			// Upsert operation using Clauses with ON CONFLICT
			if err := db.Clauses(clause.OnConflict{
				Columns: []clause.Column{{Name: "chunk_id"}, {Name: "file_hash"}, {Name: "url"}, {Name: "file_path"}}, // Columns part of the unique constraint
				DoUpdates: clause.Assignments(map[string]interface{}{ // Update these fields if there is a conflict
					"start_index": newEmb.StartIndex,
					"end_index":   newEmb.EndIndex,
					"embedding":   newEmb.Embedding,
				}),
			}).Create(&newEmb).Error; err != nil {
				log.Fatalf("failed to save or update embedding: %v", err)
			}
		}
	}
}

func processRedisEmbeddings(embClient *embedder.EmbeddingClient, tree *object.Tree, redisClient *redis.Client, redisearchClient *redisearch.Client, url string) {
	// Check for existing processed hashes