Function processRedisEmbeddings

cmd/repositoryembedder/main.go:337–442

(embClient *embedder.EmbeddingClient, tree *object.Tree, redisClient *redis.Client, redisearchClient *redisearch.Client, url string)

}

func processRedisEmbeddings(embClient *embedder.EmbeddingClient, tree *object.Tree, redisClient *redis.Client, redisearchClient *redisearch.Client, url string) {
	// Check for existing processed hashes
	existingHashes := make(map[string]bool)
	existingTreeHash, err := redisClient.Get(context.Background(), fmt.Sprintf("%s:embedding:tree", url)).Result()
	if err != nil && err != redis.Nil {
		log.Fatal(err)
	}
	if err == nil {
		existingHashesStr, err := redisClient.Get(context.Background(), fmt.Sprintf("%s:embedding:tree:%s", url, existingTreeHash)).Result()
		if err != nil && err != redis.Nil {
			log.Fatal(err)
		}
		for _, h := range strings.Split(existingHashesStr, ",") {
			existingHashes[h] = true
		}
	}

	hashes := make(map[string]bool)
	filesBatch := []embedder.CodeEmbeddingRequest{}
	treeIter := tree.Files()
	batchSize := 10
	count := 0

	for {
		fmt.Printf("Processing file\n")
		file, err := treeIter.Next()
		if err != nil {
			if err == io.EOF {
				break // No more files
			}
			log.Fatal(err)
		}

		// Skip unsupported file types
		ext := filepath.Ext(file.Name)
		if _, ok := supportedLanguages[ext]; !ok {
			fmt.Printf("Skipping file '%s' since it is not supported\n", file.Name)
			continue
		}

		hashes[file.Hash.String()] = true

		// Process only if the file hash is not in the existing hashes
		if !existingHashes[file.Hash.String()] {
			content, err := file.Contents()
			if err != nil {
				log.Fatal(err)
			}

			filesBatch = append(filesBatch, embedder.CodeEmbeddingRequest{
				Path:    file.Name,
				Content: content,
				Hash:    file.Hash.String(),
			})
			count++

			// Process in batches of 10
			if count >= batchSize {