| 65 | } |
| 66 | |
| 67 | func syncSource(src config.Source, dryRun bool) (SourceResult, error) { |
| 68 | repoDir := filepath.Join("sources", src.Name) |
| 69 | repoURL := "https://github.com/" + src.Repo + ".git" |
| 70 | |
| 71 | if _, err := os.Stat(repoDir); errors.Is(err, os.ErrNotExist) { |
| 72 | if dryRun { |
| 73 | return SourceResult{Name: src.Name, Commit: "dry-run", DownloadedFiles: 0}, nil |
| 74 | } |
| 75 | if err := run("git", "clone", "--depth=1", "--branch", src.Branch, repoURL, repoDir); err != nil { |
| 76 | return SourceResult{}, fmt.Errorf("clone %s: %w", src.Name, err) |
| 77 | } |
| 78 | } else { |
| 79 | if dryRun { |
| 80 | return SourceResult{Name: src.Name, Commit: "dry-run", DownloadedFiles: 0}, nil |
| 81 | } |
| 82 | if err := run("git", "-C", repoDir, "fetch", "--depth=1", "origin", src.Branch); err != nil { |
| 83 | return SourceResult{}, fmt.Errorf("fetch %s: %w", src.Name, err) |
| 84 | } |
| 85 | if err := run("git", "-C", repoDir, "checkout", "-B", src.Branch, "FETCH_HEAD"); err != nil { |
| 86 | return SourceResult{}, fmt.Errorf("checkout %s: %w", src.Name, err) |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | commitBytes, err := exec.Command("git", "-C", repoDir, "rev-parse", "HEAD").Output() |
| 91 | if err != nil { |
| 92 | return SourceResult{}, fmt.Errorf("rev-parse %s: %w", src.Name, err) |
| 93 | } |
| 94 | commit := strings.TrimSpace(string(commitBytes)) |
| 95 | |
| 96 | // Count .txt files without copying them |
| 97 | count := 0 |
| 98 | for _, p := range src.Paths { |
| 99 | root := repoDir |
| 100 | if p != "all" { |
| 101 | root = filepath.Join(repoDir, p) |
| 102 | } |
| 103 | if _, err := os.Stat(root); errors.Is(err, os.ErrNotExist) { |
| 104 | continue |
| 105 | } |
| 106 | _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { |
| 107 | if err != nil || d.IsDir() { |
| 108 | return err |
| 109 | } |
| 110 | if strings.HasSuffix(strings.ToLower(d.Name()), ".txt") { |
| 111 | count++ |
| 112 | } |
| 113 | return nil |
| 114 | }) |
| 115 | } |
| 116 | |
| 117 | return SourceResult{Name: src.Name, Commit: commit, DownloadedFiles: count}, nil |
| 118 | } |
| 119 | |
| 120 | func run(name string, args ...string) error { |
| 121 | attempts := 1 |