parseTokenUsageFile parses a token-usage.jsonl file and returns the aggregated summary.
(filePath string)
| 132 | |
| 133 | // parseTokenUsageFile parses a token-usage.jsonl file and returns the aggregated summary. |
| 134 | func parseTokenUsageFile(filePath string) (*TokenUsageSummary, error) { |
| 135 | tokenUsageLog.Printf("Parsing token usage file: %s", filePath) |
| 136 | |
| 137 | file, err := os.Open(filePath) |
| 138 | if err != nil { |
| 139 | return nil, fmt.Errorf("failed to open token usage file: %w", err) |
| 140 | } |
| 141 | defer file.Close() |
| 142 | |
| 143 | summary := &TokenUsageSummary{ |
| 144 | ByModel: make(map[string]*ModelTokenUsage), |
| 145 | } |
| 146 | |
| 147 | scanner := bufio.NewScanner(file) |
| 148 | // Increase buffer size for potentially large lines |
| 149 | scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) |
| 150 | |
| 151 | entries := make([]TokenUsageEntry, 0) |
| 152 | lineNum := 0 |
| 153 | for scanner.Scan() { |
| 154 | lineNum++ |
| 155 | line := strings.TrimSpace(scanner.Text()) |
| 156 | if line == "" { |
| 157 | continue |
| 158 | } |
| 159 | |
| 160 | var entry TokenUsageEntry |
| 161 | if err := json.Unmarshal([]byte(line), &entry); err != nil { |
| 162 | tokenUsageLog.Printf("Skipping invalid JSON at line %d: %v", lineNum, err) |
| 163 | continue |
| 164 | } |
| 165 | entries = append(entries, entry) |
| 166 | } |
| 167 | |
| 168 | if err := scanner.Err(); err != nil { |
| 169 | return nil, fmt.Errorf("error reading token usage file: %w", err) |
| 170 | } |
| 171 | |
| 172 | if len(entries) == 0 { |
| 173 | tokenUsageLog.Print("No token usage entries found") |
| 174 | return nil, nil |
| 175 | } |
| 176 | |
| 177 | for _, entry := range entries { |
| 178 | // Aggregate totals |
| 179 | summary.TotalInputTokens += entry.InputTokens |
| 180 | summary.TotalOutputTokens += entry.OutputTokens |
| 181 | summary.TotalCacheReadTokens += entry.CacheReadTokens |
| 182 | summary.TotalCacheWriteTokens += entry.CacheWriteTokens |
| 183 | summary.TotalRequests++ |
| 184 | summary.TotalDurationMs += entry.DurationMs |
| 185 | summary.TotalResponseBytes += entry.ResponseBytes |
| 186 | |
| 187 | // Aggregate by model |
| 188 | model := entry.Model |
| 189 | if model == "" { |
| 190 | model = "unknown" |
| 191 | } |