| 71 | } |
| 72 | |
| 73 | func doWork(id int, c *y.Closer, work <-chan interface{}, outDir string) { |
| 74 | defer c.Done() |
| 75 | |
| 76 | fd, err := os.Create(fmt.Sprintf("%s/twitter_feed_%d.json", outDir, id)) |
| 77 | if err != nil { |
| 78 | panic(err) |
| 79 | } |
| 80 | defer fd.Close() |
| 81 | |
| 82 | writer := bufio.NewWriter(fd) |
| 83 | var totalMessages, erroredMessages int |
| 84 | for message := range work { |
| 85 | totalMessages++ |
| 86 | |
| 87 | data, err := json.Marshal(message) |
| 88 | if err != nil { |
| 89 | fmt.Println("error in marshalling feed item ::", err) |
| 90 | erroredMessages++ |
| 91 | continue |
| 92 | } |
| 93 | |
| 94 | data = append(data, '\n') |
| 95 | if _, err := writer.Write(data); err != nil { |
| 96 | fmt.Println("error writing to file ::", err) |
| 97 | erroredMessages++ |
| 98 | continue |
| 99 | } |
| 100 | |
| 101 | if totalMessages%opts.ReportEveryTweets == 0 { |
| 102 | fmt.Printf("Routine: %d, Total: %d, error: %d\n", id, |
| 103 | totalMessages, erroredMessages) |
| 104 | } |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | func readCredentials(path string) (*twitterCreds, error) { |
| 109 | jsn, err := ioutil.ReadFile(path) |