| 173 | } |
| 174 | |
| 175 | func getOtxUrls(domain string, noSubs bool) ([]wurl, error) { |
| 176 | var urls []wurl |
| 177 | page := 0 |
| 178 | for { |
| 179 | r, err := http.Get(fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/hostname/%s/url_list?limit=50&page=%d", domain, page)) |
| 180 | if err != nil { |
| 181 | return []wurl{}, err |
| 182 | } |
| 183 | bytes, err := ioutil.ReadAll(r.Body) |
| 184 | if err != nil { |
| 185 | return []wurl{}, err |
| 186 | } |
| 187 | r.Body.Close() |
| 188 | |
| 189 | wrapper := struct { |
| 190 | HasNext bool `json:"has_next"` |
| 191 | ActualSize int `json:"actual_size"` |
| 192 | URLList []struct { |
| 193 | Domain string `json:"domain"` |
| 194 | URL string `json:"url"` |
| 195 | Hostname string `json:"hostname"` |
| 196 | Httpcode int `json:"httpcode"` |
| 197 | PageNum int `json:"page_num"` |
| 198 | FullSize int `json:"full_size"` |
| 199 | Paged bool `json:"paged"` |
| 200 | } `json:"url_list"` |
| 201 | }{} |
| 202 | err = json.Unmarshal(bytes, &wrapper) |
| 203 | if err != nil { |
| 204 | return []wurl{}, err |
| 205 | } |
| 206 | for _, url := range wrapper.URLList { |
| 207 | urls = append(urls, wurl{url: url.URL}) |
| 208 | } |
| 209 | if !wrapper.HasNext { |
| 210 | break |
| 211 | } |
| 212 | page++ |
| 213 | } |
| 214 | return urls, nil |
| 215 | } |