Function insert_next_page

server/db.js:257–292

* insert_next_page inserts the list of next pages to be crawled. * @param {Object} data - a valid JSON object containing data to be inserted. * @param {function} callback - callback function to be executed on success.

(data, callback)

 * @param {function} callback - callback function to be executed on success.
 */
function insert_next_page (data, callback) {
  let urls = []
  switch (data.type) {
    case 'org':
      // console.log('data.name', data.name);
      urls = data.entries.map((e) => e.url);
      urls.push('orgs/' + data.name + '/people'); // list of PUBLIC org members.
      urls.push(data.next_page); // if it exists.
      break;
    case 'profile':
      urls = data.pinned.map((e) => e.url);
      const orgs = Object.keys(data.orgs);
      orgs.forEach(org => urls.push(org));
      urls = profile_next_page(urls, data.username);
      break;
    case 'repo':
      urls.push(data.url + '/stargazers');
      break;
    case 'followers':
    case 'following':
    case 'people':
    case 'stars':
      urls.push(data.next_page);
      data.entries.forEach((e) => { urls = profile_next_page(urls, e.username)})
      break;
  }
  let len = urls.length;
  urls.filter((e) => e !== null) // filter out blanks (if next_page is null)
  .forEach((next_page, i) => { // poor person's "async parallel":
    insert_log_item(data.url, next_page, (error, data2) => {
      if(--len == 0) {
        return utils.exec_cb(callback, null, data);
      }
    })
  });
}

/**
 * select_next_page get the next url (page) to crawl