Function InsertCrawledNodes

pkg/apidb/database.go:41–121

(db *sql.DB, crawledNodes []crawlerdb.CrawledNode)

}

func InsertCrawledNodes(db *sql.DB, crawledNodes []crawlerdb.CrawledNode) error {
	log.Info("Writing nodes to db", "len", len(crawledNodes))

	tx, err := db.Begin()
	if err != nil {
		return err
	}
	stmt, err := tx.Prepare(`
		INSERT INTO nodes(
			id,
			name,
			version_major,
			version_minor,
			version_patch,
			version_tag,
			version_build,
			version_date,
			os_name,
			os_architecture,
			language_name,
			language_version,
			last_crawled,
			country_name
		)
		VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
		ON CONFLICT(id) DO UPDATE
		SET
			name = excluded.name,
			version_major = excluded.version_major,
			version_minor = excluded.version_minor,
			version_patch = excluded.version_patch,
			version_tag = excluded.version_tag,
			version_build = excluded.version_build,
			version_date = excluded.version_date,
			os_name = excluded.os_name,
			os_architecture = excluded.os_architecture,
			language_name = excluded.language_name,
			language_version = excluded.language_version,
			last_crawled = excluded.last_crawled,
			country_name = excluded.country_name
		WHERE
			name = excluded.name
			OR excluded.name != "unknown"
	`)
	if err != nil {
		return err
	}

	// It's possible for us to have the same node scraped multiple times, so
	// we want to make sure when we are upserting, we get the most recent
	// scrape upserted last.
	sort.SliceStable(crawledNodes, func(i, j int) bool {
		return strings.Compare(crawledNodes[i].Now, crawledNodes[j].Now) < 0
	})

	for _, node := range crawledNodes {
		parsed := vparser.ParseVersionString(node.ClientType)
		if parsed != nil {