(crawlerDB, nodeDB *sql.DB)
| 95 | } |
| 96 | |
| 97 | func transferNewNodes(crawlerDB, nodeDB *sql.DB) error { |
| 98 | crawlerDBTx, err := crawlerDB.Begin() |
| 99 | if err != nil { |
| 100 | // Sometimes error occur trying to read the crawler database, but |
| 101 | // they are normally recoverable, and a lot of the time, it's |
| 102 | // because the database is locked by the crawler. |
| 103 | return fmt.Errorf("error starting transaction to read nodes: %w", err) |
| 104 | } |
| 105 | defer crawlerDBTx.Rollback() |
| 106 | |
| 107 | nodes, err := crawlerdb.ReadAndDeleteUnseenNodes(crawlerDBTx) |
| 108 | if err != nil { |
| 109 | // Simiar to nodeDB.Begin() error |
| 110 | return fmt.Errorf("error reading nodes: %w", err) |
| 111 | } |
| 112 | |
| 113 | if len(nodes) > 0 { |
| 114 | err := apidb.InsertCrawledNodes(nodeDB, nodes) |
| 115 | if err != nil { |
| 116 | // This shouldn't happen because the database is not shared in this |
| 117 | // instance, so there shouldn't be lock errors, but anything can |
| 118 | // happen. We will still try again. |
| 119 | return fmt.Errorf("error inserting nodes: %w", err) |
| 120 | } |
| 121 | log.Info("Nodes inserted", "len", len(nodes)) |
| 122 | } |
| 123 | |
| 124 | crawlerDBTx.Commit() |
| 125 | return nil |
| 126 | } |
| 127 | |
| 128 | // newNodeDaemon reads new nodes from the crawler and puts them in the db |
| 129 | // Might trigger the invalidation of caches for the api in the future |
no test coverage detected