Try to determine if we have the IDs from previous searches of this log. This will allow us to limit log searches to only those records that have been added since the last matched certificate ID.
(
ct_collection, last_index_collection, source, job_number, starting_index
)
| 211 | |
| 212 | |
| 213 | def fetch_starting_index( |
| 214 | ct_collection, last_index_collection, source, job_number, starting_index |
| 215 | ): |
| 216 | """ |
| 217 | Try to determine if we have the IDs from previous searches of this log. |
| 218 | This will allow us to limit log searches to only those records that |
| 219 | have been added since the last matched certificate ID. |
| 220 | """ |
| 221 | |
| 222 | last_index_result = last_index_collection.find_one({"log_source": source}) |
| 223 | if last_index_result is not None: |
| 224 | if job_number == 0: |
| 225 | if last_index_result["last_index"] >= starting_index: |
| 226 | return last_index_result["last_index"] |
| 227 | else: |
| 228 | return starting_index |
| 229 | else: |
| 230 | index_name = "last_index_job_" + str(job_number) |
| 231 | if ( |
| 232 | index_name in last_index_result |
| 233 | and last_index_result[index_name] >= starting_index |
| 234 | ): |
| 235 | return last_index_result[index_name] |
| 236 | else: |
| 237 | return starting_index |
| 238 | |
| 239 | result = ( |
| 240 | ct_collection.find( |
| 241 | {source + "_id": {"$exists": True}}, {source + "_id": 1, "_id": 0} |
| 242 | ) |
| 243 | .sort([(source + "_id", -1)]) |
| 244 | .limit(1) |
| 245 | ) |
| 246 | |
| 247 | for entry in result: |
| 248 | return entry[source + "_id"] |
| 249 | |
| 250 | if starting_index > 0: |
| 251 | return starting_index |
| 252 | |
| 253 | return 0 |
| 254 | |
| 255 | |
| 256 | def read_leaf_header(leaf): |