Method insert

src/clusterfuzz/_internal/google_cloud_utils/big_query.py:364–396

Insert the given rows and batch them if needed. Args: inserts: a list of Inserts, each of which represents a row. Returns: A json explained here: https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll

(self, inserts)

  @environment.local_noop
  def insert(self, inserts):
    # pylint: disable=line-too-long
    """Insert the given rows and batch them if needed.

    Args:
        inserts: a list of Inserts, each of which represents a row.

    Returns:
      A json explained here:
      https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
    """
    result = {}
    for i in range(0, len(inserts), INSERT_BATCH_SIZE):
      response = self._insert_batch(
          inserts[i:min(len(inserts), i + INSERT_BATCH_SIZE)])

      # There is also quota for the total number of requests / rows per second.
      time.sleep(1)

      if not result:
        # Use result from the first batch, appending errors from the rest.
        result = response
        continue

      # If there are new errors from the current batch, append to the result.
      new_errors = response.get('insertErrors')
      if not new_errors:
        continue

      # Apparently result may not have errors, use |setdefault| to be careful.
      result.setdefault('insertErrors', []).extend(new_errors)

    return result