Method __process_record

app/src/core/worker.py:217–289

(
            self, idx: int, record, uri_id: str, snapshot_id: str
    )

        return callables

    def __process_record(
            self, idx: int, record, uri_id: str, snapshot_id: str
    ):
        # Setup document; this precomputes ngrams and hash features
        document = Document(
            record.raw_content,
            domain=record.source_domain,
            precompute_ngrams=True,
            precompute_hash_features=True,
            dsir_buckets=self._dsir_buckets
        )

        # compute signals
        rp_v2_signals = {}
        for func in self._quality_signals:
            rp_v2_signals[func.field_name] = func(document)  # noqa

        # compute minhash signatures
        minhash_signatures = self._minhash.compute_banded_signatures(
            tokens=document.normalized_words
        )

        # compute document ids
        doc_id = f"{uri_id}/{idx}"
        doc_id_int = int.from_bytes(
            hashlib.sha1(doc_id.encode("utf-8")).digest()[:8],  # take 8 bytes
            byteorder=_BYTE_ORDER, signed=False
        )

        record_data = {
            "id": f"{uri_id}/{idx}",
            "id_int": doc_id_int,
        }

        metadata = {
            "cc_segment": record.cc_segment,
            "cc_net_source": uri_id,
            "url": record.url,
            "source_domain": record.source_domain,
            "language": record.language,
            "snapshot_id": snapshot_id
        }

        ccnet_quality_signals = {
            "ccnet_length": (
                (0, len(document), float(record.length)),
            ),
            "ccnet_original_length": (
                (0, len(document), float(record.original_length)),
            ),
            "ccnet_nlines": (
                (0, len(document), float(record.nlines)),
            ),
            "ccnet_original_nlines": (
                (0, len(document), float(record.original_nlines)),
            ),
            "ccnet_language_score": (
                (0, len(document), float(record.language_score)),