Method parse

metaclip/metaclip1/cc_matching.py:196–214

(self, cc_file, verbose=False)

        self.lid = lid

    def parse(self, cc_file, verbose=False):
        data = []
        num_records = 0
        with gzip.open(cc_file) as fr:
            looking_for_json = False
            for line in fr:
                try:
                    line = line.decode().strip()
                except Exception as e:
                    print(e)
                    continue
                if line.startswith("WARC-Target-URI"):
                    target_uri = line[len("WARC-Target-URI: ") :]
                    looking_for_json = True
                if looking_for_json and line.startswith("{"):
                    looking_for_json = False
                    self.parse_json(line, target_uri, data)
                    num_records += 1
        return data

    def parse_json(self, line, target_uri, data):
        try: