Function process_binary_file

AskAgent/python/core/utils/trim_schema_json.py:157–229

Process a file in binary mode for cases where text encoding is problematic. Specifically handles UTF-16 encoded files.

(file_path, output_file)

        return result['encoding']

def process_binary_file(file_path, output_file):
    """
    Process a file in binary mode for cases where text encoding is problematic.
    Specifically handles UTF-16 encoded files.
    """
    try:
        with open(file_path, 'rb') as f:
            content = f.read()

            # Check for UTF-16 LE BOM
            if content.startswith(b'\xff\xfe'):
                encoding = 'utf-16-le'
                content = content[2:]  # Skip BOM
            # Check for UTF-16 BE BOM
            elif content.startswith(b'\xfe\xff'):
                encoding = 'utf-16-be'
                content = content[2:]  # Skip BOM
            else:
                # Default to UTF-16 LE if no BOM
                encoding = 'utf-16-le'

            try:
                text = content.decode(encoding)

                # Process lines
                lines = text.split('\n')
                with open(output_file, 'w', encoding='utf-8') as out:
                    for line in lines:
                        if not line.strip():
                            continue

                        parts = line.strip().split('\t')
                        if len(parts) != 2:
                            continue

                        url, schema_json_str = parts
                        site = url.split("/")[2].replace("www.", "").replace(".com", "")

                        # Clean up the JSON string - remove ^@ and similar artifacts
                        json_str = schema_json_str.replace("#N#", ' ').replace('^@', '')

                        # Process as normal
                        try:
                            schema_json = json.loads(json_str)
                            # Continue with normal processing
                            if not isinstance(schema_json, list):
                                continue

                            if isinstance(schema_json[0], list):
                                schema_json = schema_json[0]

                            trimmed_json = []
                            for item in schema_json:
                                try:
                                    trimmed_item = trim_schema_json(item, site)
                                    if trimmed_item is not None:
                                        trimmed_json.append(trimmed_item)
                                except Exception as e: