Function documents_from_csv_line

AskAgent/python/data_loading/db_load_utils.py:209–268

Parse a line with URL, JSON, and embedding into document objects. Args: line: Tab-separated line with URL, JSON, and embedding site: Site identifier Returns: List of document objects

(line, site)

        return [], []

def documents_from_csv_line(line, site):
    """
    Parse a line with URL, JSON, and embedding into document objects.

    Args:
        line: Tab-separated line with URL, JSON, and embedding
        site: Site identifier

    Returns:
        List of document objects
    """
    try:
        url, json_data, embedding_str = line.strip().split('\t')
        embedding_str = embedding_str.replace("[", "").replace("]", "")
        embedding = [float(x) for x in embedding_str.split(',')]
        js = json.loads(json_data)
        js = trim_schema_json(js, site)
    except Exception as e:
        print(f"Error processing line: {e!s}")
        return []

    # Skip if trim_schema_json returned None
    if js is None:
        return []

    documents = []
    if not isinstance(js, list):
        js = [js]

    for i, item in enumerate(js):
        # Skip None items in the list
        if item is None:
            continue

        # No longer filtering by should_include_item - trimming already handles this
        item_url = url if i == 0 else f"{url}#{i}"
        name = get_item_name(item)

        # Ensure no None values in the document
        doc = {
            "id": str(int64_hash(item_url)),
            "embedding": embedding,
            "schema_json": json.dumps(item),
            "url": item_url or "",
            "name": name or "Unnamed Item",
            "site": site or "unknown"
        }

        # Additional validation to ensure no None values
        for key, value in doc.items():
            if value is None:
                print(f"Warning: None value found for field '{key}' in document")
                if key == "embedding":
                    doc[key] = []
                else:
                    doc[key] = ""

        documents.append(doc)