Function build_generic

data/v2/build.py:109–129

(model_classes, file_name, csv_record_to_objects)

def build_generic(model_classes, file_name, csv_record_to_objects):
    batches = {}
    for model_class in model_classes:
        clear_table(model_class)
        batches[model_class] = []  # one batch per model class

    csv_data = load_data(file_name)
    next(csv_data, None)  # skip header

    for csv_record in csv_data:
        for obj in csv_record_to_objects(csv_record):
            model_class = type(obj)
            batches[model_class].append(obj)

            # Limit the batch size
            if len(batches[model_class]) > 200:
                model_class.objects.bulk_create(batches[model_class])
                batches[model_class] = []

    for model_class, batch in batches.items():
        model_class.objects.bulk_create(batch)


def scrub_str(string):