Function load_multilingual_data

code/ASR/Adapter/data_load.py:184–346

(root_path, datasets, args, languages)

def load_multilingual_data(root_path, datasets, args, languages):
    def collate(minibatch):
        out = []
        for b in minibatch:
            fbanks = []
            tokens = []
            language = None
            for _, info in b:
                fbanks.append(
                    torch.tensor(
                        kaldiio.load_mat(
                            info["input"][0]["feat"].replace(
                                data_config[dataset]["prefix"], root_path
                            )
                        )
                    )
                )
                tokens.append(
                    torch.tensor([int(s) for s in info["output"][0]["tokenid"].split()])
                )
                if language is not None:
                    assert language == info['category']
                else:
                    language = info['category']
            ilens = torch.tensor([x.shape[0] for x in fbanks])
            out.append((
            pad_sequence(fbanks, batch_first=True, padding_value=0),
            ilens,
            pad_sequence(tokens, batch_first=True, padding_value=-1),
            language,
        ))
        return out[0] if len(out) == 1 else out
    idim = None
    odim_dict = {}
    mtl_train_json, mtl_dev_json, mtl_test_json = {}, {}, {}
    for idx, dataset in enumerate(datasets):
        language = dataset
        if language in low_resource_languages:
            template_key = "template100"
        else:
            template_key = "template150"
        data_config[dataset] = data_config[template_key].copy()   
        for key in ["train", "val", "test", "token"]:
            data_config[dataset][key] = data_config[template_key][key].replace("template", dataset)
        
        train_json = os.path.join(root_path, data_config[dataset]["train"])
        dev_json = (
            os.path.join(root_path, data_config[dataset]["val"])
            if data_config[dataset]["val"]
            else f"{root_path}/tmp_dev_set_{dataset}.json"
        )
        test_json = os.path.join(root_path, data_config[dataset]["test"])
        train_json, dev_json, test_json = load_json(train_json, dev_json, test_json)
        for key in train_json.keys():
            train_json[key]['category'] = language
        for key in dev_json.keys():
            dev_json[key]['category'] = language
        for key in test_json.keys():