Function test_get_data_files_patterns

tests/test_data_files.py:653–682

(base_path, data_file_per_split)

    ],
)
def test_get_data_files_patterns(base_path, data_file_per_split):
    data_file_per_split = {k: v if isinstance(v, list) else [v] for k, v in data_file_per_split.items()}
    data_file_per_split = {
        split: [
            base_path + ("/" if base_path and base_path[-1] != "/" else "") + file_path
            for file_path in data_file_per_split[split]
        ]
        for split in data_file_per_split
    }
    file_paths = sum(data_file_per_split.values(), [])
    DummyTestFS = mock_fs(file_paths)
    fs = DummyTestFS()

    def resolver(pattern):
        pattern = base_path + ("/" if base_path and base_path[-1] != "/" else "") + pattern
        return [
            file_path[len(fs._strip_protocol(base_path)) :].lstrip("/")
            for file_path in fs.glob(pattern)
            if fs.isfile(file_path)
        ]

    patterns_per_split = _get_data_files_patterns(resolver)
    assert list(patterns_per_split.keys()) == list(data_file_per_split.keys())  # Test split order with list()
    for split, patterns in patterns_per_split.items():
        matched = [file_path for pattern in patterns for file_path in resolver(pattern)]
        expected = [
            fs._strip_protocol(file_path)[len(fs._strip_protocol(base_path)) :].lstrip("/")
            for file_path in data_file_per_split[split]
        ]
        assert matched == expected


def test_get_data_patterns_from_directory_with_the_word_data_twice(tmp_path):