Function get_file_map

scripts/release_notes/classifier.py:217–240

(data_folder: Path, regen_data, assert_stored=False)

def get_file_map(data_folder: Path, regen_data, assert_stored=False):
    if not regen_data and Path(data_folder / "file_map.pkl").exists():
        with open(data_folder / "file_map.pkl", "rb") as f:
            return pickle.load(f)
    else:
        if assert_stored:
            raise FileNotFoundError(
                "File map not found, you are loading for inference you need to have a file map!"
            )
        print("Regenerating File Map")
        all_data = pd.read_csv(data_folder / "commitlist.csv")
        # Lets explore files
        files = all_data.files_changed.to_list()

        all_files = []
        for file in files:
            paths = [Path(file_part) for file_part in file.split(" ")]
            all_files.extend(paths)
        all_files.append(Path(UNKNOWN_TOKEN))
        file_set = build_file_set(all_files, MAX_LEN_FILE)
        file_map = {file: i for i, file in enumerate(file_set)}
        with open(data_folder / "file_map.pkl", "wb") as f:
            pickle.dump(file_map, f)
        return file_map


#  Generate a dataset for training