get the index from scan results or cache Args: file_list: load indexer only from the file lists if provided, default as None force_rebuild: If set true, rebuild and load index, default as False, indexer_file_dir: The dir where the indexer file saved, default as INDEXER_F
(
file_list=None,
force_rebuild=False,
indexer_file_dir=INDEXER_FILE_DIR,
indexer_file=INDEXER_FILE,
)
| 681 | |
| 682 | |
| 683 | def load_index( |
| 684 | file_list=None, |
| 685 | force_rebuild=False, |
| 686 | indexer_file_dir=INDEXER_FILE_DIR, |
| 687 | indexer_file=INDEXER_FILE, |
| 688 | ): |
| 689 | """get the index from scan results or cache |
| 690 | |
| 691 | Args: |
| 692 | file_list: load indexer only from the file lists if provided, default as None |
| 693 | force_rebuild: If set true, rebuild and load index, default as False, |
| 694 | indexer_file_dir: The dir where the indexer file saved, default as INDEXER_FILE_DIR |
| 695 | indexer_file: The indexer file name, default as INDEXER_FILE |
| 696 | Returns: |
| 697 | dict: the index information for all registered modules, including key: |
| 698 | index, requirements, files last modified time, modelscope home path, |
| 699 | version and md5, the detail is shown below example: { |
| 700 | 'index': { |
| 701 | ('MODELS', 'nlp', 'bert'):{ |
| 702 | 'filepath' : 'path/to/the/registered/model', 'imports': |
| 703 | ['os', 'torch', 'typing'] 'module': |
| 704 | 'modelscope.models.nlp.bert' |
| 705 | }, |
| 706 | ... |
| 707 | }, 'requirements': { |
| 708 | 'modelscope.models.nlp.bert': ['os', 'torch', 'typing'], |
| 709 | 'modelscope.models.nlp.structbert': ['os', 'torch', 'typing'], |
| 710 | ... |
| 711 | }, 'files_mtime' : { |
| 712 | '/User/Path/To/Your/Modelscope/modelscope/preprocessors/nlp/text_generation_preprocessor.py': |
| 713 | 16554565445, ... |
| 714 | },'version': '0.2.3', 'md5': '8616924970fe6bc119d1562832625612', |
| 715 | 'modelscope_path': '/User/Path/To/Your/Modelscope' |
| 716 | } |
| 717 | """ |
| 718 | # env variable override |
| 719 | cache_dir = os.getenv('MODELSCOPE_CACHE', indexer_file_dir) |
| 720 | index_file = os.getenv('MODELSCOPE_INDEX_FILE', indexer_file) |
| 721 | file_path = os.path.join(cache_dir, index_file) |
| 722 | index = None |
| 723 | |
| 724 | if force_rebuild: |
| 725 | logger.info('Force rebuilding ast index from scanning every file!') |
| 726 | index = file_scanner.get_files_scan_results(file_list) |
| 727 | return index |
| 728 | |
| 729 | # when developing, we need to generator as need. |
| 730 | if __is_develop_model(): |
| 731 | logger.info(f'Loading ast index from {file_path}') |
| 732 | if os.path.exists(file_path): # already exist, check it's latest |
| 733 | wrapped_index = _load_index(file_path) |
| 734 | md5, files_mtime = file_scanner.files_mtime_md5( |
| 735 | file_list=file_list) |
| 736 | index = wrapped_index |
| 737 | from modelscope.version import __version__ |
| 738 | if (wrapped_index[VERSION_KEY] == __version__ |
| 739 | and wrapped_index[MD5_KEY] != md5) or \ |
| 740 | wrapped_index[VERSION_KEY] != __version__: |
searching dependent graphs…