Function build_index

src/licensedcode/cache.py:209–273

Return an index built from rules and licenses directories If ``index_all_languages`` is True, include texts and rules in all languages. Otherwise, only include the English license texts and rules (the default) If ``additional_directories`` is not None, we will include licenses and

(
    licenses_db=None,
    licenses_data_dir=None,
    rules_data_dir=None,
    index_all_languages=False,
    additional_directories=None,
)

def build_index(
    licenses_db=None,
    licenses_data_dir=None,
    rules_data_dir=None,
    index_all_languages=False,
    additional_directories=None,
):
    """
    Return an index built from rules and licenses directories

    If ``index_all_languages`` is True, include texts and rules in all languages.
    Otherwise, only include the English license texts and rules (the default)
    If ``additional_directories`` is not None, we will include licenses and rules
    from these additional directories in the returned index.
    """
    from licensedcode.index import LicenseIndex
    from licensedcode.models import get_license_dirs
    from licensedcode.models import get_rule_dirs
    from licensedcode.models import get_rules_from_multiple_dirs
    from licensedcode.models import get_all_spdx_key_tokens
    from licensedcode.models import get_license_tokens
    from licensedcode.models import licenses_data_dir as ldd
    from licensedcode.models import rules_data_dir as rdd
    from licensedcode.models import load_licenses_from_multiple_dirs
    from licensedcode.models import validate_ignorable_clues
    from licensedcode.legalese import common_license_words

    licenses_data_dir = licenses_data_dir or ldd
    rules_data_dir = rules_data_dir or rdd

    if not licenses_db:
        # combine the licenses in these additional directories with the licenses in the original DB
        additional_license_dirs = get_license_dirs(additional_dirs=additional_directories)
        # generate a single combined license db with all licenses
        licenses_db = load_licenses_from_multiple_dirs(
            builtin_license_data_dir=licenses_data_dir,
            additional_license_data_dirs=additional_license_dirs,
            with_deprecated=False,
        )

    # if we have additional directories, extract the rules from them
    additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories)
    validate_ignorable_clues(rule_directories=additional_rule_dirs, is_builtin=False)
    # then combine the rules in these additional directories with the rules in the original rules directory
    rules = get_rules_from_multiple_dirs(
        licenses_db=licenses_db,
        additional_rules_data_dirs=additional_rule_dirs,
        builtin_rule_data_dir=rules_data_dir,
    )

    legalese = common_license_words
    spdx_tokens = set(get_all_spdx_key_tokens(licenses_db))
    license_tokens = set(get_license_tokens())

    # only skip licenses to be indexed
    if not index_all_languages:
        rules = (r for r in rules if r.language == 'en')