Remove duplicates and remove hosts that we are excluding. We check for duplicate hostnames as well as remove any hostnames that have been explicitly excluded by the user. Parameters ---------- mergefile : file The file object that contains the hostnames that we are
(mergefile, exclusionregexes, outputfile=None)
| 917 | |
| 918 | |
| 919 | def remove_dups_and_excl(mergefile, exclusionregexes, outputfile=None): |
| 920 | """ |
| 921 | Remove duplicates and remove hosts that we are excluding. |
| 922 | |
| 923 | We check for duplicate hostnames as well as remove any hostnames that |
| 924 | have been explicitly excluded by the user. |
| 925 | |
| 926 | Parameters |
| 927 | ---------- |
| 928 | mergefile : file |
| 929 | The file object that contains the hostnames that we are pruning. |
| 930 | exclusionregexes : list |
| 931 | The list of regex patterns used to exclude domains. |
| 932 | outputfile : file |
| 933 | The file object in which the result is written. If None, the file |
| 934 | 'settings["outputpath"]' will be created. |
| 935 | """ |
| 936 | |
| 937 | numberofrules = settings["numberofrules"] |
| 938 | maybe_copy_example_file(settings["whitelistfile"]) |
| 939 | |
| 940 | if os.path.isfile(settings["whitelistfile"]): |
| 941 | with open(settings["whitelistfile"], "r") as ins: |
| 942 | for line in ins: |
| 943 | line = line.strip(" \t\n\r") |
| 944 | if line and not line.startswith("#"): |
| 945 | settings["exclusions"].append(line) |
| 946 | |
| 947 | if not os.path.exists(settings["outputpath"]): |
| 948 | os.makedirs(settings["outputpath"]) |
| 949 | |
| 950 | if outputfile is None: |
| 951 | finalfile = open(path_join_robust(settings["outputpath"], "hosts"), "w+b") |
| 952 | else: |
| 953 | finalfile = outputfile |
| 954 | |
| 955 | # analyze any post.json here |
| 956 | post_json_path = os.path.join(os.path.dirname(finalfile.name), "post.json") |
| 957 | filters = [] |
| 958 | if os.path.isfile(post_json_path): |
| 959 | try: |
| 960 | with open(post_json_path, "r", encoding="UTF-8") as post_file: |
| 961 | post_data = json.load(post_file) |
| 962 | filters = post_data.get("filters", []) |
| 963 | except Exception as e: |
| 964 | print_failure(f"Error reading post.json: {e}") |
| 965 | |
| 966 | mergefile.seek(0) # reset file pointer |
| 967 | hostnames = {"localhost", "localhost.localdomain", "local", "broadcasthost"} |
| 968 | exclusions = settings["exclusions"] |
| 969 | |
| 970 | for line in mergefile.readlines(): |
| 971 | write_line = True |
| 972 | |
| 973 | # Explicit encoding |
| 974 | line = line.decode("UTF-8") |
| 975 | |
| 976 | # Apply post.json filters |
no test coverage detected