()
| 52 | } |
| 53 | |
| 54 | func (it *FilterTestSuite) TestNeverIncludeSubnets() { |
| 55 | t := it.T() |
| 56 | // set up file system interface |
| 57 | afs := afero.NewMemMapFs() |
| 58 | afs2 := afero.NewOsFs() |
| 59 | err := afero.WriteFile(afs, "testsuite_config.hjson", []byte(` |
| 60 | { |
| 61 | filtering: { |
| 62 | filter_external_to_internal: true, |
| 63 | internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"], |
| 64 | never_included_subnets: ["10.55.100.0/24"], |
| 65 | }, |
| 66 | threat_intel: { |
| 67 | online_feeds: ["https://feodotracker.abuse.ch/downloads/ipblocklist.txt"], |
| 68 | custom_feeds_directory: "./deployment/threat_intel_feeds" |
| 69 | }, |
| 70 | http_extensions_file_path: "../deployment/http_extensions_list.csv" |
| 71 | } |
| 72 | `), 0755) |
| 73 | require.NoError(t, err) |
| 74 | |
| 75 | cfg, err := config.ReadFileConfig(afs, "testsuite_config.hjson") |
| 76 | require.NoError(t, err) |
| 77 | cfg.Env.DBConnection = dockerInfo.clickhouseConnection |
| 78 | it.cfg = cfg |
| 79 | require.Contains(t, cfg.Filtering.NeverIncludedSubnets, util.NewSubnet(&net.IPNet{IP: net.IP{10, 55, 100, 0}, Mask: net.IPMask{255, 255, 255, 0}})) |
| 80 | |
| 81 | // // import data |
| 82 | _, err = cmd.RunImportCmd(time.Now(), cfg, afs2, "../test_data/valid_tsv", "never_include_subnet", false, true) |
| 83 | require.NoError(t, err) |
| 84 | |
| 85 | // connect to database |
| 86 | db, err := database.ConnectToDB(context.Background(), "never_include_subnet", cfg, nil) |
| 87 | require.NoError(t, err) |
| 88 | |
| 89 | var count uint64 |
| 90 | |
| 91 | // verify that not all connections in 10.0.0.0/8 were filtered |
| 92 | err = db.Conn.QueryRow(db.GetContext(), ` |
| 93 | SELECT count(DISTINCT hash) FROM conn |
| 94 | WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104') |
| 95 | `).Scan(&count) |
| 96 | require.NoError(t, err) |
| 97 | require.EqualValues(t, 14452-12584, count, "conn table should contain 1868 entries in 10.0.0.0/8, got: %d", count) |
| 98 | |
| 99 | // 5531 in 10.55.100.0/24 |
| 100 | err = db.Conn.QueryRow(db.GetContext(), ` |
| 101 | SELECT count(DISTINCT hash) FROM ssl |
| 102 | WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104') |
| 103 | `).Scan(&count) |
| 104 | require.NoError(t, err) |
| 105 | require.EqualValues(t, 5615-(5531), count, "ssl table should contain 63 entries in 10.0.0.0/8, got: %d", count) |
| 106 | |
| 107 | err = db.Conn.QueryRow(db.GetContext(), ` |
| 108 | SELECT count(DISTINCT hash) FROM http |
| 109 | WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104') |
| 110 | `).Scan(&count) |
| 111 | require.NoError(t, err) |
nothing calls this directly
no test coverage detected