| 8 | |
| 9 | |
| 10 | class FileLoader(object): |
| 11 | |
| 12 | def __init__(self, directory, prefix, config_file='train.conf'): |
| 13 | directory = os.path.join(os.path.dirname(os.path.realpath(__file__)), directory) |
| 14 | self.directory = directory |
| 15 | self.prefix = prefix |
| 16 | self.params = {'gpu_use_dp': True} |
| 17 | with open(os.path.join(directory, config_file), 'r') as f: |
| 18 | for line in f.readlines(): |
| 19 | line = line.strip() |
| 20 | if line and not line.startswith('#'): |
| 21 | key, value = [token.strip() for token in line.split('=')] |
| 22 | if 'early_stopping' not in key: # disable early_stopping |
| 23 | self.params[key] = value if key != 'num_trees' else int(value) |
| 24 | |
| 25 | def load_dataset(self, suffix, is_sparse=False): |
| 26 | filename = self.path(suffix) |
| 27 | if is_sparse: |
| 28 | X, Y = load_svmlight_file(filename, dtype=np.float64, zero_based=True) |
| 29 | return X, Y, filename |
| 30 | else: |
| 31 | mat = np.loadtxt(filename, dtype=np.float64) |
| 32 | return mat[:, 1:], mat[:, 0], filename |
| 33 | |
| 34 | def load_field(self, suffix): |
| 35 | return np.loadtxt(os.path.join(self.directory, self.prefix + suffix)) |
| 36 | |
| 37 | def load_cpp_result(self, result_file='LightGBM_predict_result.txt'): |
| 38 | return np.loadtxt(os.path.join(self.directory, result_file)) |
| 39 | |
| 40 | def train_predict_check(self, lgb_train, X_test, X_test_fn, sk_pred): |
| 41 | gbm = lgb.train(self.params, lgb_train) |
| 42 | y_pred = gbm.predict(X_test) |
| 43 | cpp_pred = gbm.predict(X_test_fn) |
| 44 | np.testing.assert_allclose(y_pred, cpp_pred) |
| 45 | np.testing.assert_allclose(y_pred, sk_pred) |
| 46 | |
| 47 | def file_load_check(self, lgb_train, name): |
| 48 | lgb_train_f = lgb.Dataset(self.path(name), params=self.params).construct() |
| 49 | for f in ('num_data', 'num_feature', 'get_label', 'get_weight', 'get_init_score', 'get_group'): |
| 50 | a = getattr(lgb_train, f)() |
| 51 | b = getattr(lgb_train_f, f)() |
| 52 | if a is None and b is None: |
| 53 | pass |
| 54 | elif a is None: |
| 55 | assert np.all(b == 1), f |
| 56 | elif isinstance(b, (list, np.ndarray)): |
| 57 | np.testing.assert_allclose(a, b) |
| 58 | else: |
| 59 | assert a == b, f |
| 60 | |
| 61 | def path(self, suffix): |
| 62 | return os.path.join(self.directory, self.prefix + suffix) |
| 63 | |
| 64 | |
| 65 | class TestEngine(unittest.TestCase): |
no outgoing calls