| 7 | |
| 8 | |
| 9 | def read_data_from_csv(): |
| 10 | |
| 11 | # --- Cols 0 and 9 are timestamps --- |
| 12 | # --- Cols 6-8 are labels --- |
| 13 | |
| 14 | # TODO(ryancao)! |
| 15 | # --- Cols 15-17 are Bitcoin prices, but in the future :'( --- |
| 16 | |
| 17 | idx_to_field_data = list() |
| 18 | idx_to_field_labels = list() |
| 19 | data_features = list() |
| 20 | labels = list() |
| 21 | |
| 22 | with open(constants.RAW_DATA_FILE, newline="") as data_file: |
| 23 | data_reader = csv.reader(data_file, delimiter=",") |
| 24 | for idx, row in enumerate(data_reader): |
| 25 | if idx == 0: |
| 26 | idx_to_field_data = row[1:6] + row[10:] |
| 27 | idx_to_field_labels = row[6:9] |
| 28 | else: |
| 29 | # --- Data has some holes in it --- |
| 30 | if row[6] == "" or row[7] == "" or row[8] == "": |
| 31 | continue |
| 32 | skip = False |
| 33 | # --- Data has infinities in it --- |
| 34 | for x in row[1:9] + row[10:]: |
| 35 | if float(x) > 1e10: |
| 36 | skip = True |
| 37 | if skip: |
| 38 | continue |
| 39 | |
| 40 | row_features = list(float(x) for x in (row[1:6] + row[10:])) |
| 41 | row_labels = list(float(x) for x in row[6:9]) |
| 42 | data_features.append(row_features) |
| 43 | labels.append(row_labels) |
| 44 | |
| 45 | data_features = np.asarray(data_features) |
| 46 | labels = np.asarray(labels) |
| 47 | |
| 48 | # --- Check for infinities --- |
| 49 | for idx, row in enumerate(data_features): |
| 50 | if np.sum(row > 1e8) > 0: |
| 51 | print(idx, row) |
| 52 | print() |
| 53 | for idx, row in enumerate(labels): |
| 54 | if np.sum(row > 1e8) > 0: |
| 55 | print(idx, row) |
| 56 | |
| 57 | return idx_to_field_data, idx_to_field_labels, np.asarray(data_features), np.asarray(labels) |
| 58 | |
| 59 | |
| 60 | def preprocess_data(idx_to_field_data, idx_to_field_labels, data_features, labels): |