| 26 | |
| 27 | |
| 28 | class DB: |
| 29 | |
| 30 | def __init__( |
| 31 | self, |
| 32 | nrows, |
| 33 | dtype, |
| 34 | chunksize, |
| 35 | userandom, |
| 36 | datadir, |
| 37 | docompress=0, |
| 38 | complib="zlib", |
| 39 | ): |
| 40 | self.dtype = dtype |
| 41 | self.docompress = docompress |
| 42 | self.complib = complib |
| 43 | self.filename = "-".join( |
| 44 | [rdm_cod[userandom], "n" + nrows, "s" + chunksize, dtype] |
| 45 | ) |
| 46 | # Complete the filename |
| 47 | self.filename = "lookup-" + self.filename |
| 48 | if docompress: |
| 49 | self.filename += "-" + complib + str(docompress) |
| 50 | self.filename = datadir + "/" + self.filename + ".h5" |
| 51 | print("Processing database:", self.filename) |
| 52 | self.userandom = userandom |
| 53 | self.nrows = get_nrows(nrows) |
| 54 | self.chunksize = get_nrows(chunksize) |
| 55 | self.step = self.chunksize |
| 56 | self.scale = NOISE |
| 57 | |
| 58 | def get_db_size(self): |
| 59 | sout = subprocess.Popen( |
| 60 | "sync;du -s %s" % self.filename, shell=True, stdout=subprocess.PIPE |
| 61 | ).stdout |
| 62 | line = sout[0] |
| 63 | return int(line.split()[0]) |
| 64 | |
| 65 | def print_mtime(self, t1, explain): |
| 66 | mtime = clock() - t1 |
| 67 | print(f"{explain}: {mtime:.6f}") |
| 68 | print(f"Krows/s: {self.nrows / 1000 / mtime:.6f}") |
| 69 | |
| 70 | def print_db_sizes(self, init, filled): |
| 71 | array_size = (filled - init) / 1024 |
| 72 | print(f"Array size (MB): {array_size:.3f}") |
| 73 | |
| 74 | def open_db(self, remove=0): |
| 75 | if remove and Path(self.filename).is_file(): |
| 76 | Path(self.filename).unlink() |
| 77 | con = tb.open_file(self.filename, "a") |
| 78 | return con |
| 79 | |
| 80 | def create_db(self, verbose): |
| 81 | self.con = self.open_db(remove=1) |
| 82 | self.create_array() |
| 83 | init_size = self.get_db_size() |
| 84 | t1 = clock() |
| 85 | self.fill_array() |