| 82 | // ====== Dataset ====== |
| 83 | |
| 84 | ggml_opt_dataset_t ggml_opt_dataset_init( |
| 85 | enum ggml_type type_data, |
| 86 | enum ggml_type type_label, |
| 87 | int64_t ne_datapoint, |
| 88 | int64_t ne_label, |
| 89 | int64_t ndata, |
| 90 | int64_t ndata_shard) { |
| 91 | GGML_ASSERT(ne_datapoint > 0); |
| 92 | GGML_ASSERT(ne_label >= 0); |
| 93 | GGML_ASSERT(ndata > 0); |
| 94 | GGML_ASSERT(ndata_shard > 0); |
| 95 | |
| 96 | ggml_opt_dataset_t result = new ggml_opt_dataset; |
| 97 | result->ndata = ndata; |
| 98 | result->ndata_shard = ndata_shard; |
| 99 | |
| 100 | { |
| 101 | struct ggml_init_params params = { |
| 102 | /*.mem_size =*/ 2*ggml_tensor_overhead(), |
| 103 | /*.mem_buffer =*/ nullptr, |
| 104 | /*.no_alloc =*/ true, |
| 105 | }; |
| 106 | result->ctx = ggml_init(params); |
| 107 | } |
| 108 | |
| 109 | result->data = ggml_new_tensor_2d(result->ctx, type_data, ne_datapoint, ndata); |
| 110 | result->nbs_data = ggml_nbytes(result->data) * ndata_shard/ndata; |
| 111 | |
| 112 | if (ne_label > 0) { |
| 113 | result->labels = ggml_new_tensor_2d(result->ctx, type_label, ne_label, ndata); |
| 114 | result->nbs_labels = ggml_nbytes(result->labels) * ndata_shard/ndata; |
| 115 | } else { |
| 116 | result->labels = nullptr; |
| 117 | result->nbs_labels = 0; |
| 118 | } |
| 119 | |
| 120 | result->buf = ggml_backend_alloc_ctx_tensors_from_buft(result->ctx, ggml_backend_cpu_buffer_type()); |
| 121 | |
| 122 | const int64_t nshards = ndata/ndata_shard; |
| 123 | result->permutation.resize(nshards); |
| 124 | for (int64_t i = 0; i < nshards; ++i) { |
| 125 | result->permutation[i] = i; |
| 126 | } |
| 127 | return result; |
| 128 | } |
| 129 | |
| 130 | void ggml_opt_dataset_free(ggml_opt_dataset_t dataset) { |
| 131 | ggml_backend_buffer_free(dataset->buf); |