| 49 | } |
| 50 | |
| 51 | struct BERTLayer { |
| 52 | explicit BERTLayer(NPZLoader params, int64_t n_heads) { |
| 53 | // define layer network here |
| 54 | attention_.reset(new layers::BertAttention( |
| 55 | params["attention.qkv.weight"], params["attention.qkv.bias"], |
| 56 | params["attention.output.dense.weight"], |
| 57 | params["attention.output.dense.bias"], |
| 58 | params["attention.output.LayerNorm.weight"], |
| 59 | params["attention.output.LayerNorm.bias"], n_heads)); |
| 60 | intermediate_.reset( |
| 61 | new layers::BertIntermediate(params["intermediate.dense.weight"], |
| 62 | params["intermediate.dense.bias"])); |
| 63 | output_.reset(new layers::BertOutput( |
| 64 | params["output.dense.weight"], params["output.dense.bias"], |
| 65 | params["output.LayerNorm.weight"], params["output.LayerNorm.bias"])); |
| 66 | } |
| 67 | |
| 68 | void operator()(core::Tensor &hidden, core::Tensor &mask, |
| 69 | core::Tensor *attention_out, core::Tensor *intermediate_out, |
| 70 | core::Tensor *output) { |
| 71 | (*attention_)(hidden, mask, attention_out); |
| 72 | (*intermediate_)(*attention_out, intermediate_out); |
| 73 | (*output_)(*intermediate_out, *attention_out, output); |
| 74 | } |
| 75 | |
| 76 | std::unique_ptr<layers::BertAttention> attention_; |
| 77 | std::unique_ptr<layers::BertIntermediate> intermediate_; |
| 78 | std::unique_ptr<layers::BertOutput> output_; |
| 79 | }; |
| 80 | |
| 81 | struct BertModel::Impl { |
| 82 | explicit Impl(const std::string &filename, DLDeviceType device_type, |
nothing calls this directly
no outgoing calls
no test coverage detected