| 3 | from configparser import ConfigParser |
| 4 | |
| 5 | def gen_ctor_code(): |
| 6 | kernel_code = "\n\ |
| 7 | #include \"ggml-bitnet.h\"\n\ |
| 8 | #define GGML_BITNET_MAX_NODES 8192\n\ |
| 9 | static bool initialized = false;\n\ |
| 10 | static bitnet_tensor_extra * bitnet_tensor_extras = nullptr;\n\ |
| 11 | static size_t bitnet_tensor_extras_index = 0;\n\ |
| 12 | static void * aligned_malloc(size_t size) {{\n\ |
| 13 | #if defined(_WIN32)\n\ |
| 14 | return _aligned_malloc(size, 64);\n\ |
| 15 | #else\n\ |
| 16 | void * ptr = nullptr;\n\ |
| 17 | posix_memalign(&ptr, 64, size);\n\ |
| 18 | return ptr;\n\ |
| 19 | #endif\n\ |
| 20 | }}\n\ |
| 21 | static void aligned_free(void * ptr) {{\n\ |
| 22 | #if defined(_WIN32)\n\ |
| 23 | _aligned_free(ptr);\n\ |
| 24 | #else\n\ |
| 25 | free(ptr);\n\ |
| 26 | #endif\n\ |
| 27 | }}\n\ |
| 28 | \n\ |
| 29 | void per_tensor_quant(int k, void* lut_scales_, void* b_) {{\n\ |
| 30 | bitnet_float_type* lut_scales = (bitnet_float_type*)lut_scales_;\n\ |
| 31 | bitnet_float_type* b = (bitnet_float_type*)b_;\n\ |
| 32 | #ifdef __ARM_NEON\n\ |
| 33 | float32x4_t temp_max = vdupq_n_f32(0);\n\ |
| 34 | for (int i=0; i < k / 4; i++) {{\n\ |
| 35 | float32x4_t vec_bs = vld1q_f32(b + 4 * i);\n\ |
| 36 | float32x4_t abssum = vabsq_f32(vec_bs);\n\ |
| 37 | temp_max = vmaxq_f32(abssum, temp_max);\n\ |
| 38 | }}\n\ |
| 39 | float32_t scales = 127 / vmaxvq_f32(temp_max);\n\ |
| 40 | *lut_scales = scales;\n\ |
| 41 | #elif defined __AVX2__\n\ |
| 42 | __m256 max_vec = _mm256_set1_ps(0.f);\n\ |
| 43 | const __m256 vec_sign = _mm256_set1_ps(-0.0f);\n\ |
| 44 | // #pragma unroll\n\ |
| 45 | for (int i = 0; i < k / 8; i++) {{\n\ |
| 46 | __m256 vec_b = _mm256_loadu_ps(b + i * 8);\n\ |
| 47 | __m256 vec_babs = _mm256_andnot_ps(vec_sign, vec_b);\n\ |
| 48 | max_vec = _mm256_max_ps(vec_babs, max_vec);\n\ |
| 49 | }}\n\ |
| 50 | __m128 max1 = _mm_max_ps(_mm256_extractf128_ps(max_vec, 1), _mm256_castps256_ps128(max_vec));\n\ |
| 51 | max1 = _mm_max_ps(max1, _mm_movehl_ps(max1, max1));\n\ |
| 52 | max1 = _mm_max_ss(max1, _mm_movehdup_ps(max1));\n\ |
| 53 | float scales = 127 / _mm_cvtss_f32(max1);\n\ |
| 54 | *lut_scales = scales;\n\ |
| 55 | #endif\n\ |
| 56 | }}\n\ |
| 57 | \n\ |
| 58 | void partial_max_reset(void* lut_scales_) {{\n\ |
| 59 | bitnet_float_type* lut_scales = (bitnet_float_type*)lut_scales_;\n\ |
| 60 | *lut_scales = 0.0;\n\ |
| 61 | }}\n\ |
| 62 | \n\ |