Function testGelu

experimental/legacy/transformer/test_kernels.cpp:133–166

}

void testGelu(Context &ctx) {
  static constexpr size_t N = 3072;
  std::array<float, N> inputArr;
  // range(inputArr);
  auto gen = std::mt19937(31415);
  // TODO(avh): investigate - on metal tanh seems to produce nan for values > 10
  randint(inputArr, gen, 0, 10); // for debugging
  std::array<float, N> outputArr;
  Tensor geluIn = createTensor(ctx, {N}, kf32, inputArr.data());
  Tensor geluOut = createTensor(ctx, {N}, kf32, outputArr.data());
  LOG(kDefLog, kInfo, "Creating GELU Shader");
  KernelCode shader = {kShaderGelu, 256, kf32};
  Kernel op = createKernel(ctx, shader, Bindings{geluIn, geluOut},
                           {cdiv(N, 256), 1, 1});
  LOG(kDefLog, kInfo, "Workgroup size: %s",
      toString(shader.workgroupSize).c_str());
  LOG(kDefLog, kInfo, "dispatching GELU Shader");
  std::promise<void> promise;
  std::future<void> future = promise.get_future();
  dispatchKernel(ctx, op, promise);
  wait(ctx, future);
  toCPU(ctx, geluOut, outputArr.data(), sizeof(outputArr));
  LOG(kDefLog, kInfo, "%s", show<float, N, 1>(inputArr, "GELU Input").c_str());
  LOG(kDefLog, kInfo, "%s",
      show<float, N, 1>(outputArr, "GELU Output").c_str());
  std::array<float, N> refOutputArr;
  ref::gelu_forward_cpu(refOutputArr.data(), inputArr.data(), N);
  LOG(kDefLog, kInfo, "%s",
      show<float, N, 1>(refOutputArr, "GELU Reference Output").c_str());
  bool passed = isclose(outputArr.data(), refOutputArr.data(), N);
  assert(passed);
  LOG(kDefLog, kInfo, "Gelu passed? %d", passed);
  LOG(kDefLog, kInfo, "Done with Gelu Test");
}

void testLayerNorm(Context &ctx) {
  struct LNParam {