| 50 | , s_tmp_(s_tmp) {} |
| 51 | |
| 52 | void operator()(sycl::nd_item<2> it) const { |
| 53 | sycl::group g = it.get_group(); |
| 54 | const uint lidx = it.get_local_id(0); |
| 55 | const uint lidy = it.get_local_id(1); |
| 56 | |
| 57 | const uint zid = g.get_group_id(0) / groups_x_; |
| 58 | const uint wid = g.get_group_id(1) / groups_y_; |
| 59 | const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid; |
| 60 | const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid; |
| 61 | const uint xid = groupId_x * g.get_local_range(0) * lim_ + lidx; |
| 62 | const uint yid = groupId_y * g.get_local_range(1) + lidy; |
| 63 | |
| 64 | bool cond_yzw = (yid < oInfo_.dims[1]) && (zid < oInfo_.dims[2]) && |
| 65 | (wid < oInfo_.dims[3]); |
| 66 | |
| 67 | // if (!cond_yzw) return; // retire warps early TODO: move |
| 68 | |
| 69 | const Ti *iptr = in_acc_.get_pointer(); |
| 70 | To *optr = out_acc_.get_pointer(); |
| 71 | To *tptr = tmp_acc_.get_pointer(); |
| 72 | |
| 73 | iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + |
| 74 | yid * iInfo_.strides[1] + iInfo_.offset; |
| 75 | optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] + |
| 76 | yid * oInfo_.strides[1]; |
| 77 | tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] + |
| 78 | yid * tInfo_.strides[1]; |
| 79 | |
| 80 | To *sptr = s_val_.get_pointer() + lidy * (2 * DIMX_ + 1); |
| 81 | |
| 82 | common::Transform<Ti, To, op> transform; |
| 83 | common::Binary<To, op> binop; |
| 84 | |
| 85 | const To init = common::Binary<To, op>::init(); |
| 86 | int id = xid; |
| 87 | To val = init; |
| 88 | |
| 89 | const bool isLast = (lidx == (DIMX_ - 1)); |
| 90 | for (int k = 0; k < lim_; k++) { |
| 91 | if (isLast) s_tmp_[lidy] = val; |
| 92 | |
| 93 | bool cond = (id < oInfo_.dims[0]) && cond_yzw; |
| 94 | val = cond ? transform(iptr[id]) : init; |
| 95 | sptr[lidx] = val; |
| 96 | group_barrier(g); |
| 97 | |
| 98 | int start = 0; |
| 99 | for (int off = 1; off < DIMX_; off *= 2) { |
| 100 | if (lidx >= off) val = binop(val, sptr[(start - off) + lidx]); |
| 101 | start = DIMX_ - start; |
| 102 | sptr[start + lidx] = val; |
| 103 | |
| 104 | group_barrier(g); |
| 105 | } |
| 106 | |
| 107 | val = binop(val, s_tmp_[lidy]); |
| 108 | |
| 109 | if (inclusive_scan_) { |