Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def conv2d(x, weight, bias=None, stride=1, padding=0, dilation=1, groups=1):
- if bias is None:
- bias = torch.zeros((weight.shape[0],), device=x.device, dtype=x.dtype)
- if isinstance(stride, int):
- stride = (stride, stride)
- if isinstance(padding, int):
- padding = (padding, padding)
- if isinstance(dilation, int):
- dilation = (dilation, dilation)
- N, Cin, H, W = x.shape
- Cout, Cin_per_w, Kh, Kw = weight.shape
- sh, sw = stride
- ph, pw = padding
- dh, dw = dilation
- assert Cin % groups == 0, "Cin must be divisible by groups"
- assert Cout % groups == 0, "Cout must be divisible by groups"
- assert Cin_per_w == (Cin // groups), "w.shape[1] must equal Cin/groups"
- Kh_eff = dh * (Kh - 1) + 1
- Kw_eff = dw * (Kw - 1) + 1
- Hout = (H + 2 * ph - Kh_eff) // sh + 1
- Wout = (W + 2 * pw - Kw_eff) // sw + 1
- y = torch.empty((N, Cout, Hout, Wout), device=x.device, dtype=x.dtype)
- s_x_n, s_x_c, s_x_h, s_x_w = x.stride()
- s_w_co, s_w_cinperg, s_w_kh, s_w_kw = weight.stride()
- s_y_n, s_y_c, s_y_h, s_y_w = y.stride()
- BLOCK_OC = 8
- BLOCK_HW = 128
- hw_tiles = triton.cdiv(Hout * Wout, BLOCK_HW)
- Cout_g = Cout // groups
- oc_tiles_per_group = triton.cdiv(Cout_g, BLOCK_OC)
- grid = (hw_tiles * oc_tiles_per_group, N, groups)
- conv2d_kernel[grid](
- x, weight, bias, y,
- N, Cin, H, W,
- Cout, Kh, Kw,
- Hout, Wout,
- sh, sw,
- ph, pw,
- dh, dw,
- groups,
- s_x_n, s_x_c, s_x_h, s_x_w,
- s_w_co, s_w_cinperg, s_w_kh, s_w_kw,
- s_y_n, s_y_c, s_y_h, s_y_w,
- BLOCK_OC=BLOCK_OC,
- BLOCK_HW=BLOCK_HW,
- num_warps=4,
- num_stages=1,
- )
- return y
- @triton.jit
- def conv2d_kernel(
- X_ptr, W_ptr, B_ptr, Y_ptr,
- N, Cin, H, W,
- Cout, Kh, Kw,
- Hout, Wout,
- sh, sw,
- ph, pw,
- dh, dw,
- groups,
- s_x_n, s_x_c, s_x_h, s_x_w,
- s_w_co, s_w_cinperg, s_w_kh, s_w_kw,
- s_y_n, s_y_c, s_y_h, s_y_w,
- BLOCK_OC: tl.constexpr,
- BLOCK_HW: tl.constexpr,
- ):
- pid0 = tl.program_id(0) # spatial * oc
- pid1 = tl.program_id(1) # batch index
- pid2 = tl.program_id(2) # group index
- Cin_g = Cin // groups
- Cout_g = Cout // groups
- hw_tiles = tl.cdiv(Hout * Wout, BLOCK_HW)
- oc_tiles_per_group = tl.cdiv(Cout_g, BLOCK_OC)
- hw_block = pid0 % hw_tiles
- oc_block_in_group = pid0 // hw_tiles
- oc_base = pid2 * Cout_g + oc_block_in_group * BLOCK_OC
- oc_offsets = oc_base + tl.arange(0, BLOCK_OC)
- hw_offsets = hw_block * BLOCK_HW + tl.arange(0, BLOCK_HW)
- h = hw_offsets // Wout
- w = hw_offsets % Wout
- mask_oc = (oc_offsets < (pid2 + 1) * Cout_g) & (oc_offsets < Cout)
- mask_hw = hw_offsets < Hout * Wout
- mask_n = pid1 < N
- mask_group = pid2 < groups
- acc = tl.full((BLOCK_OC, BLOCK_HW), 0.0, tl.float32)
- Xb = X_ptr + pid1 * s_x_n
- Yb = Y_ptr + pid1 * s_y_n
- cin_group_start = pid2 * Cin_g
- Wb = W_ptr + oc_offsets * s_w_co
- for icg in range(Cin_g):
- ic_base = Wb + icg * s_w_cinperg
- for ky in range(Kh):
- for kx in range(Kw):
- in_h = h * sh + ky * dh - ph
- in_w = w * sw + kx * dw - pw
- in_bounds = (in_h >= 0) & (in_h < H) & (in_w >= 0) & (in_w < W)
- x_ptrs = Xb + (cin_group_start + icg) * s_x_c + in_h * s_x_h + in_w * s_x_w
- x = tl.load(x_ptrs, mask=mask_hw & in_bounds & mask_n, other=0.0)
- w_ptrs = ic_base + ky * s_w_kh + kx * s_w_kw
- w_val = tl.load(w_ptrs, mask=mask_oc, other=0.0)
- acc += x[None, :] * w_val[:, None]
- bias = tl.load(B_ptr + oc_offsets, mask=mask_oc, other=0.0)
- acc += bias[:, None]
- out_ptrs = Yb + oc_offsets[:, None] * s_y_c + h[None, :] * s_y_h + w[None, :] * s_y_w
- tl.store(out_ptrs, acc, mask=mask_oc[:, None] & mask_hw & mask_n & mask_group)
Advertisement
Add Comment
Please, Sign In to add comment