HQDering mod v0.4

######
###
### HQDering mod v0.4 by mawen1250
###
### Requirements: masktools v2.0a48, RemoveGrain v1.0pre, dfttest v1.9.2, dither v1.21.0
###
### Applies deringing by using a smart smoother near edges (where ringing occurs) only.
###
######

Function HQDeringmod(clip input, clip "smoothed", int "mthr", int "mrad", float "sigma", float "sigma2", int "sbsize", int "sosize", bool "sharp", int "drrep", bool "lsb_in", bool "lsb", int "dither") {

 mthr     = Default(mthr,   80     )   # threshold of edge mask, lower  value means more aggressive processing
 mrad     = Default(mrad,   2      )   # radius    of ring mask, higher value means more aggressive processing

 sigma    = Default(sigma,  64.0   )   # dfttest: sigma
 sigma2   = Default(sigma2, sigma/8)   # dfttest: sigma2
 sbsize   = Default(sbsize, 4      )   # dfttest: length of the sides of the spatial window
 sosize   = Default(sosize, 3      )   # dfttest: spatial overlap amount

 sharp    = Default(sharp,  false  )   # whether to use contra-sharpening to resharp deringed clip
 drrep    = Default(drrep,  0      )   # use repair for details retention, recommended values are 13/12/1

 lsb_in   = Default(lsb_in, false  )   # input  clip is 16-bit stacked or not
 lsb      = Default(lsb,    false  )   # output clip is 16-bit stacked or not
 dither   = Default(dither, 6      )   # dither mode for 8-bit output

 input8   = lsb_in ? input.DitherPost(mode=0) : input
 input16  = lsb_in ? input                    : input.Dither_convert_8_to_16()

 smoothed = Defined(smoothed) ? lsb_in ? smoothed : smoothed.Dither_convert_8_to_16()
 \        : input16.dfttest(Y=true, U=false, V=false, sigma=sigma, sigma2=sigma2, sbsize=sbsize, sosize=sosize, tbsize=1, lsb_in=true, lsb=true)

 pre           = smoothed .Dither_removegrain16(4 , -1)
 method        = pre      .Dither_removegrain16(11, -1)
 sharpdiff     = pre      .Dither_sub16(method, U=1, V=1, dif=true)
 allD          = input16  .Dither_sub16(smoothed, U=1, V=1, dif=true)
 ssDD          = sharpdiff.Dither_repair16(allD, 1, -1)
 ssDD          = ssDD     .HQDeringmod_limitdiff16(sharpdiff, U=1, V=1)
 sclp16        = sharp     ?
 \               smoothed .Dither_add16(ssDD, U=1, V=1, dif=true) : smoothed
 sclp8         = sclp16   .DitherPost(mode=dither)

 sclp16        =  lsb ? drrep > 0 ? input16.Dither_repair16(sclp16, drrep, -1) : sclp16 : NOP()
 sclp8         = !lsb ? drrep > 0 ? input8 .       Repair  (sclp8,  drrep, -1) : sclp8  : NOP()

 prewittm      = input8.mt_edge("prewitt", mthr, 255, 0, 0, V=1, U=1)
 fmask         = mt_hysteresis(prewittm.RemoveGrain(4, -1), prewittm, U=1, V=1)
 omask         = mrad > 1 ? fmask.mt_expand(U=1, V=1) : fmask
 omask         = mrad > 2 ? omask.mt_expand(U=1, V=1) : omask
 omask         = mrad > 3 ? omask.mt_expand(U=1, V=1) : omask
 omask         = mrad > 4 ? omask.mt_expand(U=1, V=1) : omask
 imask         = fmask.mt_inflate(U=1, V=1).mt_inpand(U=1, V=1)
 ringmask      = omask.mt_lutxy(imask, "x 255 y - * 255 /", U=1, V=1)

 return lsb ? Dither_merge16_8(input16, sclp16, ringmask, luma=false, Y=3, U=2, V=2)
 \          : mt_merge        (input8,  sclp8,  ringmask, luma=false, Y=3, U=2, V=2)

}

Function HQDeringmod_limitdiff16(clip diff1, clip diff2, int "Y", int "U", int "V")
{
Y          = Default(Y,        3)
U          = Default(U,        3)
V          = Default(V,        3)

abdiff1    = Dither_lut16(diff1, "x 32768 - abs", Y=Y==3?3:1, U=U==3?3:1, V=V==3?3:1)
abdiff2    = Dither_lut16(diff2, "x 32768 - abs", Y=Y==3?3:1, U=U==3?3:1, V=V==3?3:1)
abdiffdiff = Dither_sub16(abdiff1, abdiff2, Y=Y==3?3:1, U=U==3?3:1, V=V==3?3:1, dif=true)
bin        = Dither_lut16(abdiffdiff, "x 32768 <= 0 65535 ?", Y=Y==3?3:1, U=U==3?3:1, V=V==3?3:1)

return Dither_merge16(diff1, diff2, bin, luma=false, Y=Y, U=U, V=V)
}