HQDering mod v0.6

######
###
### HQDering mod v0.6 by mawen1250
###
### Requirements: masktools v2.0a48, RemoveGrain v1.0pre, dfttest v1.9.2, dither v1.21.0
###
### Applies deringing by using a smart smoother near edges (where ringing occurs) only.
###
######

Function HQDeringmod(clip input, clip "smoothed", int "Y", int "U", int "V", int "mthr", int "mrad", float "sigma", float "sigma2", int "sbsize", int "sosize", bool "sharp", int "drrep", bool "lsb_in", bool "lsb", int "dither") {

 HD       = input.width > 1024 || input.height > 576 ? true : false

 Y        = Default(Y,      3       )
 U        = Default(U,      2       )
 V        = Default(V,      2       )

 mthr     = Default(mthr,   HD?40:60)   # threshold of edge mask, lower  value means more aggressive processing
 mrad     = Default(mrad,   2       )   # radius    of ring mask, higher value means more aggressive processing

 sigma    = Default(sigma,  64.0    )   # dfttest: sigma
 sigma2   = Default(sigma2, sigma/8 )   # dfttest: sigma2
 sbsize   = Default(sbsize, HD?6:4  )   # dfttest: length of the sides of the spatial window
 sosize   = Default(sosize, HD?4:3  )   # dfttest: spatial overlap amount

 sharp    = Default(sharp,  false   )   # whether to use contra-sharpening to resharp deringed clip
 drrep    = Default(drrep,  0       )   # use repair for details retention, recommended values are 13/12/1

 lsb_in   = Default(lsb_in, false   )   # input  clip is 16-bit stacked or not
 lsb      = Default(lsb,    false   )   # output clip is 16-bit stacked or not
 dither   = Default(dither, 6       )   # dither mode for 8-bit output

 Y        = min(Y, 3)
 U        = min(U, 3)
 V        = min(V, 3)
 Yt       = Y == 3
 Ut       = U == 3
 Vt       = V == 3
 Y31      = Yt ? 3 : 1
 U31      = Ut ? 3 : 1
 V31      = Vt ? 3 : 1

 input8   = lsb_in ? input.DitherPost(mode=0, Y=Y, U=U, V=V) : input
 input16  = lsb_in ? input                                   : input.Dither_convert_8_to_16()

 smoothed = Defined(smoothed) ? lsb_in ? smoothed : smoothed.Dither_convert_8_to_16()
 \        : input16.dfttest(Y=Yt, U=Ut, V=Vt, sigma=sigma, sigma2=sigma2, sbsize=sbsize, sosize=sosize, tbsize=1, lsb_in=true, lsb=true)

 pre           = smoothed .Dither_removegrain16(Yt ? 4  : -1, Ut ? 4  : -1, Vt ? 4  : -1)
 method        = pre      .Dither_removegrain16(Yt ? 11 : -1, Ut ? 11 : -1, Vt ? 11 : -1)
 sharpdiff     = pre      .Dither_sub16(method,   Y=Y31, U=U31, V=V31, dif=true)
 allD          = input16  .Dither_sub16(smoothed, Y=Y31, U=U31, V=V31, dif=true)
 ssDD          = sharpdiff.Dither_repair16(allD, Yt ? 1 : -1, Ut ? 1 : -1, Vt ? 1 : -1)
 ssDD          = ssDD     .HQDeringmod_limitdiff16(sharpdiff, Y=Y31, U=U31, V=V31)
 sclp16        = sharp     ?
 \               smoothed .Dither_add16(ssDD,     Y=Y31, U=U31, V=V31, dif=true) : smoothed
 sclp8         = sclp16   .DitherPost(mode=dither, Y=Y31, U=U31, V=V31)

 sclp16        =  lsb ? drrep > 0 ? input16.Dither_repair16(sclp16, Yt ? drrep : -1, Ut ? drrep : -1, Vt ? drrep : -1) : sclp16 : NOP()
 sclp8         = !lsb ? drrep > 0 ? input8 .       Repair  (sclp8,  Yt ? drrep : -1, Ut ? drrep : -1, Vt ? drrep : -1) : sclp8  : NOP()

 prewittm      = input8.mt_edge("prewitt", mthr, 255, 0, 0, V=1, U=1)
 fmask         = mt_hysteresis(prewittm.RemoveGrain(4, -1), prewittm, U=1, V=1)
 omask         = mrad > 1 ? fmask.mt_expand(U=1, V=1) : fmask
 omask         = mrad > 2 ? omask.mt_expand(U=1, V=1) : omask
 omask         = mrad > 3 ? omask.mt_expand(U=1, V=1) : omask
 omask         = mrad > 4 ? omask.mt_expand(U=1, V=1) : omask
 imask         = fmask.mt_inflate(U=1, V=1).mt_inpand(U=1, V=1)
 ringmaskl     = omask.mt_lutxy(imask, "x 255 y - * 255 /", U=1, V=1)
 ringmask      = Ut || Vt ? ringmaskl.HQDeringmod_YtoYUV : ringmaskl

 return lsb ? Dither_merge16_8(input16, sclp16, ringmask, luma=false, Y=Y, U=U, V=V)
 \          : mt_merge        (input8,  sclp8,  ringmask, luma=false, Y=U, U=U, V=V)

}


Function HQDeringmod_limitdiff16(clip diff1, clip diff2, int "Y", int "U", int "V")
{
Y          = Default(Y,        3)
U          = Default(U,        3)
V          = Default(V,        3)

abdiff1    = Dither_lut16(diff1, "x 32768 - abs", Y=Y==3?3:1, U=U==3?3:1, V=V==3?3:1)
abdiff2    = Dither_lut16(diff2, "x 32768 - abs", Y=Y==3?3:1, U=U==3?3:1, V=V==3?3:1)
abdiffdiff = Dither_sub16(abdiff1, abdiff2, Y=Y==3?3:1, U=U==3?3:1, V=V==3?3:1, dif=true)
bin        = Dither_lut16(abdiffdiff, "x 32768 <= 0 65535 ?", Y=Y==3?3:1, U=U==3?3:1, V=V==3?3:1)

return Dither_merge16(diff1, diff2, bin, luma=false, Y=Y, U=U, V=V)
}


Function HQDeringmod_YtoYUV(clip inputl, string "colorspace")
{
 sw          = inputl.width
 sh          = inputl.height
 wmod4       = (sw/4*4==sw) ? true : false
 hmod4       = (sh/4*4==sh) ? true : false

 icolorspace = inputl.YtoYUV_GetCSP
 ocolorspace = Defined(colorspace) ? colorspace : icolorspace

 try {
   inputp = inputl.ConvertToY8
   inputc = ocolorspace == "YV24" ? inputp
   \      : ocolorspace == "YV16" ? inputp.BicubicResize(sw/2, sh  , -0.50)
   \      :                         inputp.BicubicResize(sw/2, sh/2, -0.50)

   output = YtoUV(inputc, inputc, inputp)

 } catch ( error_msg ) {
   inputp = wmod4&&hmod4 ? inputl : inputl.PointResize(wmod4?sw:sw+2, hmod4?sh:sh+2, 0, 0, wmod4?sw:sw+2, hmod4?sh:sh+2)
   inputc = inputp.BicubicResize(sw/2, sh/2, -0.50)

   output = YtoUV(inputc, inputc, inputp)
   output = wmod4&&hmod4 ? output : output.Crop(0, 0, wmod4?0:-2, hmod4?0:-2)

 }

 return output

 Function YtoYUV_GetCSP(clip c) {
   return c.IsPlanar ? c.IsYV12 ? "YV12" :
   \                   c.IsYV16 ? "YV16" :
   \                   c.IsYV24 ? "YV24" : c.GetCSP_Y8_YV411 :
   \      c.IsYUY2   ? "YUY2"   :
   \      c.IsRGB32  ? "RGB32"  :
   \      c.IsRGB24  ? "RGB24"  : "Unknown"

   Function GetCSP_Y8_YV411(clip c) {
     try {
       c.UtoY
       csp = "YV411"
     } catch ( error_msg ) {
      csp = "Y8"
     }
     return csp
   }
 }
}


Function HQDeringmod_Spline36Resize(clip input, int "target_width", int "target_height", float "src_left", float "src_top", float "src_width", float "src_height", bool "chroma") {

 w             = input.width
 h             = input.height

 target_width  = Default(target_width,      w)
 target_height = Default(target_height,     h)
 src_left      = Default(src_left,          0)
 src_top       = Default(src_top,           0)
 src_width     = Default(src_width,         w)
 src_height    = Default(src_height,        h)
 chroma        = Default(chroma,         true)

 try {
   inputp = chroma ? input  : input .ConvertToY8
   resize = inputp.Spline36Resize(target_width, target_height, src_left, src_top, src_width, src_height)
   resize = chroma ? resize : resize.ConvertToYV12
 } catch ( error_msg ) {
   resize = input.Spline36Resize(target_width, target_height, src_left, src_top, src_width, src_height)
 }

  return resize
}