luma adaptive debanding mask for AVS & VS (AVS fix 6-4-19)

# meant as a faster version of the retinex-type deband mask
# lo and hi are the cutoffs for luma
# lothr and hithr are the Binarize thresholds of the mask at lo/hi luma levels
# luma values falling between lo and hi are scaled linearly from lothr to hithr

### Vapoursynth ###
import vapoursynth as vs
import havsfunc as haf

def debandmask(clip, lo=6144, hi=12288, lothr=320, hithr=384, mrad=2):
    core = vs.core

    f = clip.format
    bits = f.bits_per_sample
    isINT = f.sample_type==vs.INTEGER

    peak = (1 << bits) - 1 if isINT else 1
    clip = clip.std.ShufflePlanes(0, vs.GRAY)

    ma = haf.mt_expand_multi(clip, mode='ellipse', sw=mrad, sh=mrad)
    mi = haf.mt_inpand_multi(clip, mode='ellipse', sw=mrad, sh=mrad)

    rmask = core.std.Expr([ma, mi], 'x y -')

    mexpr = 'x {lo} < y {lothr} >= {peak} 0 ? x {hi} > y {hithr} >= {peak} 0 ? y x {lo} - {r} / {tr} * {lothr} + >= {peak} 0 ? ? ?'.format(lo=lo, hi=hi, lothr=lothr, hithr=hithr, peak=peak, r=hi-lo, tr=hithr-lothr)

    return core.std.Expr([clip, rmask], mexpr)


### Avisynth+ version, fixed by saiclabs from D-Z0N3 ###
Function debandmask(clip c, int lo, int hi, int  lothr, int  hithr, int  mrad, bool "stack16_in", bool "stack16_out")
{
	stack16_in = Default(stack16_in, false)
	stack16_out = Default(stack16_out, false)

	c
	Assert(lo >= 0 && hi >= 0 && lothr >= 0 && hithr >= 0, "lo/hi/lothr/hithr must be >= 0")
	Assert(lo <= 255 && hi <= 255 && lothr <= 255 && hithr <= 255, "lo/hi/lothr/hithr must be <= 255")

	stack16_in == false ? last : ConvertFromStacked()
	s = last
	ExtractY()

	rmask = Dither_build_gf3_range_mask (mrad)

	mexpr = "x "+String(lo)+" scaleb < y "+String(lothr)+" scaleb >= range_max 0 ? x "+String(hi)+" scaleb > y "+String(hithr)+" scaleb >= range_max  0 ? y x "+String(lo)+" scaleb - "+String(hi)+" scaleb "+String(lo)+" scaleb - / "+String(hithr)+" scaleb "+String(lothr)+" scaleb - * "+String(lothr)+" scaleb + >= range_max  0 ? ? ?"

	stack16_out ? BitsPerComponent(s) != 16 ? s.mt_lutxy(rmask, mexpr).ConvertBits(16).ConvertToStacked() : s.mt_lutxy(rmask, mexpr).ConvertToStacked() : s.mt_lutxy(rmask, mexpr)
}