ddt + xbr edge detection + gamma correction

/*
   Hyllian's xBR+ddt+gamma Shader

   Copyright (C) 2011-2012 Hyllian/Jararaca - sergiogdb@gmail.com

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; either version 2
   of the License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

*/

const static float coef           = 2.0;
const static float3 dtt           = float3(65536,255,1);
const static half y_weight        = 48.0;
const static half u_weight        = 7.0;
const static half v_weight        = 6.0;
const static half3x3 yuv          = half3x3(0.299, 0.587, 0.114, -0.169, -0.331, 0.499, 0.499, -0.418, -0.0813);
const static half3x3 yuv_weighted = half3x3(y_weight*yuv[0], u_weight*yuv[1], v_weight*yuv[2]);

        // Constants used with gamma correction.
        #define InputGamma 2.4
        #define OutputGamma 2.2

        #define GAMMA_IN(color)     pow(color, float3(InputGamma))
        #define GAMMA_OUT(color)    pow(color, float3(1.0 / OutputGamma))

    #define TEX2D(coords)   GAMMA_IN( tex2D(decal, coords).rgb )


float reduce(half3 color)
{
    return dot(color, dtt);
}


half3 bilinear(float p, float q, half3 A, half3 B, half3 C, half3 D)
{
    return ((1-p)*(1-q)*A + p*(1-q)*B + (1-p)*q*C + p*q*D);
}

float RGBtoYUV(half3 color)
{
    return dot(color, yuv_weighted[0]);
}

float df(float A, float B)
{
    return abs(A-B);
}

bool eq(float A, float B)
{
    return (df(A, B) < 15.0);
}



float weighted_distance(float a, float b, float c, float d, float e, float f, float g, float h)
{
    return (df(a,b) + df(a,c) + df(d,e) + df(d,f) + 4.0*df(g,h));
}



struct input
{
    half2 video_size;
    float2 texture_size;
    half2 output_size;
};


struct out_vertex {
    half4 position : POSITION;
    half4 color    : COLOR;
    float2 texCoord : TEXCOORD0;
    half4 t1;
};

/*    VERTEX_SHADER    */
out_vertex main_vertex
(
    half4 position  : POSITION,
    half4 color : COLOR,
    float2 texCoord : TEXCOORD0,

    uniform half4x4 modelViewProj,
    uniform input IN
)
{
    out_vertex OUT;

    OUT.position = mul(modelViewProj, position);
    OUT.color = color;

    half2 ps = half2(1.0/IN.texture_size.x, 1.0/IN.texture_size.y);
    half dx = ps.x;
    half dy = ps.y;

    OUT.texCoord = texCoord;
    OUT.t1.xy = half2(  0,-dy); // B
    OUT.t1.zw = half2(-dx,  0); // D

    return OUT;
}

/*
    xBR LVL1 works over the pixels below:

        |B |C |
     |D |E |F |F4|
     |G |H |I |I4|
        |H5|I5|

    Consider E as the central pixel. xBR LVL1 needs only to look at 12 texture pixels.
*/

/*    FRAGMENT SHADER    */
float4 main_fragment(in out_vertex VAR, uniform sampler2D decal : TEXUNIT0, uniform input IN) : COLOR
{
    bool edr, px; // px = pixel, edr = edge detection rule

    float2 fx  = frac(VAR.texCoord*IN.texture_size);
    float2 pos = fx-float2(0.5); // pos = pixel position
    float2 dir = sign(pos); // dir = pixel direction

    float2 g1 = dir*VAR.t1.xy;
    float2 g2 = dir*VAR.t1.zw;

    half3 B = TEX2D(VAR.texCoord +g1   );
    half3 C = TEX2D(VAR.texCoord +g1-g2);
    half3 D = TEX2D(VAR.texCoord    +g2);
    half3 E = TEX2D(VAR.texCoord       );
    half3 F = TEX2D(VAR.texCoord    -g2);
    half3 G = TEX2D(VAR.texCoord -g1+g2);
    half3 H = TEX2D(VAR.texCoord -g1   );
    half3 I = TEX2D(VAR.texCoord -g1-g2);

    half3 F4 = TEX2D(VAR.texCoord    -2.0*g2   );
    half3 I4 = TEX2D(VAR.texCoord -g1-2.0*g2   );
    half3 H5 = TEX2D(VAR.texCoord -2.0*g1      );
    half3 I5 = TEX2D(VAR.texCoord -2.0*g1-g2   );


    float b = RGBtoYUV( B );
    float c = RGBtoYUV( C );
    float d = RGBtoYUV( D );
    float e = RGBtoYUV( E );
    float f = RGBtoYUV( F );
    float g = RGBtoYUV( G );
    float h = RGBtoYUV( H );
    float i = RGBtoYUV( I );

    float i4 = RGBtoYUV( I4 );
    float i5 = RGBtoYUV( I5 );
    float h5 = RGBtoYUV( H5 );
    float f4 = RGBtoYUV( F4 );

    edr = (weighted_distance( e, c, g, i, h5, f4, h, f) < weighted_distance( h, d, i5, f, i4, b, e, i));

    float p = abs(pos.x);
    float q = abs(pos.y);

    float k = distance(pos,g1);
    float l = distance(pos,g2);

    if ((abs(e-i) <= abs(f-h)) && !edr)
    {
        if (k < l)
        {
            H = E + I - F;
        }
        else if (k > l)
        {
            F = E + I - H;
        }
    }
    else if ((abs(e-i) >= abs(f-h)) && edr)
    {
        I = F + H - E;
    }

    float3 color = bilinear(p, q, E, F, H, I);

    return float4(clamp( GAMMA_OUT(color), 0.0, 1.0 ), 1.0);
}