Cel Shader

////////////////////////////////////////////////////////////////////////////////////////////////////
// file:    CelShader.fx
//
// summary: cel shader class
////////////////////////////////////////////////////////////////////////////////////////////////////

///////////////////////
//      Matrixs
///////////////////////

/// <summary> Same as with the ToonShader these store the matrixs used
///           for postioning the sprite. </summary>
matrix World;
matrix View;
matrix Projection;

///////////////////////
//      Textures
///////////////////////

/// <summary> When both techniques of the toon shader
///           are rendered to render tragets it will create
///           a basic Scene map which the end result will
///           multiply with.
///
///           The normal depth texture though can be sampled
///           to detect the edges. </summary>
Texture2D Scene;
Texture2D NormDepth;

///////////////////////
//  Global Variables
///////////////////////

/// <summary> These variables effect the intensity, strength and
///           depth of the cel shading
///           However I have not experimented on all of them yet
///           get a full understanding of their effects. </summary>
float edgeWidth = 1;

float edgeIntensity = 1;

float normalSensitivity = 1;

float depthSensitivity = 10;

float normalThreshold = 0.5;

float depthThreshold = 0.1;

/// <summary> This should not be set in the shader and should be
///           defined outside of the shader. for now though
///           I have set it to the size of my render target
///           (which was the full screen) </summary>
float2 screenResolution = float2(1280, 720);

///////////////////////
//  Sample state
///////////////////////

SamplerState SampleType
{
    Filter = MIN_MAG_MIP_LINEAR;
    AddressU = Wrap;
    AddressV = Wrap;
};

///////////////////////
// Inputs and outputs
// structures
///////////////////////

/// <summary> Since this shader is going to be used on a sprite
///           the vertes and pixel structure only need position and
///           texture coordinates. </summary>
struct PS_INPUT
{
    float4 Pos      : SV_POSITION;
    float2 tex      : TEXCOORD0;
};

struct VS_INPUT
{
    float4 Pos      : POSITION;
    float2 tex      : TEXCOORD0;
};

///////////////////////
//    VertexShader
///////////////////////

/// <summary> Standard vertex shader which positions the sprite and sends
///           the texture coord to the pixel shader </summary>
PS_INPUT VS( VS_INPUT input )
{
    PS_INPUT output;

    // Change the position vector to be 4 units for proper matrix calculations.
    input.Pos.w = 1.0f;

    output.Pos = mul( input.Pos, World );
    output.Pos = mul( output.Pos, View );
    output.Pos = mul( output.Pos, Projection );

    // Store the texture coordinates for the pixel shader.
    output.tex = input.tex;

    return output;
}

///////////////////////
//   PixelShader
///////////////////////
/// <summary> This is where all the cel shading and edge detection
///           happens. I dont understand all of what happens here
///           and have commeneted as much as I can. </summary>
float4 PS( PS_INPUT input ) : SV_Target
{

    /// <summary> The scene textue is sampled to get the colour
    ///           that will be mulitplied later </summary>
    float3 scene = Scene.Sample(SampleType, input.tex);


    /// <summary> This edge detection process will output the edges as black
    ///           and will produce white for the rest (i believe) </summary>

    float2 edgeOffset = edgeWidth / screenResolution;

    /// <summary> The normal map is sampled  </summary>
    float4 n1 = NormDepth.Sample(SampleType, input.tex + float2(-1, -1) * edgeOffset);

    float4 n2 = NormDepth.Sample(SampleType, input.tex + float2( 1, 1) * edgeOffset);

    float4 n3 = NormDepth.Sample(SampleType, input.tex + float2(-1, 1) * edgeOffset);

    float4 n4 = NormDepth.Sample(SampleType, input.tex + float2( 1, -1) * edgeOffset);

    float4 diagonalDelta = abs(n1 - n2) + abs(n3 - n4);

    float normalDelta = dot(diagonalDelta.xyz, 1);

    float depthDelta = diagonalDelta.w;

    normalDelta = saturate((normalDelta - normalThreshold) * normalSensitivity);

    depthDelta = saturate((depthDelta - depthThreshold) * depthSensitivity);

    float edgeAmount = saturate(normalDelta + depthDelta) * edgeIntensity;

    /// <summary> Once the edges are found and created it will then multiply it
    ///           with the Scene. now since the edges are black any colour
    ///           value multiplied with black will be black. </summary>
    scene *= (1 - edgeAmount);

    /// <summary> The final colour is then outputted </summary>
    return float4(scene, 1);

}


////////////////////////////////////////////////////////////////////////////////
// Technique
////////////////////////////////////////////////////////////////////////////////

technique10 render
{
    pass P0
    {
        SetVertexShader(CompileShader(vs_4_0, VS()));
        SetPixelShader (CompileShader(ps_4_0, PS()));
        SetGeometryShader(NULL);
    }
}

////////////////////////////////////////////////////////////////////////////////////////////////////
// End of CelShader.fx
////////////////////////////////////////////////////////////////////////////////////////////////////