MXAO for Framework

//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//LICENSE AGREEMENT AND DISTRIBUTION RULES:
//1 Copyrights of the Master Effect exclusively belongs to author - Gilcher Pascal aka Marty McFly.
//2 Master Effect (the SOFTWARE) is DonateWare application, which means you may or may not pay for this software to the author as donation.
//3 If included in ENB presets, credit the author (Gilcher Pascal aka Marty McFly).
//4 Software provided "AS IS", without warranty of any kind, use it on your own risk.
//5 You may use and distribute software in commercial or non-commercial uses. For commercial use it is required to warn about using this software (in credits, on the box or other places). Commercial distribution of software as part of the games without author permission prohibited.
//6 Author can change license agreement for new versions of the software.
//7 All the rights, not described in this license agreement belongs to author.
//8 Using the Master Effect means that user accept the terms of use, described by this license agreement.
//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//For more information about license agreement contact me:
//https://www.facebook.com/MartyMcModding
//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//Copyright (c) 2009-2016 Gilcher Pascal aka Marty McFly
//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

#pragma message "                                                                                                        MXAO Ambient Occlusion with Indirect Lighting v1.2.Framework\n                                                                                                        visit site for news/updates\n                                                                                                        fb.com/martymcmodding\n                                                                                                        Copyright (c) 2008-2016 Marty McFly\n"

#include EFFECT_CONFIG(MartyMcFly)
#include "Common.fx"

#if USE_MXAO

namespace MartyMcFly
{

texture2D texSSAO	{ Width = BUFFER_WIDTH*fMXAOSizeScale; 	          Height = BUFFER_HEIGHT*fMXAOSizeScale;            Format = RGBA8; };
texture2D texDither  <string source = "bayer16x16.png";> { Width = 16;Height = 16;Format = R8;};
texture2D texLOD 	{ Width = BUFFER_WIDTH; 			  Height = BUFFER_HEIGHT; 			    Format = RGBA8; MipLevels = 5+fMXAOMipLevelIL;};
#if(bMXAOLowPrecisionEnable != 0)
texture2D texDepthLOD 	{ Width = BUFFER_WIDTH; 			  Height = BUFFER_HEIGHT;  			    Format = R16F;  MipLevels = 5+fMXAOMipLevelAO;};
#else
texture2D texDepthLOD 	{ Width = BUFFER_WIDTH; 			  Height = BUFFER_HEIGHT;  			    Format = R32F;  MipLevels = 5+fMXAOMipLevelAO;};
#endif
#if(bMXAOBackfaceCheckEnable != 0)
texture2D texNormal	{ Width = BUFFER_WIDTH;                           Height = BUFFER_HEIGHT; 		            Format = RGBA8; MipLevels = 5+fMXAOMipLevelIL;};
#else
texture2D texNormal	{ Width = BUFFER_WIDTH;                           Height = BUFFER_HEIGHT; 		            Format = RGBA8; 				  };
#endif

sampler2D SamplerLOD
{
	Texture = texLOD;
	MinFilter = LINEAR;
	MagFilter = LINEAR;
	MipFilter = LINEAR;
	AddressU = Clamp;
	AddressV = Clamp;
};

sampler2D SamplerDepthLOD
{
	Texture = texDepthLOD;
	MinFilter = LINEAR;
	MagFilter = LINEAR;
	MipFilter = LINEAR;
	AddressU = Clamp;
	AddressV = Clamp;
};

sampler2D SamplerNormal
{
	Texture = texNormal;
	MinFilter = LINEAR;
	MagFilter = LINEAR;
	MipFilter = LINEAR;
	AddressU = Clamp;
	AddressV = Clamp;
};

sampler2D SamplerSSAO
{
	Texture = texSSAO;
	MinFilter = LINEAR;
	MagFilter = LINEAR;
	MipFilter = LINEAR;
	AddressU = Clamp;
	AddressV = Clamp;
};

sampler2D SamplerDither
{
	Texture = texDither;
	MinFilter = POINT;
	MagFilter = POINT;
	MipFilter = POINT;
	AddressU = Wrap;
	AddressV = Wrap;
};

//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//
//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

float MXAOGetLinearDepth(float2 coords)
{
	float depth = tex2Dlod(ReShade::LinearizedDepth, float4(coords.xy,0,0)).x;
	return depth;
}

float3 MXAOGetPosition(float2 coords)
{
	float EyeDepth = MXAOGetLinearDepth(coords.xy)*RESHADE_DEPTH_LINEARIZATION_FAR_PLANE;
	return float3((coords.xy * 2.0 - 1.0)*EyeDepth,EyeDepth);
}

float3 MXAOGetPositionLOD(float2 coords, int mipLevel)
{
	float EyeDepth = tex2Dlod(SamplerDepthLOD, float4(coords.xy,0,mipLevel)).x;
	return float3((coords.xy * 2.0 - 1.0)*EyeDepth,EyeDepth);
}

float3 MXAOGetNormalFromDepth(float2 coords)
{
	float3 centerPos = MXAOGetPosition(coords.xy);
	float2 offs =  ReShade::PixelSize.xy*1.0;
	float3 ddx1 = MXAOGetPosition(coords.xy + float2(offs.x, 0)) - centerPos;
	float3 ddx2 = centerPos - MXAOGetPosition(coords.xy + float2(-offs.x, 0));

	float3 ddy1 = MXAOGetPosition(coords.xy + float2(0, offs.y)) - centerPos;
	float3 ddy2 = centerPos - MXAOGetPosition(coords.xy + float2(0, -offs.y));

	ddx1 = lerp(ddx1, ddx2, abs(ddx1.z) > abs(ddx2.z));
	ddy1 = lerp(ddy1, ddy2, abs(ddy1.z) > abs(ddy2.z));

	float3 normal = cross(ddy1, ddx1);

	return normalize(normal);
}

float4 MXAOGetBlurFactors(float2 coords)
{
	return float4(tex2Dlod(SamplerNormal, float4(coords.xy,0,0)).xyz*2.0-1.0,MXAOGetLinearDepth(coords.xy));
}

float MXAOGetBlurWeight(float r, float4 z, float4 z0)
{
	float normaldiff = distance(z.xyz,z0.xyz);
	float depthdiff = abs(z.w-z0.w);

	float depthfalloff = pow(saturate(1.0 - z0.w),3.0);
	float fresnelfactor  = saturate(min(-z0.z,-z.z));

	float normalweight = saturate(1.0-normaldiff * fMXAOBlurSharpness);
	float depthweight = saturate(1.0-depthdiff * RESHADE_DEPTH_LINEARIZATION_FAR_PLANE * fMXAOBlurSharpness * fresnelfactor * depthfalloff * 0.5);

	return min(depthweight,normalweight);
}

float2 MXAOGetRandom2FromCoord(float2 coords)
{
	coords *= 1000.0;
	float3 coords3 = frac(float3(coords.xyx) * 0.1031);
    	coords3 += dot(coords3.xyz, coords3.yzx+19.19);
    	return frac(float2((coords3.x + coords3.y)*coords3.z, (coords3.x+coords3.z)*coords3.y));
}

//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//
//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

void PS_AO_Pre(float4 vpos : SV_Position, float2 texcoord : TEXCOORD, out float4 color : SV_Target0, out float4 depth : SV_Target1, out float4 normal : SV_Target2)
{
	color = tex2D(ReShade::BackBuffer, texcoord.xy);
	depth = MXAOGetLinearDepth(texcoord.xy)*RESHADE_DEPTH_LINEARIZATION_FAR_PLANE;
	normal = MXAOGetNormalFromDepth(texcoord.xy).xyzz*0.5+0.5; // * 0.5 + 0.5; //packing into 2 components not possible.
}

void PS_AO_Gen(float4 vpos : SV_Position, float2 texcoord : TEXCOORD, out float4 res : SV_Target0)
{
	float3 ScreenSpaceNormals = MXAOGetNormalFromDepth(texcoord.xy); //tex2D(SamplerNormal, texcoord.xy).xyz * 2.0 - 1.0; //better to use best possible data than rounded texture values
	float3 ScreenSpacePosition = MXAOGetPositionLOD(texcoord.xy, 0);

	float scenedepth = ScreenSpacePosition.z / RESHADE_DEPTH_LINEARIZATION_FAR_PLANE;
	ScreenSpacePosition += ScreenSpaceNormals * scenedepth;

	float numSamples = lerp(iMXAOSampleCount,12,scenedepth / AO_FADE____END); //AO FADEOUT //12 is minimum acceptable sampling count and max(12,...) makes falloff ineffective for small sample counts.
	float2 SampleRadiusScaled  = fMXAOSampleRadius / (numSamples * ScreenSpacePosition.z * float2(1.0,  ReShade::PixelSize.x/ ReShade::PixelSize.y) * 0.6);
	float radialJitter = (MXAOGetRandom2FromCoord(texcoord.xy).x-0.5) * fMXAOSampleRandomization;

	float rotAngle = tex2D(SamplerDither, texcoord.xy * float2(BUFFER_WIDTH,BUFFER_HEIGHT) * fMXAOSizeScale * 0.0625).x;
	float mipFactor = SampleRadiusScaled.x*numSamples*19.0;

	float4 AOandGI = 0.0;

	float2x2 radialMatrix = float2x2(0.575,0.81815,-0.81815,0.575); //E.F
	float2 currentVector = float2(cos(rotAngle*6.283), sin(rotAngle*6.283));

	float fNegInvR2 = -1.0/(fMXAOSampleRadius*fMXAOSampleRadius);

	[loop]
    	for (float i=1.0; i <= numSamples; i++)
	{
		currentVector = mul(currentVector.xy,  radialMatrix);
		float2 currentOffset = texcoord.xy + currentVector.xy * SampleRadiusScaled.xy * (i+radialJitter);

		#if(bMXAOBoundaryCheckEnable != 0)
		[branch]
		if(currentOffset.x < 1.0 && currentOffset.y < 1.0 && currentOffset.x > 0.0 && currentOffset.y > 0.0)
		{
		#endif
			float mipLevel = clamp((int)floor(log2(mipFactor*i)) - 3, fMXAOMipLevelAO, 5); //AO must not go beyond 5

			float3 occlVec = MXAOGetPositionLOD(currentOffset.xy, mipLevel) - ScreenSpacePosition;
			float occlDistance = length(occlVec);
 			float SurfaceAngle = dot(occlVec/occlDistance, ScreenSpaceNormals);

			float fAO = saturate(occlDistance * fNegInvR2 + 1.0)  * saturate(SurfaceAngle - fMXAONormalBias);

			#if(bMXAOIndirectLightingEnable != 0)
				float3 fIL = tex2Dlod(SamplerLOD, float4(currentOffset,0,mipLevel + fMXAOMipLevelIL)).xyz;
				#if(bMXAOBackfaceCheckEnable != 0)
					float3 offsetNormals = tex2Dlod(SamplerNormal, float4(currentOffset,0,mipLevel + fMXAOMipLevelIL)).xyz * 2.0 - 1.0;
					float facingtoSource = dot(-normalize(occlVec),offsetNormals);
					facingtoSource = smoothstep(-0.5,0.0,facingtoSource);
					fIL *= facingtoSource;
				#endif
				AOandGI.w += fAO*saturate(1-dot(fIL,float3(0.299,0.587,0.114)));
				AOandGI.xyz += fIL*fAO;
			#else
				AOandGI.w += fAO;
			#endif

		#if(bMXAOBoundaryCheckEnable != 0)
		}
		#endif
	}
	#define noinfomessages showfps
	AOandGI *= 20.0 / ((1.0-fMXAONormalBias)*numSamples*fMXAOSampleRadius);
	res = lerp(AOandGI,float4(0.0.xxx,0.0), AO_FADE____END < scenedepth); //AO FADEOUT
}

void PS_AO_Blur1(float4 vpos : SV_Position, float2 texcoord : TEXCOORD, out float4 res : SV_Target0)
{
	float4 center_factor = MXAOGetBlurFactors(texcoord.xy);
	float4 temp_factor = 0.0;

	float totalweight = 1.0;
	float tempweight = 0.0;

	float4 total_ao = tex2Dlod(SamplerSSAO, float4(texcoord.xy,0,0));
	float4 temp_ao = 0.0;

	[loop]
	for(float r = 1.0; r <= fMXAOBlurSteps; r += 1.0)
	{
		float2 axis = float2(-r,r)/fMXAOSizeScale*1.25;

		temp_factor = MXAOGetBlurFactors(texcoord.xy + axis *  ReShade::PixelSize.xy);
		temp_ao = tex2Dlod(SamplerSSAO, float4(texcoord.xy + axis *  ReShade::PixelSize.xy,0,0));
		tempweight = MXAOGetBlurWeight(r, temp_factor, center_factor);

		total_ao += temp_ao * tempweight;
		totalweight += tempweight;

		temp_factor = MXAOGetBlurFactors(texcoord.xy - axis *  ReShade::PixelSize.xy);
		temp_ao = tex2Dlod(SamplerSSAO, float4(texcoord.xy - axis *  ReShade::PixelSize.xy,0,0));
		tempweight = MXAOGetBlurWeight(r, temp_factor, center_factor);

		total_ao += temp_ao * tempweight;
		totalweight += tempweight;
	}

	total_ao /= totalweight;
	res = total_ao;
}

void PS_AO_Blur2(float4 vpos : SV_Position, float2 texcoord : TEXCOORD, out float4 res : SV_Target0)
{
	float4 center_factor = MXAOGetBlurFactors(texcoord.xy);
	float4 temp_factor = 0.0;

	float totalweight = 1.0;
	float tempweight = 0.0;

	float4 total_ao = tex2Dlod(ReShade::BackBuffer, float4(texcoord.xy,0,0));
	float4 temp_ao = 0.0;

	[loop]
	for(float r = 1.0; r <= fMXAOBlurSteps; r += 1.0)
	{
		float2 axis = float2(r,r)/fMXAOSizeScale*1.25;

		temp_factor = MXAOGetBlurFactors(texcoord.xy + axis *  ReShade::PixelSize.xy);
		temp_ao = tex2Dlod(ReShade::BackBuffer, float4(texcoord.xy + axis *  ReShade::PixelSize.xy,0,0));
		tempweight = MXAOGetBlurWeight(r, temp_factor, center_factor);

		total_ao += temp_ao * tempweight;
		totalweight += tempweight;

		temp_factor = MXAOGetBlurFactors(texcoord.xy - axis *  ReShade::PixelSize.xy);
		temp_ao = tex2Dlod(ReShade::BackBuffer, float4(texcoord.xy - axis *  ReShade::PixelSize.xy,0,0));
		tempweight = MXAOGetBlurWeight(r, temp_factor, center_factor);

		total_ao += temp_ao * tempweight;
		totalweight += tempweight;
	}

	total_ao /= totalweight;
	float4 mxao = saturate(total_ao);

	float scenedepth = MXAOGetLinearDepth(texcoord.xy); //might change center_factor so better fetch depth directly here.
	float4 color = max(0.0,tex2D(SamplerLOD, texcoord.xy));
	float colorgray = dot(color.xyz,float3(0.299,0.587,0.114));

	mxao.xyz  = lerp(dot(mxao.xyz,float3(0.299,0.587,0.114)),mxao.xyz,fMXAOIndirectLightingSaturation) * fMXAOIndirectLightingAmount;
	mxao.w    = 1.0-pow(1.0-mxao.w, fMXAOAmbientOcclusionAmount * 2.0);

#if(bMXAODebugViewEnable == 0)
	mxao    = lerp(mxao, 0.0, pow(colorgray,2.0));
#endif
	mxao.w    = lerp(mxao.w, 0.0,smoothstep(AO_FADE____START, AO_FADE____END, scenedepth)); 			//AO FADEOUT
	mxao.xyz  = lerp(mxao.xyz,0.0,smoothstep(AO_FADE____START*0.5, AO_FADE____END*0.5, scenedepth)); 		//AO FADEOUT //IL can look really bad on far objects.

	float3 GI = mxao.w - mxao.xyz;
	GI = max(0.0,1-GI);
	color.xyz *= GI;

#if(bMXAODebugViewEnable != 0)
 	#if(bMXAOIndirectLightingEnable != 0)
		color.xyz = (texcoord.x > 0.5) ? mxao.xyz : 1-mxao.w;
 	#else
 		color.xyz = 1-mxao.w;
 	#endif
#endif

	res = color;
}

technique MXAO_Tech <bool enabled = RESHADE_START_ENABLED; int toggle = MXAO_ToggleKey; >
{
	pass P0
	{
		VertexShader = ReShade::VS_PostProcess;
		PixelShader  = PS_AO_Pre;
		RenderTarget0 = texLOD;
		RenderTarget1 = texDepthLOD;
		RenderTarget2 = texNormal;
	}
	pass P1
	{
		VertexShader = ReShade::VS_PostProcess;
		PixelShader  = PS_AO_Gen;
		RenderTarget = texSSAO;
	}
	pass P2_0
	{
		VertexShader = ReShade::VS_PostProcess;
		PixelShader  = PS_AO_Blur1;
	}
	pass P2
	{
		VertexShader = ReShade::VS_PostProcess;
		PixelShader  = PS_AO_Blur2;
	}
}

}
#endif

#include EFFECT_CONFIG_UNDEF(MartyMcFly)