var depthWidth = depthFrame.FrameDescription.Width;
var depthHeight = depthFrame.FrameDescription.Height;
ushort[] depthData = new ushort[depthWidth * depthHeight];

var colorWidth = colorFrame.FrameDescription.Width;
var colorHeight = colorFrame.FrameDescription.Height;
byte[] pixels = new byte[colorWidth * colorHeight * 4];

CameraSpacePoint[] cameraSpacePoints = new CameraSpacePoint[depthData.Length];
ColorSpacePoint[] colorSpacePoints = new ColorSpacePoint[depthData.Length];

depthFrame.CopyFrameDataToArray(depthData);
coordinateMapper.MapDepthFrameToCameraSpace(depthData, cameraSpacePoints);
coordinateMapper.MapDepthFrameToColorSpace(depthData, colorSpacePoints);

// Assuming RGBA format here
colorFrame.CopyConvertedFrameDataToArray(pixels, ColorImageFormat.Rgba);

for (var index = 0; index < depthData.Length; index++)
{
    var u = colorSpacePoints[index].X;
    var v = colorSpacePoints[index].Y;
    int pixelsBaseIndex = (int)(v * depthWidth + u);

    byte red   = pixels[4 * pixelsBaseIndex + 0];
    byte green = pixels[4 * pixelsBaseIndex + 1];
    byte blue  = pixels[4 * pixelsBaseIndex + 2];
    byte alpha = pixels[4 * pixelsBaseIndex + 3];
}

IndexOutOfRangeException: Index was outside the bounds of the array.

List<int> allIndex = new List<int>();
for (var index = 0; index < depthData.Length; index++)
{
    var u = colorpoints[index].X;
    var v = colorpoints[index].Y;
    int pixelsBaseIndex = (int)(v * depthWidth + u);

    allIndex.Add(pixelsBaseIndex);
}

var maxIndex = allIndex.Max();
var minIndex = allIndex.Min();
Console.WriteLine(minIndex);//Prints -2147483648
Console.WriteLine((maxIndex < pixels.Length) && (minIndex >= 0));//Prints False