var array = { 1, 2, 3, 4, 5 }
array = array.Where(item => item != 3).ToArray(); //retira o 4o. elemento

var array = { 1, 2, 3, 4, 5 }
array = = Array.FindAll(numbers, x => x != 3).ToArray(); //retira o 4o. elemento

public static T[] RemoveAt<T>(this T[] source, int index) {
    T[] dest = new T[source.Length - 1];
    if( index > 0 )
        Array.Copy(source, 0, dest, 0, index);

    if( index < source.Length - 1 )
        Array.Copy(source, index + 1, dest, index, source.Length - index - 1);

    return dest;
}

List/foreach: 3054ms (589725196)
Array/foreach: 1860ms (589725196)