C# SOCKS5 Proxy Implementation doesn't always work on local

/// <summary>
/// SOCK Reference is at https://www.ietf.org/rfc/rfc1928.txt
/// </summary>
public class SocksServer
{
    public void Start()
    {
        new Thread(StartInternal).Start();
    }

    private void StartInternal()
    {
        IPEndPoint localEndPoint = new IPEndPoint(IPAddress.Any, 32128);
        TcpListener listener = new TcpListener(localEndPoint);

        listener.Start();
        Console.Out.WriteLine("Waiting for clients...");

        while (true)
        {
            var client = listener.AcceptTcpClient();
            Console.Out.WriteLine("Connected");

            new Thread(() =>
            {
                var stream = client.GetStream();
                var reader = new BinaryReader(stream);
                var writer = new BinaryWriter(stream);

                // ======== CLIENT -> SERVER: Methods Offer (Page 3; I) ========
                byte socksVersion = reader.ReadByte(); // VER
                if (socksVersion != 5)
                {
                    client.Close();
                    return;
                }

                bool methodNoAuth = false;
                byte methods = reader.ReadByte(); // NMETHODS
                for (int i = 0; i < methods; i++)
                    methodNoAuth |= reader.ReadByte() == 0; // METHODS

                if (!methodNoAuth)
                {
                    client.Close();
                    return;
                }

                // ======== SERVER -> CLIENT: Method Select (Page 3; II) ========
                writer.Write((byte)5); // VER
                writer.Write((byte)0); // METHOD
                writer.Flush();

                // ======== CLIENT -> SERVER: Request (Page 4) ========
                reader.ReadByte(); // VER
                reader.ReadByte(); // CMD
                reader.ReadByte(); // RSV
                byte type = reader.ReadByte(); // ATYP

                if (type == 1)
                {
                    byte[] ipAddrBytes = reader.ReadBytes(4); // DST.ADDR
                    short port = BitConverter.ToInt16(reader.ReadBytes(2).Reverse().ToArray(), 0); // DST.PORT

                    TcpClient forwardClient = new TcpClient();
                    forwardClient.Connect(new IPAddress(ipAddrBytes), port);
                    NetworkStream forwardStream = forwardClient.GetStream();

                    // ======== SERVER -> CLIENT: Reply (Page 5-6) ========
                    writer.Write((byte)5); // VER
                    writer.Write((byte)0); // REP
                    writer.Write((byte)0); // RSV
                    writer.Write((byte)1); // ATYP
                    writer.Write(new byte[4]); // BND.ADDR
                    writer.Write(new byte[2]); // BND.PORT
                    writer.Flush();

                    // Relay stream to my own servers
                    try
                    {
                        while (true)
                        {
                            byte[] buffer = new byte[4096];
                            int clientByteCount = stream.Read(buffer, 0, buffer.Length);
                            forwardStream.Write(buffer, 0, clientByteCount);
                            forwardStream.Flush();

                            int forwardReadBytes = forwardStream.Read(buffer, 0, buffer.Length);
                            stream.Write(buffer, 0, forwardReadBytes);
                            stream.Flush();
                        }
                    }
                    catch { }
                }
            }).Start();
        }
    }
}