RS2LoginProtocol

package main.astraeus.network.protocol.codec.login;

import java.math.BigInteger;
import java.security.SecureRandom;
import java.util.List;
import java.util.Random;
import java.util.logging.Logger;

import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;
import main.astraeus.network.NetworkConstants;
import main.astraeus.network.packet.PacketBuilder;
import main.astraeus.network.protocol.ProtocolConstants;
import main.astraeus.network.protocol.codec.IsaacCipher;

/**
 * The class that handles a connection through the login protocol.
 *
 * @author SeVen
 */
public final class LoginDecoder extends ByteToMessageDecoder {

    /**
     * The single logger for this class.
     */
    private static final Logger LOGGER = Logger.getLogger(LoginDecoder.class.getName());

    /**
     * Generates random numbers via secure cryptography. Generates the session key for packet
     * encryption.
     */
    private static final Random RANDOM = new SecureRandom();

    /**
     * The size of the encrypted data.
     */
    private int encryptedLoginBlockSize;

    /**
     * The buffer that is holding the crypted data.
     */
    private ByteBuf rsaBuffer;

    /**
     * The encryptor for outgoing data.
     */
    private IsaacCipher encryptor;

    /**
     * The deccryptor for incoming data.
     */
    private IsaacCipher decryptor;

    /**
     * The current state of this connection through the login protocol.
     */
    private State state = State.REQUEST;


    @Override
    protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
        switch (state) {

        case REQUEST:
            decodeRequest(ctx, in);
            break;

        case CONNECTION_TYPE:
            decodeConnectionType(ctx, in);
            break;

        case PRECRYPTED:
            decodePreCrypted(in);
            break;

        case CRYPTED:
            decodeCrypted(ctx, in, out);
            break;

        }
    }

    /**
     * The stage that decodes the request opcode, and nashHash.
     *
     * The request opcode, is either the game server {@code 14} or update server {@code 15}.
     *
     * @param ctx The context for this ChannelHandler
     *
     * @param in The incoming data.
     */
    private void decodeRequest(ChannelHandlerContext ctx, ByteBuf in) throws Exception {
        if (in.readableBytes() >= 2) {
            System.out.println("decoding request");
            int request = in.readUnsignedByte();

            @SuppressWarnings("unused")
            int nameHash = in.readUnsignedByte();

            if (request != ProtocolConstants.GAME_SEVER_OPCODE && request != ProtocolConstants.FILE_SERVER_OPCODE) {
                sendResponseCode(ctx, LoginResponse.INVALID_LOGIN_SERVER);
                return;
            }

            ByteBuf buf = ctx.alloc().buffer(17);
            buf.writeLong(0);
            buf.writeByte(0);
            buf.writeLong(RANDOM.nextLong());
            ctx.writeAndFlush(buf);

            state = State.CONNECTION_TYPE;
        }
    }

    /**
     * The stage that decodes the type of connection.
     *
     * {@code 16} denotes a new connection
     *
     * {@code 18} denotes a reconnection.
     *
     * @param in The incoming data.
     */
    private void decodeConnectionType(ChannelHandlerContext ctx, ByteBuf in) throws Exception {
        if (in.readableBytes() >= 2) {
            System.out.println("decoding connection type");
            int connectionType = in.readUnsignedByte();

            if (connectionType != ProtocolConstants.NEW_CONNECTION_OPCODE && connectionType != ProtocolConstants.RECONNECTION_OPCODE) {
                sendResponseCode(ctx, LoginResponse.LOGIN_SERVER_REJECTED_SESSION);
                return;
            }

            state = State.PRECRYPTED;
        }
    }

    /**
     * The stage that decodes the login block size.
     *
     * @param in The incoming data.
     */
    private void decodePreCrypted(ByteBuf in) throws Exception {
        if (in.readableBytes() >= 2) {
            System.out.println("decoding precrypted");

            encryptedLoginBlockSize = in.readUnsignedByte();

            if ((encryptedLoginBlockSize - 40) <= 0) {
                LOGGER.info("encryptedLoginBlockSize <= 0");
                return;
            }

            state = State.CRYPTED;
        }
    }

    /**
     * The stage that decodes the encrypted data.
     *
     * @param in The incoming data.
     */
    private void decodeCrypted(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
        if (in.isReadable(encryptedLoginBlockSize)) {
            System.out.println("decoding crypted");
            int magicId = in.readUnsignedByte();

            if (magicId != 255) {
                LOGGER.info("MagicId != 255");
                return;
            }

            int clientVersion = in.readUnsignedShort();

            if (clientVersion != 317) {
                LOGGER.info("clientVersion != 317");
                return;
            }

           int memoryVersion = in.readUnsignedByte();

            if (memoryVersion != 0 && memoryVersion != 1) {
                sendResponseCode(ctx, LoginResponse.LOGIN_SERVER_REJECTED_SESSION);
                return;
            }

            int[] crcs = new int[9];

            for (int index = 0; index < crcs.length; index++) {
                crcs[index] = in.readInt();
            }

            int expectedSize = in.readUnsignedByte();

            if (expectedSize != encryptedLoginBlockSize - 41) {
                sendResponseCode(ctx, LoginResponse.LOGIN_SERVER_REJECTED_SESSION);
                return;
            }

            byte[] rsaBytes = new byte[encryptedLoginBlockSize - 41];
            in.readBytes(rsaBytes);

            rsaBuffer = ctx.alloc().buffer();
            rsaBuffer.writeBytes(new BigInteger(rsaBytes).modPow(NetworkConstants.RSA_EXPONENT, NetworkConstants.RSA_MODULUS).toByteArray());

            int rsa = rsaBuffer.readUnsignedByte();

            if (rsa != 10) {
                sendResponseCode(ctx, LoginResponse.LOGIN_SERVER_REJECTED_SESSION);
                return;
            }

            long clientHalf = rsaBuffer.readLong();
            long serverHalf = rsaBuffer.readLong();

            int[] isaacSeed = {
                    (int) (clientHalf >> 32),
                    (int)  clientHalf,
                    (int) (serverHalf >> 32),
                    (int) serverHalf };

            decryptor = new IsaacCipher(isaacSeed);
            for (int index = 0; index < isaacSeed.length; index++) {
                isaacSeed[index] += 50;
            }

            encryptor = new IsaacCipher(isaacSeed);

            System.out.println("decoding details");

            @SuppressWarnings("unused") int uid = rsaBuffer.readInt();

            PacketBuilder builder = new PacketBuilder(rsaBuffer);
            String username = builder.getString();
            String password = builder.getString();

            if (password.length() < 6 || password.length() > 20 || username.isEmpty() || username.length() > 12) {
                sendResponseCode(ctx, LoginResponse.INVALID_CREDENTIALS);
                return;
            }

            rsaBuffer.release();

            out.add(new LoginDetailsPacket(ctx, username, password, encryptor, decryptor));
        }
    }

    /**
     * Sends a response code to the client to notify the user logging in.
     *
     * @param ctx
     *      The context of the channel handler.
     *
     * @param response
     *      The response code to send.
     */
    private void sendResponseCode(ChannelHandlerContext ctx, LoginResponse response) {
        ByteBuf buffer = ctx.alloc().buffer(Byte.BYTES);
        buffer.writeByte(response.getOpcode());
        ctx.writeAndFlush(buffer).addListener(ChannelFutureListener.CLOSE);
    }

    /**
     * Represents the enumerated states of the login protocol
     *
     * @author SeVen
     */
    private enum State {

        /**
         * The state where a login server is selected.
         * <p>
         * Game Server : 14
         * <p>
         * File Server : 15
         * <p>
         */
        REQUEST,

        /**
         * The state that where the type of login connection is indicated.
         * <p>
         * New Connection : 16
         * <p>
         * Reconnecting : 18
         * <p>
         */
        CONNECTION_TYPE,

        /**
         * The state that determines the size of the encrypted data.
         */
        PRECRYPTED,

        /**
         * The state that decodes the encrypted data.
         */
        CRYPTED;
    }


}