CI_ASS5_Reinforcement Learning_C

#include "rl_homework_lib.h"

// Q1
State getNextState(State s_t, Action a_t) {
    if      (a_t == DOWN  && s_t.y != world_height - 2) s_t.y += 1;
    else if (a_t == UP    && s_t.y != 1               ) s_t.y -= 1;
    else if (a_t == RIGHT && s_t.x != world_width  - 2) s_t.x += 1;
    else if (a_t == LEFT  && s_t.x != 1               ) s_t.x -= 1;
    return s_t;
}

float getNextReward(State s_t) {
    if (s_t.y == world_height - 2) {
        if (s_t.x >= 2 && s_t.x <= world_width - 3)
            return dropOffPenalty;
        else if (s_t.x == world_width - 2)
            return goalReward;
    }
    return stepPenalty;
}

int isTerminalState(State s_t) {
    return s_t.x >= 2 && s_t.y == world_height - 2;
}


// Q2
Action getMax (float QValues[], State s_t) {
    int i;
    float maxQValue    = -9999999.0;
    int   maxDirection = 0;
    for (i = 0; i < 4; i++) {
        float qvalue = QValues[stateAction2idx(s_t, (Action) i)];
        if (maxQValue >= qvalue) continue;
        maxQValue    = qvalue;
        maxDirection = i;
    }
    return (Action) maxDirection;
}

Action getNextAction (Agent* a, State s_t) {
    return ((float) rand() / (float) RAND_MAX) >= a->epsilon ? getGreedyAction(a, s_t) : getRandomAction();
}

Action getRandomAction () {
    return (Action) (rand() % 4);
}

Action getGreedyAction(Agent* a, State s_t) {
    return getMax(a->QValues, s_t);
}


// Q3 & Q4
State currentState;
Action nextAction;

Agent* trainAgent(int nIter, int verbose, Agent* a) {
    int i;
    float accumReward = 0;

    for (i = 0; i < nIter; i++) {
        // initialize the current state to left bottom and
        // the accumulated reward for current iteration
        currentState   = (State) { .x = 1, .y = world_height - 2 };
        a->accumReward = 0;

        // start the episode
        playEpisode(0, verbose, a);

        // now add the accumalted reward to the total space
        accumReward += a->accumReward;
    }

    // calculate the average reward use (total / n)
    a->avgReward = accumReward / (float) nIter;

    return a;
}

Agent* playEpisode(int nEpisode, int verbose, Agent* a) {
    // for SARSA, only generate an action in updating process
    if ((a->type == SARSA && nEpisode == 0) || a->type == QLearning)
        nextAction = getNextAction(a, currentState);

    // retrieve all the essential data
    State nextState   = getNextState(currentState, nextAction);
    int isGoal        = nextState.x == world_width - 2 && nextState.y == world_height - 2;
    int isTerminal    = isTerminalState(nextState);
    float reward      = isGoal ? goalReward : (isTerminal ? dropOffPenalty : stepPenalty);

    // here we accumaltes the local reward for current iteration
    a->accumReward   += reward;

    // now applying the updating rule according to the training type
    if   (a->type == SARSA) sarsaUpdate    (a, currentState, nextState, nextAction, reward);
    else                    qLearningUpdate(a, currentState, nextState, nextAction, reward);

    // save the current state
    currentState = nextState;

    return isTerminal ? a : playEpisode(++nEpisode, verbose, a);
}

// SARSA: Q(s_t1, a_t1) = Q(s_t1, a_t1) + alpha * (r_t + gamma * Q(s_t2, a_t2)) - Q(s_t1, a_t1))
Agent* sarsaUpdate(Agent* a, State s_t1, State s_t2, Action a_t1, float r_t) {
    int index          = stateAction2idx(s_t1, a_t1);
    Action a_t2        = getNextAction(a, s_t2);
    a->QValues[index] += a->alpha * (r_t + a->gamma * a->QValues[stateAction2idx(s_t2, a_t2)] - a->QValues[index]);
    nextAction         = a_t2;
    return a;
}

// QLearning: Q(s_t1, a_t1) = r_t + gamma * MAX(Q(s_t2, A))
Agent* qLearningUpdate(Agent* a, State s_t1, State s_t2, Action a_t1, float r_t) {
    int i;
    float maxQValue  = -9999999.0;
    int maxDirection = 0;
    for (i = 0; i < 4; i++) {
        float qValue = a->QValues[stateAction2idx(s_t2, (Action) i)];
        if (qValue <= maxQValue) continue;
        maxQValue    = qValue;
        maxDirection = i;
    }
    a->QValues[stateAction2idx(s_t1, a_t1)] = r_t + a->gamma * a->QValues[stateAction2idx(s_t2, (Action) maxDirection)];
    return a;
}