rnnTimeStep[params_Association, input_, state_] :=
      Tanh[params["StateWeights"].state + params["InputWeights"].input +
        params["Biases"]];
    rnnLayerForward[params_Association, input_, state_] :=
     Rest@FoldList[rnnTimeStep[params, #2, #1] &, state, input]

sequenceLength = 3;
featureSize = 2;
rnnStateSize = 1;
inData = RandomReal[1, {sequenceLength, featureSize}];
state = ConstantArray[0, rnnStateSize];

net = NetInitialize@
  BasicRecurrentLayer[rnnStateSize, "Input" -> Dimensions@inData]

param = Normal@NetExtract[net, "Arrays"]

net[inData]

rnnLayerForward[param, inData, state]