g = x [Function] x^2;
Caputo[f_, [Alpha]_] := (x [Function]
   Evaluate[
    1/Gamma[1 - [Alpha]] Integrate[
      D[f[s], s]/(x - s)^[Alpha], {s, 0, x}, Assumptions -> 0 < x]])
Nest[Caputo[#, 1/2] &, g, 3]