f = a[x] + c1*b[x] + c2*c[x];
g = Expand[f*f];
h = Distribute[Integrate[g, x]];
Solve[{D[h, c1] == 0, D[h, c2] == 0}, {c1, c2}]