[Beta][M] = 0;
[Psi] =  JacobiDN[u, m] Exp[I [Omega] x];
[Alpha][j_] := [Alpha][
   j] = -I Integrate[([Psi] Conjugate[[Beta][j]] -
       Conjugate[[Psi]] [Beta][j]), u] + 2^(j - 1) f[j]
[Beta][j_] := [Beta][j] = [Psi] [Alpha][j + 1] +
   I/2 D[[Beta][j + 1], u]

$Assumptions =
  x [Element] Reals && [Omega] [Element] Reals && m >= 0 && m < 1 &&
   u [Element] Reals && M [Element] Integers &&
   f[_] [Element] Reals

In[76]:= q = M - 4 ;
FullSimplify[TrigToExp[ComplexExpand[[Beta][q]]]]

Out[77]= 2^(-4 + M) E^(
 I x [Omega]) ((f[-3 + M] - m f[-1 + M]) JacobiDN[u, m] +
   I m (-f[-2 + M] + m f[M]) JacobiCN[u, m] JacobiSN[u, m])

In[79]:= FullSimplify[TrigToExp[ComplexExpand[[Alpha][q]]]]

Out[79]= 2^(-5 +
  M) (f[-4 + M] +
   m (-m f[M] - 2 (f[-2 + M] - m f[M]) JacobiCN[u, m]^2))

In[81]:= B  =
  2^(-4 + M) E^(
   I x [Omega]) ((f[-3 + M] - m f[-1 + M]) JacobiDN[u, m] +
     I m (-f[-2 + M] + m f[M]) JacobiCN[u, m] JacobiSN[u, m]);
Integrand = FullSimplify[Conjugate[B] [Psi] - Conjugate[[Psi]] B];
A = FullSimplify[- I Integrate[Integrand, u] + 2^(M - 4 - 1) f[M - 4]]

Out[83]= 2^(-5 +
  M) (f[-4 + M] + 2 m (-f[-2 + M] + m f[M]) JacobiCN[u, m]^2)