a = DiagonalMatrix[{5, 4, 3}]
b = DiagonalMatrix[{3, 4, 5}]
X = Array[x, {3, 3}]
r = NSolve[
  Transpose[X].a.X == b && (Transpose[X].X)[[1, 1]] ==
    1 && (Transpose[X].X)[[2, 2]] == 1 && (Transpose[X].X)[[3, 3]] ==
    1, Flatten[X], Reals, WorkingPrecision -> 32]

X={{0,0,1},{0,1,0},{1,0,0}}