Case statement in SQL Rownumber() Order By clause not working on varchar and int values

DECLARE @temp table
(ShipNo int,
 Supplier varchar(10)
)

INSERT INTO @temp VALUES (1,'CFA')
INSERT INTO @temp VALUES (1, 'TFA')
INSERT INTO @temp VALUES (2, 'LRA')
INSERT INTO @temp VALUES (2, 'LRB')
INSERT INTO @temp VALUES (3, 'ABC')
INSERT INTO @temp VALUES (4, 'TFA')

Declare @OrderBy varchar(255)
sET @OrderBy = 'Supplier'



SELECT ROW_NUMBER() OVER (ORDER BY
CASE @OrderBy
 WHEN 'Supplier' THEN Supplier
 WHEN 'ShipNo' THEN ShipNo
END
) AS RowNo,
ShipNo,
Supplier
FROM @temp

CASE
WHEN Boolean_expression THEN result_expression
    [ ...n ]
[
    ELSE else_result_expression
]
END