select to_number('AA', 'xx') from dual;

select to_char(111, 'xxxx') from dual;

SELECT  TO_NUMBER('DEADBEEF', 'XXXXXXXX')
FROM    dual

---
3735928559

SELECT  TO_CHAR(3735928559, 'XXXXXXXX')
FROM    dual
---
 DEADBEEF

SQL> select to_char(123,'XX') to_hex,
2    to_number('7B','XX') from_hex
3   from dual
4  /
TO_     FROM_HEX
------  -----------------
7B     123

CREATE OR REPLACE FUNCTION hex2dec (hexnum IN CHAR) RETURN NUMBER IS
  i                 NUMBER;
  digits            NUMBER;
  result            NUMBER := 0;
  current_digit     CHAR(1);
  current_digit_dec NUMBER;
BEGIN
  digits := LENGTH(hexnum);
  FOR i IN 1..digits LOOP
     current_digit := SUBSTR(hexnum, i, 1);
     IF current_digit IN ('A','B','C','D','E','F') THEN
        current_digit_dec := ASCII(current_digit) - ASCII('A') + 10;
     ELSE
        current_digit_dec := TO_NUMBER(current_digit);
     END IF;
     result := (result * 16) + current_digit_dec;
  END LOOP;
  RETURN result;
END hex2dec;
/
show errors

CREATE OR REPLACE FUNCTION num2hex (N IN NUMBER) RETURN VARCHAR2 IS
  H  VARCHAR2(64) :='';
  N2 INTEGER      := N;
BEGIN
  LOOP
     SELECT RAWTOHEX(CHR(N2))||H
     INTO   H
     FROM   dual;

     N2 := TRUNC(N2 / 256);
     EXIT WHEN N2=0;
  END LOOP;
  RETURN H;
END num2hex;
/
show errors