newcommand{set}[2]{ left{ #1 ,middlevert, #2right} }

set{A}{B}{Big}

documentclass{article}
usepackage{mathtools}

DeclarePairedDelimiterX{innset}[2]{lbrace}{rbrace}{%
  #1;delimsizevert;#2}

newcommand{set}[3][0]{%
  ifcase#1relax
    innset{#2}{#3}or         % 0
    innset[big]{#2}{#3}or   % 1
    innset[Big]{#2}{#3}or   % 2
    innset[bigg]{#2}{#3}or  % 3
    innset[Bigg]{#2}{#3}     % 4
  else
    innset*{#2}{#3}
  fi}

begin{document}
begin{gather*}
set{A}{frac{B}{C}}\
set[0]{A}{frac{B}{C}}\
set[1]{A}{frac{B}{C}}\
set[2]{A}{frac{B}{C}}\
set[3]{A}{frac{B}{C}}\
set[4]{A}{frac{B}{C}}\
set[-1]{A}{frac{B}{C}}
end{gather*}
end{document}

newcommandset[3]{csname #3lendcsname{#1%
                   ,csname#3endcsnamevert,%
                   #2csname #3rendcsname}}