documentclass{article}
usepackage{color}
usepackage{listings}

lstdefinelanguage{new}{
    moredelim=*[s][color{red}]{(}{)},
    moredelim=*[s][color{green}]{[}{]},
  }
lstset{
  language=new
  %,breaklines=true
  }

begin{document}

begin{figure}
begin{lstlisting}
outside(in paren) [in brackets (paren+brackets) brackets again] outside again
end{lstlisting}
end{figure}

end{document}

lst@Def{`)}{lst@breakProcessOther)}

lst@Def=macro:
#1->lccode `~=#1lowercase {def ~}.

documentclass{article}

usepackage{color}

usepackage{listings}

lstdefinelanguage{new}{
  moredelim=*[s][color{red}]{(}{)},
  moredelim=*[s][color{green}]{[}{]},
}

lstset{
  language=new,
  breaklines=true
}

usepackage{etoolbox}

makeatletter

patchcmd{lsthk@SelectCharTable}{%
  lst@ifbreaklineslst@Def{`)}{lst@breakProcessOther)}fi
}{%
}{
}{
}

makeatother

begin{document}

begin{lstlisting}
outside (in paren) [in brackets (paren+brackets) brackets again] outside again
end{lstlisting}

end{document}

begin{lstlisting[literate={(}{{textbf{(}}}1
                           {]}{{textbf{)}}}1]
(In text I'm closing using]
end{lstlisting}