colordiff $file1 $file2 | less -R

-R or --RAW-CONTROL-CHARS
          Like -r, but only ANSI  "color"  escape  sequences  are
          output in "raw" form.  Unlike -r, the screen appearance
          is maintained correctly in most  cases.   ANSI  "color"
          escape sequences are sequences of the form:

               ESC [ ... m

          where  the  "..."  is  zero or more color specification
          characters For the purpose of keeping track  of  screen
          appearance,  ANSI color escape sequences are assumed to
          not move the cursor.  You  can  make  less  think  that
          characters  other  than  "m"  can end ANSI color escape
          sequences by setting the environment  variable  LESSAN‐
          SIENDCHARS  to  the  list of characters which can end a
          color escape sequence.  And you  can  make  less  think
          that characters other than the standard ones may appear
          between the ESC and the m by  setting  the  environment
          variable  LESSANSIMIDCHARS  to  the  list of characters
          which can appear.

diff a b |
   perl -lpe 'if(/^</){$_ = "e[1;31m$_e[0m"}
              elsif(/^>/){$_ = "e[1;34m$_e[0m"}'

diff $file1 $file2 | colordiff | less -r

diff -uw $file1 $file2 | colordiff | less -r

diff -uw $file1 $file2 | tee /dev/stderr | colordiff | less -r -F

diff -uw $file1 $file2 | colordiff | less -r -X -F