Can you propose a more elegant way to 'tokenize' c# code for html formatting?

//Types of tokens we are going to detect
type Token =
    | Whitespace of string
    | Comment of string
    | Strng of string
    | Keyword of string
    | Text of string
    | EOF

//turn a string into a list of recognised tokens
let tokenize (s:String) =
    //this is the 'parser' - should we look at compiling the regexs in advance?
    let nexttoken (st:String) =
        match st with
        | st when Regex.IsMatch(st, "^s+") -> Whitespace(Regex.Match(st, "^s+").Value)
        | st when Regex.IsMatch(st, "^//.*?r?n") -> Comment(Regex.Match(st, "^//.*?r?n").Value) //this is double slash-style comments
        | st when Regex.IsMatch(st, "^/*(.|[r?n])*?*/") -> Comment(Regex.Match(st, "^/*(.|[r?n])*?*/").Value) // /* */ style comments http://ostermiller.org/findcomment.html
        | st when Regex.IsMatch(st, @"^""([^""\]|\.|"""")*""") -> Strng(Regex.Match(st, @"^""([^""\]|\.|"""")*""").Value) // unescaped = "([^"\]|\.|"")*" http://wordaligned.org/articles/string-literals-and-regular-expressions
        | st when Regex.IsMatch(st, "^#(end)?region") -> Keyword(Regex.Match(st, "^#(end)?region").Value)
        | st when st <> "" ->
                match Regex.Match(st, @"^[^""s]*").Value with //all text until next whitespace or quote (this may be wrong)
                | x when iskeyword x -> Keyword(x)  //iskeyword uses Microsoft.CSharp.CSharpCodeProvider.IsValidIdentifier - a bit fragile...
                | x -> Text(x)
        | _ -> EOF

    //tail-recursive use of next token to transform string into token list
    let tokeneater s =
        let rec loop s acc =
            let t = nexttoken s
            match t with
            | EOF -> List.rev acc //return accumulator (have to reverse it because built backwards with tail recursion)
            | Whitespace(x) | Comment(x)
            | Keyword(x) | Text(x) | Strng(x) ->
                loop (s.Remove(0, x.Length)) (t::acc)  //tail recursive
        loop s []

    tokeneater s

let (|Matches|_|) re s =
  let m = Regex(re).Match(s)
  if m.Success then
    Some(Matches (m.Value))
  else
    None

let nexttoken (st:String) =
  match st with
  | Matches "^s+" s -> Whitespace(s)
  | Matches "^//.*?r?n" s -> Comment(s)
  ...