Ramon Snir

{
open System
open System.Text
open Microsoft.FSharp.Text.Lexing
}

let digit = ['0'-'9']
let alpha = ['a'-'z' 'A'-'Z']
let whitespace = [' ' '\t']
let nonwhitespace = ^[' ' '\t']
let newline = ('\n' | '\r' '\n')

rule tokenize = parse
| whitespace				{ tokenize lexbuf }
| newline					{ lexbuf.EndPos <- lexbuf.EndPos.NextLine; tokenize lexbuf }
| ['-']?digit+['.']?digit*	{ Digits <| Double.Parse(Encoding.UTF8.GetString(lexbuf.Lexeme)) }
| '('						{ LeftParen }
| ')'						{ RightParen }
| '\''						{ Apostrophe }
| ','						{ Comma }
| nonwhitespace+			{ let s = Encoding.UTF8.GetString(lexbuf.Lexeme) in Text <| s.Substring(1, s.Length - 2) }
| '"'[^'"']*'"'				{ let s = Encoding.UTF8.GetString(lexbuf.Lexeme) in Text <| s.Substring(1, s.Length - 2) }
| eof						{ Eof }