Ramon Snir

{
module Lexers.English

open Core
open Core.Parser
open System
open System.Text
open Microsoft.FSharp.Text.Lexing

let keywords =
    [
        "new", NEW;
        "if", IF;
        "then", THEN;
        "else", ELSE;
        "endif", ENDIF;
        "for", FOR;
        "to", TO;
        "inc", INC;
        "do", DO;
        "while", WHILE;
        "next", NEXT;
        "define", DEFINE;
        "as", AS;
        "enddefine", ENDDEFINE;
        "ctor", CTOR
    ] |> Map.ofList

let ops =
    [
        ")", RP;
        "(", LP;
        "<=", ASSIGN;
        ",", COMMA;
        "#", HASH
    ] |> Map.ofList
}

let number = ['+''-']?['0'-'9']+('.'['0'-'9']*)?
let string = ("@\""[^'"']*'"'|'"'(('\\''"')|[^'"'])*'"')
let id = [\w'.']+
let op = [\W]+
let use = "#use "[^'\n''\r']

let whitespace = [' ' '\t']+
let newline = ('\n' | '\r' '\n')

rule tokenize = parse
| whitespace    { tokenize lexbuf }
| newline       { NL }
| eof           { EOF }
| id            {
let text = Encoding.UTF8.GetString(lexbuf.Lexeme)
if Map.containsKey text keywords then
    keywords.[text]
else
    ID <| Encoding.UTF8.GetString(lexbuf.Lexeme)
}
| string        { STRING <| Encoding.UTF8.GetString(lexbuf.Lexeme) }
| op            {
let text = Encoding.UTF8.GetString(lexbuf.Lexeme)
if Map.containsKey text ops then
    ops.[text]
else
    OP <| Encoding.UTF8.GetString(lexbuf.Lexeme).Trim()
}
| use           { USE <| Encoding.UTF8.GetString(lexbuf.Lexeme).Substring(5) }
| number        { NUMBER <| Convert.ToDouble(Encoding.UTF8.GetString(lexbuf.Lexeme)) }