module Tokens : sig
  type t
  [@@deriving show]

  type reversed
  [@@deriving show]

  val empty : reversed
  val prepend : Token.t -> reversed -> reversed
  val reverse : reversed -> t
  val to_list : t -> Token.t list
end = Raw_tokens

let lex_string (x : string) : Tokens.t =
  let rec lex_loop lexbuf acc =
    let current = Raw_lexer.read lexbuf in
    match current with
    | EOF -> acc
    | _ -> lex_loop lexbuf (Tokens.prepend current acc)
  in
  let initial_lexbuf = Lexing.from_string x in
  let reversed_tokens = lex_loop initial_lexbuf Tokens.empty in
  Tokens.reverse reversed_tokens