;; This buffer is for text that is not saved, and for Lisp evaluation.
;; To create a file, visit it with C-x C-f and enter text in its buffer.

(defun tokenise (string)
  (loop
   for start = 0 then (1+ space)
   for space = (position #\Space string :start start)
   for token = (subseq string start space)
   unless (equal token "") ; upewnij się że nie jest pusty
   collect  (clean token)
   until (not space)))

(defun clean (string)
  (let ((result ""))
     (loop
      for character across string
      do
      (if
      ;(alpha-char-p character)    ; works only for simple letters & ASCII mode, to define form scratch:
      (not (member character '(#\0 #\1 #\2 #\3 #\4 #\5 #\6 #\7 #\8 #\9 #\, #\. #\? #\; #\: #\" #\! #\- #\Return)))
      (setf result (concatenate 'string
                    result (string character)))))
     result))

(defun file-to-tokens (file)
  (with-open-file (stream file)
          (loop
           for line = (read-line stream nil)
           while line
           append (tokenise line))))

;>(char "ściemniać" 0)

;>(char "ściemniać" 1)

;>(char "ściemniać" 2)

;>(concatenate 'string (string (char "ściemniać" 0)) (string (char "ściemniać" 1)))

;>(loop for x across "ściemniać" collect x)
                    ;>(subseq "ściemniać" 0 2)

(defun string-to-tokens (string)
  (loop
   for start = 0 then (1+ space)
   for space = (position #\Space string start start)
   for token = (subseq string start space)))