use utf8;
use strict;
use warnings;
use open qw<:encoding(utf8) :std>;
use Data::Dumper;
use Hailo::Tokenizer::Words;

my $self = bless {} => __PACKAGE__;
my $text = 'STARGΛ̊TE';
print Dumper($text);

my $toke = Hailo::Tokenizer::Words->new();
my $parsed = $toke->make_tokens($text);
print Dumper($parsed);


#################


$ perlbrew switch perl-5.10.1
$ perl utf8-text.t
$VAR1 = "STARG\x{39b}\x{30a}TE";
$VAR1 = [
          [
            0,
            "STARG\x{39b}\x{30a}TE"
          ]
        ];

$ perlbrew switch perl-5.13.5
$ perl utf8-text.t
$VAR1 = "STARG\x{39b}\x{30a}TE";
$VAR1 = [
          [
            0,
            "STARG\x{39b}"
          ],
          [
            3,
            "\x{30a}"
          ],
          [
            0,
            'TE'
          ]
        ];