Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- use utf8;
- use strict;
- use warnings;
- use open qw<:encoding(utf8) :std>;
- use Data::Dumper;
- use Data::Section -setup;
- use Hailo::Tokenizer::Words;
- my $self = bless {} => __PACKAGE__;
- my $text = ${ $self->section_data("UTF-8 encoded sample plain-text file") };
- print $text;
- my $toke = Hailo::Tokenizer::Words->new();
- my $parsed = $toke->make_tokens($text);
- print Dumper($parsed);
- __DATA__
- __[ UTF-8 encoded sample plain-text file ]__
- STARGΛ̊TE SG-1, a = v̇ = r̈, a⃑ ⊥ b⃑
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement