betoesquivel

CONLL task evaluation Perl script modified to evaluate NER

Mar 1st, 2016
805
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Perl 12.89 KB | None | 0 0
  1. #!/usr/bin/env per
  2. # This code was adapted from http://www.cnts.ua.ac.be/conll2000/chunking/output.html by
  3. # Jose Alberto Esquivel Patiño for Name Entity Recognition evaluation.
  4.  
  5. # conlleval: evaluate result of processing CoNLL-2000 shared task
  6. # usage:     conlleval [-l] [-r] [-d delimiterTag] [-o oTag] < file
  7. #            README: http://cnts.uia.ac.be/conll2000/chunking/output.html
  8. # options:   l: generate LaTeX output for tables like in
  9. #               http://cnts.uia.ac.be/conll2003/ner/example.tex
  10. #            r: accept raw result tags (without B- and I- prefix;
  11. #                                       assumes one word per chunk)
  12. #            d: alternative delimiter tag (default is single space)
  13. #            o: alternative outside tag (default is O)
  14. # note:      the file should contain lines with items separated
  15. #            by $delimiter characters (default space). The final
  16. #            two items should contain the correct tag and the
  17. #            guessed tag in that order. Sentences should be
  18. #            separated from each other by empty lines or lines
  19. #            with $boundary fields (default -X-).
  20. # url:       http://lcg-www.uia.ac.be/conll2000/chunking/
  21. # started:   1998-09-25
  22. # version:   2004-01-26
  23. # author:    Erik Tjong Kim Sang <[email protected]>
  24.  
  25. use strict;
  26.  
  27. my $false = 0;
  28. my $true = 42;
  29.  
  30. my $boundary = "";        # sentence boundary
  31. my $correct;              # current corpus chunk tag (I,O,B)
  32. my $correctChunk = 0;     # number of correctly identified chunks
  33. my $correctTags = 0;      # number of correct chunk tags
  34. my $correctType;          # type of current corpus chunk tag (NP,VP,etc.)
  35. my $delimiter = "\s";     # field delimiter
  36. my $FB1 = 0.0;            # FB1 score (Van Rijsbergen 1979)
  37. my $firstItem;            # first feature (for sentence boundary checks)
  38. my $foundCorrect = 0;     # number of chunks in corpus
  39. my $foundGuessed = 0;     # number of identified chunks
  40. my $guessed;              # current guessed chunk tag
  41. my $guessedType;          # type of current guessed chunk tag
  42. my $i;                    # miscellaneous counter
  43. my $inCorrect = $false;   # currently processed chunk is correct until now
  44. my $lastCorrect = "O";    # previous chunk tag in corpus
  45. my $latex = 0;            # generate LaTeX formatted output
  46. my $lastCorrectType = ""; # type of previously identified chunk tag
  47. my $lastGuessed = "O";    # previously identified chunk tag
  48. my $lastGuessedType = ""; # type of previous chunk tag in corpus
  49. my $lastType;             # temporary storage for detecting duplicates
  50. my $line;                 # line
  51. my $nbrOfFeatures = -1;   # number of features per line
  52. my $precision = 0.0;      # precision score
  53. my $oTag = "O";           # outside tag, default O
  54. my $raw = 0;              # raw input: add B to every token
  55. my $recall = 0.0;         # recall score
  56. my $tokenCounter = 0;     # token counter (ignores sentence breaks)
  57.  
  58. my %correctChunk = ();    # number of correctly identified chunks per type
  59. my %foundCorrect = ();    # number of chunks in corpus per type
  60. my %foundGuessed = ();    # number of identified chunks per type
  61.  
  62. my @features;             # features on line
  63. my @sortedTypes;          # sorted list of chunk type names
  64.  
  65. # sanity check
  66. while (@ARGV and $ARGV[0] =~ /^-/) {
  67.    if ($ARGV[0] eq "-l") { $latex = 1; shift(@ARGV); }
  68.    elsif ($ARGV[0] eq "-r") { $raw = 1; shift(@ARGV); }
  69.    elsif ($ARGV[0] eq "-d") {
  70.       shift(@ARGV);
  71.       if (not defined $ARGV[0]) {
  72.          die "conlleval: -d requires delimiter character";
  73.       }
  74.       $delimiter = shift(@ARGV);
  75.    } elsif ($ARGV[0] eq "-o") {
  76.       shift(@ARGV);
  77.       if (not defined $ARGV[0]) {
  78.          die "conlleval: -o requires delimiter character";
  79.       }
  80.       $oTag = shift(@ARGV);
  81.    } else { die "conlleval: unknown argument $ARGV[0]\n"; }
  82. }
  83. if (@ARGV) { die "conlleval: unexpected command line argument\n"; }
  84. # process input
  85. while (<STDIN>) {
  86.    chomp($line = $_);
  87.    @features = split(/\s+/,$line);
  88.    print "@{[join(':', @features)]}\n";
  89.    if ($nbrOfFeatures < 0) { $nbrOfFeatures = $#features; }
  90.    elsif ($nbrOfFeatures != $#features and @features != 0) {
  91.       printf STDERR "unexpected number of features: %d (%d)\n",
  92.          $#features+1,$nbrOfFeatures+1;
  93.       exit(1);
  94.    }
  95.    if ($#features == 0 or
  96.        $features[0] eq $boundary) { @features = ($boundary,"O","O"); }
  97.    if (@features < 1) {
  98.       die "conlleval: unexpected number of features () in line $line\n";
  99.    }
  100.    if ($raw) {
  101.       if ($features[$#features] eq $oTag) { $features[$#features] = "O"; }
  102.       if ($features[$#features-1] eq $oTag) { $features[$#features-1] = "O"; }
  103.       if ($features[$#features] ne "O") {
  104.          $features[$#features] = "B-$features[$#features]";
  105.       }
  106.       if ($features[$#features-1] ne "O") {
  107.          $features[$#features-1] = "B-$features[$#features-1]";
  108.       }
  109.    }
  110.    # 20040126 ET code which allows hyphens in the types
  111.    if ($features[$#features] =~ /^([^-]*)-(.*)$/) {
  112.       $guessed = $1;
  113.       $guessedType = $2;
  114.    } else {
  115.       $guessed = $features[$#features];
  116.       $guessedType = "";
  117.    }
  118.    pop(@features);
  119.    if ($features[$#features] =~ /^([^-]*)-(.*)$/) {
  120.       $correct = $1;
  121.       $correctType = $2;
  122.    } else {
  123.       $correct = $features[$#features];
  124.       $correctType = "";
  125.    }
  126.    pop(@features);
  127. #  ($guessed,$guessedType) = split(/-/,pop(@features));
  128. #  ($correct,$correctType) = split(/-/,pop(@features));
  129.    $guessedType = $guessedType ? $guessedType : "";
  130.    $correctType = $correctType ? $correctType : "";
  131.    $firstItem = shift(@features);
  132.  
  133.    # 1999-06-26 sentence breaks should always be counted as out of chunk
  134.    if ( $firstItem eq $boundary ) { $guessed = "O"; }
  135.  
  136.    if ($inCorrect) {
  137.       if ( &endOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) and
  138.            &endOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) and
  139.            $lastGuessedType eq $lastCorrectType) {
  140.          $inCorrect=$false;
  141.          $correctChunk++;
  142.          $correctChunk{$lastCorrectType} = $correctChunk{$lastCorrectType} ?
  143.              $correctChunk{$lastCorrectType}+1 : 1;
  144.       } elsif (
  145.            &endOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) !=
  146.            &endOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) or
  147.            $guessedType ne $correctType ) {
  148.          $inCorrect=$false;
  149.       }
  150.    }
  151.  
  152.    if ( &startOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) and
  153.         &startOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) and
  154.         $guessedType eq $correctType) { $inCorrect = $true; }
  155.  
  156.    if ( &startOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) ) {
  157.       $foundCorrect++;
  158.       $foundCorrect{$correctType} = $foundCorrect{$correctType} ?
  159.           $foundCorrect{$correctType}+1 : 1;
  160.    }
  161.    if ( &startOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) ) {
  162.       $foundGuessed++;
  163.       $foundGuessed{$guessedType} = $foundGuessed{$guessedType} ?
  164.           $foundGuessed{$guessedType}+1 : 1;
  165.    }
  166.    if ( $firstItem ne $boundary ) {
  167.       if ( $correct eq $guessed and $guessedType eq $correctType ) {
  168.          $correctTags++;
  169.       }
  170.       $tokenCounter++;
  171.    }
  172.  
  173.    $lastGuessed = $guessed;
  174.    $lastCorrect = $correct;
  175.    $lastGuessedType = $guessedType;
  176.    $lastCorrectType = $correctType;
  177. }
  178. if ($inCorrect) {
  179.    $correctChunk++;
  180.    $correctChunk{$lastCorrectType} = $correctChunk{$lastCorrectType} ?
  181.        $correctChunk{$lastCorrectType}+1 : 1;
  182. }
  183.  
  184. if (not $latex) {
  185.    # compute overall precision, recall and FB1 (default values are 0.0)
  186.    $precision = 100*$correctChunk/$foundGuessed if ($foundGuessed > 0);
  187.    $recall = 100*$correctChunk/$foundCorrect if ($foundCorrect > 0);
  188.    $FB1 = 2*$precision*$recall/($precision+$recall)
  189.       if ($precision+$recall > 0);
  190.  
  191.    # print overall performance
  192.    printf "processed $tokenCounter tokens with $foundCorrect entities; ";
  193.    printf "found: $foundGuessed entities; correct: $correctChunk.\n";
  194.    if ($tokenCounter>0) {
  195.       printf "accuracy: %6.2f%%; ",100*$correctTags/$tokenCounter;
  196.       printf "precision: %6.2f%%; ",$precision;
  197.       printf "recall: %6.2f%%; ",$recall;
  198.       printf "FB1: %6.2f\n",$FB1;
  199.    }
  200. }
  201.  
  202. # sort chunk type names
  203. undef($lastType);
  204. @sortedTypes = ();
  205. foreach $i (sort (keys %foundCorrect,keys %foundGuessed)) {
  206.    if (not($lastType) or $lastType ne $i) {
  207.       push(@sortedTypes,($i));
  208.    }
  209.    $lastType = $i;
  210. }
  211. # print performance per chunk type
  212. if (not $latex) {
  213.    for $i (@sortedTypes) {
  214.       $correctChunk{$i} = $correctChunk{$i} ? $correctChunk{$i} : 0;
  215.       if (not($foundGuessed{$i})) { $foundGuessed{$i} = 0; $precision = 0.0; }
  216.       else { $precision = 100*$correctChunk{$i}/$foundGuessed{$i}; }
  217.       if (not($foundCorrect{$i})) { $recall = 0.0; }
  218.       else { $recall = 100*$correctChunk{$i}/$foundCorrect{$i}; }
  219.       if ($precision+$recall == 0.0) { $FB1 = 0.0; }
  220.       else { $FB1 = 2*$precision*$recall/($precision+$recall); }
  221.       printf "%17s: ",$i;
  222.       printf "precision: %6.2f%%; ",$precision;
  223.       printf "recall: %6.2f%%; ",$recall;
  224.       printf "FB1: %6.2f  %d\n",$FB1,$foundGuessed{$i};
  225.    }
  226. } else {
  227.    print "        & Precision &  Recall  & F\$_{\\beta=1} \\\\\\hline";
  228.    for $i (@sortedTypes) {
  229.       $correctChunk{$i} = $correctChunk{$i} ? $correctChunk{$i} : 0;
  230.       if (not($foundGuessed{$i})) { $precision = 0.0; }
  231.       else { $precision = 100*$correctChunk{$i}/$foundGuessed{$i}; }
  232.       if (not($foundCorrect{$i})) { $recall = 0.0; }
  233.       else { $recall = 100*$correctChunk{$i}/$foundCorrect{$i}; }
  234.       if ($precision+$recall == 0.0) { $FB1 = 0.0; }
  235.       else { $FB1 = 2*$precision*$recall/($precision+$recall); }
  236.       printf "\n%-7s &  %6.2f\\%% & %6.2f\\%% & %6.2f \\\\",
  237.              $i,$precision,$recall,$FB1;
  238.    }
  239.    print "\\hline\n";
  240.    $precision = 0.0;
  241.    $recall = 0;
  242.    $FB1 = 0.0;
  243.    $precision = 100*$correctChunk/$foundGuessed if ($foundGuessed > 0);
  244.    $recall = 100*$correctChunk/$foundCorrect if ($foundCorrect > 0);
  245.    $FB1 = 2*$precision*$recall/($precision+$recall)
  246.       if ($precision+$recall > 0);
  247.    printf "Overall &  %6.2f\\%% & %6.2f\\%% & %6.2f \\\\\\hline\n",
  248.           $precision,$recall,$FB1;
  249. }
  250.  
  251. exit 0;
  252.  
  253. # endOfChunk: checks if a chunk ended between the previous and current word
  254. # arguments:  previous and current chunk tags, previous and current types
  255. # note:       this code is capable of handling other chunk representations
  256. #             than the default CoNLL-2000 ones, see EACL'99 paper of Tjong
  257. #             Kim Sang and Veenstra http://xxx.lanl.gov/abs/cs.CL/9907006
  258.  
  259. sub endOfChunk {
  260.    my $prevTag = shift(@_);
  261.    my $tag = shift(@_);
  262.    my $prevType = shift(@_);
  263.    my $type = shift(@_);
  264.    my $chunkEnd = $false;
  265.  
  266.    if ( $prevTag eq "B" and $tag eq "B" ) { $chunkEnd = $true; }
  267.    if ( $prevTag eq "B" and $tag eq "O" ) { $chunkEnd = $true; }
  268.    if ( $prevTag eq "I" and $tag eq "B" ) { $chunkEnd = $true; }
  269.    if ( $prevTag eq "I" and $tag eq "O" ) { $chunkEnd = $true; }
  270.  
  271.    if ( $prevTag eq "E" and $tag eq "E" ) { $chunkEnd = $true; }
  272.    if ( $prevTag eq "E" and $tag eq "I" ) { $chunkEnd = $true; }
  273.    if ( $prevTag eq "E" and $tag eq "O" ) { $chunkEnd = $true; }
  274.    if ( $prevTag eq "I" and $tag eq "O" ) { $chunkEnd = $true; }
  275.  
  276.    if ($prevTag ne "O" and $prevTag ne "." and $prevType ne $type) {
  277.       $chunkEnd = $true;
  278.    }
  279.  
  280.    # corrected 1998-12-22: these chunks are assumed to have length 1
  281.    if ( $prevTag eq "]" ) { $chunkEnd = $true; }
  282.    if ( $prevTag eq "[" ) { $chunkEnd = $true; }
  283.  
  284.    return($chunkEnd);
  285. }
  286.  
  287. # startOfChunk: checks if a chunk started between the previous and current word
  288. # arguments:    previous and current chunk tags, previous and current types
  289. # note:         this code is capable of handling other chunk representations
  290. #               than the default CoNLL-2000 ones, see EACL'99 paper of Tjong
  291. #               Kim Sang and Veenstra http://xxx.lanl.gov/abs/cs.CL/9907006
  292.  
  293. sub startOfChunk {
  294.    my $prevTag = shift(@_);
  295.    my $tag = shift(@_);
  296.    my $prevType = shift(@_);
  297.    my $type = shift(@_);
  298.    my $chunkStart = $false;
  299.  
  300.    if ( $prevTag eq "B" and $tag eq "B" ) { $chunkStart = $true; }
  301.    if ( $prevTag eq "I" and $tag eq "B" ) { $chunkStart = $true; }
  302.    if ( $prevTag eq "O" and $tag eq "B" ) { $chunkStart = $true; }
  303.    if ( $prevTag eq "O" and $tag eq "I" ) { $chunkStart = $true; }
  304.  
  305.    if ( $prevTag eq "E" and $tag eq "E" ) { $chunkStart = $true; }
  306.    if ( $prevTag eq "E" and $tag eq "I" ) { $chunkStart = $true; }
  307.    if ( $prevTag eq "O" and $tag eq "E" ) { $chunkStart = $true; }
  308.    if ( $prevTag eq "O" and $tag eq "I" ) { $chunkStart = $true; }
  309.  
  310.    if ($tag ne "O" and $tag ne "." and $prevType ne $type) {
  311.       $chunkStart = $true;
  312.    }
  313.  
  314.    # corrected 1998-12-22: these chunks are assumed to have length 1
  315.    if ( $tag eq "[" ) { $chunkStart = $true; }
  316.    if ( $tag eq "]" ) { $chunkStart = $true; }
  317.  
  318.    return($chunkStart);
  319. }
Advertisement
Add Comment
Please, Sign In to add comment