# Saved only the last 20 columns of the file first. No need to grab the rest.

# Calculate percent agreement
install.packages("irr")
agree(matrixdata, tolerance=0)
   Percentage agreement (Tolerance=0)
   Subjects = 7000
     Raters = 20
   %-agree = 0.0143

agree(matrixdata, 1)
   Percentage agreement (Tolerance=1)
   Subjects = 7000
     Raters = 20
   %-agree = 11.8

# Calculate krip's alpha. NOTE: This took hours on a MacBook Pro. If you've a
# spare $10 just rent a cluster compute instance on EC2 for a few hours.
install.packages("concord")
dataset = read.delim("/path/to/results", header=T)
matrixdata = data.matrix(dataset)
krip.alpha(matrixdata, method = "nominal")
    Krippendorffs alpha (data level - nominal) = 0.1529467