library(irr)

# two datasets available
# first: 6 raters classified 30 patients into 5 nominal categories 
data(diagnoses)
str(diagnoses)
head(diagnoses[, 1:3])

# second: 3 raters classified 20 subjects on ordinal scales. Values are ranging from 1 to 6
data(anxiety)
str(anxiety)
head(anxiety, 4)

# Cohen's Kappa can be used for nominal or ordinal categorical variables (and 2 coders)
kappa2(diagnoses[, c("rater1", "rater2")], weight = "unweighted")
kappa2(diagnoses[, c("rater3", "rater4")], weight = "unweighted")

# Light's kappa returns the average Cohen's kappa when you have multiple raters 
# for example, average Cohen's kappa between the first 3 raters
kappam.light(diagnoses[, 1:3])
# just to be sure
kappa2(diagnoses[, c("rater1", "rater2")], weight = "unweighted")
kappa2(diagnoses[, c("rater1", "rater3")], weight = "unweighted")
kappa2(diagnoses[, c("rater2", "rater3")], weight = "unweighted")
round((0.651+0.384+0.631)/3,3)

# with Fleiss kappa the (multiple) raters are not assumed to be the same for all subjects/texts
kappam.fleiss(diagnoses[, 1:3])

# "Weighted kappa" can be considered only when ratings are performed in ordinal scale (& 2 coders).
# If you have more than 2 coders, you replicate the analysis and then you take the average as above.
# What's the difference, in the case of an ordinal scale, between weighted Kappa and Cohen's Kappa?
# Immagine that your possible values are low, medium, and high, then according to a weighted Kappa 
# if a case were rated medium and high by the two coders, they would be in better agreement than 
# if the ratings were low and high; with Cohen's Kappa all differences are on the contrary treated as the same

# The unweighted Kappa
kappa2(diagnoses[, c("rater2", "rater3")], weight = "unweighted")
# The weighted Kappa calculation can use either linear or squared weights of the differences
kappa2(anxiety[, c("rater2", "rater3")], weight = "equal")
kappa2(anxiety[, c("rater2", "rater3")], weight = "squared")

# Note: a weighted Kappa can also be calculated for factors (but remember: the factor levels must be in the correct order!)