library(irr)
# two datasets available
# first: 6 raters classified 30 patients into 5 nominal categories
data(diagnoses)
str(diagnoses)
head(diagnoses[, 1:3])
# second: 3 raters classified 20 subjects on ordinal scales. Values are ranging from 1 to 6
data(anxiety)
str(anxiety)
head(anxiety, 4)
# Cohen’s Kappa: it can be used for nominal or ordinal categorical variables (and 2 coders)
kappa2(diagnoses[, c("rater1", "rater2")], weight = "unweighted")
kappa2(diagnoses[, c("rater3", "rater4")], weight = "unweighted")
# Light’s kappa: it returns the average Cohen’s kappa when you have multiple raters
# for example, average Cohen’s kappa between the first 3 raters
kappam.light(diagnoses[, 1:3])
# just to be sure
kappa2(diagnoses[, c("rater1", "rater2")], weight = "unweighted")
kappa2(diagnoses[, c("rater1", "rater3")], weight = "unweighted")
kappa2(diagnoses[, c("rater2", "rater3")], weight = "unweighted")
(0.651+0.384+0.631)/3
# Fleiss’ kappa: in this case the (multiple) raters are not assumed to be the same for all subjects/texts
kappam.fleiss(diagnoses[, 1:3])
# Weighted kappa: it can be considered only when ratings are performed in ordinal scale (& 2 coders).
# If you have more than 2 coders, you replicate the analysis and then you take the average as above
# Note: a weighted Kappa can also be calculated for factors (but remember: the factor levels must be in the correct order!)
kappa2(anxiety[, c("rater2", "rater3")], weight = "equal")
# The weighted Kappa calculation can use either linear or squared weights of the differences
kappa2(anxiety[, c("rater2", "rater3")], weight = "squared")
# what's the difference, in the case of an ordinal scale, between weighted Kappa and Cohen’s Kappa?
# Immagine that your possible values are low, medium, and high, then according to a weighted Kappa
# if a case were rated medium and high by the two coders, they would be in better agreement than if the ratings were low and high;
# with Cohen’s Kappa all differences are on the contrary treated as the same
kappa2(diagnoses[, c("rater2", "rater3")], weight = "unweighted")
# Intraclass correlation coefficients: continuous scales (by treating the previous ordinal scale as it were a continuous variable)
# and any number of raters
icc(
anxiety, model = "twoway",
type = "agreement", unit = "single"
)