library(irr) # two datasets available # first: 6 raters classified 30 patients into 5 nominal categories data(diagnoses) str(diagnoses) head(diagnoses[, 1:3]) # second: 3 raters classified 20 subjects on ordinal scales. Values are ranging from 1 to 6 data(anxiety) str(anxiety) head(anxiety, 4) # Cohen’s Kappa: it can be used for nominal or ordinal categorical variables (and 2 coders) kappa2(diagnoses[, c("rater1", "rater2")], weight = "unweighted") kappa2(diagnoses[, c("rater3", "rater4")], weight = "unweighted") # Light’s kappa: it returns the average Cohen’s kappa when you have multiple raters # for example, average Cohen’s kappa between the first 3 raters kappam.light(diagnoses[, 1:3]) # just to be sure kappa2(diagnoses[, c("rater1", "rater2")], weight = "unweighted") kappa2(diagnoses[, c("rater1", "rater3")], weight = "unweighted") kappa2(diagnoses[, c("rater2", "rater3")], weight = "unweighted") (0.651+0.384+0.631)/3 # Fleiss’ kappa: in this case the (multiple) raters are not assumed to be the same for all subjects/texts kappam.fleiss(diagnoses[, 1:3]) # Weighted kappa: it can be considered only when ratings are performed in ordinal scale (& 2 coders). # If you have more than 2 coders, you replicate the analysis and then you take the average as above # Note: a weighted Kappa can also be calculated for factors (but remember: the factor levels must be in the correct order!) kappa2(anxiety[, c("rater2", "rater3")], weight = "equal") # The weighted Kappa calculation can use either linear or squared weights of the differences kappa2(anxiety[, c("rater2", "rater3")], weight = "squared") # what's the difference, in the case of an ordinal scale, between weighted Kappa and Cohen’s Kappa? # Immagine that your possible values are low, medium, and high, then according to a weighted Kappa # if a case were rated medium and high by the two coders, they would be in better agreement than if the ratings were low and high; # with Cohen’s Kappa all differences are on the contrary treated as the same kappa2(diagnoses[, c("rater2", "rater3")], weight = "unweighted") # Intraclass correlation coefficients: continuous scales (by treating the previous ordinal scale as it were a continuous variable) # and any number of raters icc( anxiety, model = "twoway", type = "agreement", unit = "single" )