diff --git a/inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/krippendorffalpha/KrippendorffAlphaAgreementMeasure.java b/inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/krippendorffalpha/KrippendorffAlphaAgreementMeasure.java
index 9d677ae3a35..f14ef7f24b5 100644
--- a/inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/krippendorffalpha/KrippendorffAlphaAgreementMeasure.java
+++ b/inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/krippendorffalpha/KrippendorffAlphaAgreementMeasure.java
@@ -81,7 +81,12 @@ public double calculateAgreement()
                 double D_O = calculateObservedDisagreement();
                 double D_E = calculateExpectedDisagreement();
                 if (D_O == 0.0 && D_E == 0.0) {
-                    return 1.0;
+                    if (study.getItemCount() == 0) {
+                        return Double.NaN;
+                    }
+                    else {
+                        return 1.0;
+                    }
                 }
                 return 1.0 - (D_O / D_E);
             }