From ecb3bd38080f54bcaa5afed6ba3b202c42118280 Mon Sep 17 00:00:00 2001
From: Richard Eckart de Castilho <richard.eckart@gmail.com>
Date: Sat, 17 Apr 2021 14:03:08 +0200
Subject: [PATCH] #2191 - Agreement does not take tagset into account

- Adjust based on discussion at https://github.com/dkpro/dkpro-statistics/issues/35
---
 .../KrippendorffAlphaAgreementMeasure.java                 | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/krippendorffalpha/KrippendorffAlphaAgreementMeasure.java b/inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/krippendorffalpha/KrippendorffAlphaAgreementMeasure.java
index 9d677ae3a35..f14ef7f24b5 100644
--- a/inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/krippendorffalpha/KrippendorffAlphaAgreementMeasure.java
+++ b/inception/inception-agreement/src/main/java/de/tudarmstadt/ukp/clarin/webanno/agreement/measures/krippendorffalpha/KrippendorffAlphaAgreementMeasure.java
@@ -81,7 +81,12 @@ public double calculateAgreement()
                 double D_O = calculateObservedDisagreement();
                 double D_E = calculateExpectedDisagreement();
                 if (D_O == 0.0 && D_E == 0.0) {
-                    return 1.0;
+                    if (study.getItemCount() == 0) {
+                        return Double.NaN;
+                    }
+                    else {
+                        return 1.0;
+                    }
                 }
                 return 1.0 - (D_O / D_E);
             }