convert langid confidences to floats

Helsinki-NLP · Jun 26, 2024 · 1003490 · 1003490
1 parent b722f04
commit 1003490
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/opusfilter/filters.py b/opusfilter/filters.py
@@ -388,7 +388,7 @@ def confidence(self, sentence: str, lan: str) -> float:
 
         if self.id_method == 'langid':
             lidetails = self.identifier.classify(sentence)
-            lilan, liconf = lidetails[0], round(lidetails[1], 2)
+            lilan, liconf = lidetails[0], round(float(lidetails[1]), 2)
             if lilan != lan:
                 liconf = 0.0
             return liconf

diff --git a/opusfilter/opusfilter.py b/opusfilter/opusfilter.py
@@ -556,7 +556,11 @@ def _write_jsonl(objects, fname):
         """Write objects to file as JSON lines"""
         with file_open(fname, 'w') as fobj:
             for obj in objects:
-                fobj.write(json.dumps(obj, sort_keys=True)+'\n')
+                try:
+                    fobj.write(json.dumps(obj, sort_keys=True)+'\n')
+                except TypeError as err:
+                    logger.error("Could not convert to JSON: %s", obj)
+                    raise err
 
     @staticmethod
     def _read_jsonl(fname):