Affix class cleanup

bindings/python-examples/tests.py

@@ -77,11 +77,11 @@ def test_open_nonexistent_dictionary(self):
 
         save_stderr = divert_start(2)
         self.assertRaises(LG_DictionaryError, Dictionary, dummy_lang + '1')
-        self.assertIn(dummy_lang + '1', save_stderr.divert_end())
+        self.assertIn(dummy_lang + '1', str(save_stderr.divert_end()))
 
         save_stderr = divert_start(2)
         self.assertRaises(LG_Error, Dictionary, dummy_lang + '2')
-        self.assertIn(dummy_lang + '2', save_stderr.divert_end())
+        self.assertIn(dummy_lang + '2', str(save_stderr.divert_end()))
 
 # Check absolute and relative dictionary access.
 # Check also that the dictionary language is set correctly.
@@ -506,7 +506,7 @@ def test_22_defaut_handler_param(self):
         dummy_lang = "a dummy dict name (bad param test)"
         self.assertRaises(LG_Error, Dictionary, dummy_lang)
         LG_Error.printall(self.error_handler_test, self) # grab a valid errinfo
-        #self.assertIn(dummy_lang, save_stderr.divert_end())
+        #self.assertIn(dummy_lang, str(save_stderr.divert_end()))
         self.assertRaisesRegexp(TypeError, "must be an integer",
                                 self.__class__.handler["default"],
                                 self.errinfo, "bad param")
@@ -520,7 +520,7 @@ def test_22_defaut_handler_param(self):
             self.param_ok = False
             save_stdout = divert_start(1) # Note: Handler parameter is stdout
             self.__class__.handler["default"](self.errinfo, 1)
-            self.assertIn(dummy_lang, save_stdout.divert_end())
+            self.assertIn(dummy_lang, str(save_stdout.divert_end()))
             self.param_ok = True
         except (TypeError, ValueError):
             self.assertTrue(self.param_ok)
@@ -598,7 +598,7 @@ def test_50_set_orig_error_handler(self):
         dummy_lang = "a dummy dict name (default handler test)"
         save_stderr = divert_start(2)
         self.assertRaises(LG_Error, Dictionary, dummy_lang)
-        self.assertIn(dummy_lang, save_stderr.divert_end())
+        self.assertIn(dummy_lang, str(save_stderr.divert_end()))
         self.assertEqual(self.errinfo, "dummy")
 
 class FSATsolverTestCase(unittest.TestCase):
@@ -1281,9 +1281,16 @@ def test_d_morphology(self):
              '.', 'RIGHT-WALL'])
 
 
+# The Thai 4.0.affix files currently contain strippable affixes that are
+# not in the dict. This causes an annoying multiline error output that are
+# filtered out here using divert().
 class ZTHLangTestCase(unittest.TestCase):
     def test_thai(self):
+        save_stderr = divert_start(2)
         linkage_testfile(self, Dictionary(lang='th'), ParseOptions())
+        for line in save_stderr.divert_end().decode().split("\n"):
+           if 'Token(s) not in the dictionary' not in line:
+              print(line)
 
 
 class ZXDictDialectTestCase(unittest.TestCase):
@@ -1501,7 +1508,7 @@ def divert_end(self):
         os.close(self.savedfd)
         os.unlink(self.filename)
         self.filename = None
-        return str(content)
+        return content
 
     __del__ = divert_end
 

data/de/4.0.affix

@@ -1,6 +1,6 @@
 
-")" "%" "," "." ":" ";" "?" "!" "''" "'" "'s": RPUNC+;
-"(" "$" "``": LPUNC+;
+")" "," "." ":" ";" "?" "!" "'" „ « ...: RPUNC+;
+"(" “ » ...: LPUNC+;
 
 %e en er es em: SUF+;
 .=: STEMSUBSCR+;

data/de/4.0.dict

@@ -7,7 +7,7 @@
  %                                                                           %
  %***************************************************************************%
 
-#define dictionary-version-number 5.9.0;
+#define dictionary-version-number 5.11.0;
 #define dictionary-locale         de_DE.UTF-8;
 
 % NOUNS, PRONOUNS
@@ -521,3 +521,7 @@ PL-CAPITALIZED-WORDS:    BZZT+;
 % will output an error for any unknown words.
 <UNKNOWN-WORD>: NO+;
 
+% Unhandled, but getting split according to 4.0.affix.
+")" ":" ";" "'" „ « ....y: <UNKNOWN-WORD>;     % RPUNC
+"(" “ » ....x: <UNKNOWN-WORD>;                 % LPUNC
+

data/fa/4.0.affix

@@ -1,3 +1,3 @@
 
-")" "%" "," "." ":" ";" "?" "!" "''" "'" : RPUNC+;
-"(" "$" "``": LPUNC+;
+")" "%" "," "." ":" ";" "?" "!" "'" : RPUNC+;
+"(" "$": LPUNC+;

data/fa/4.0.dict

@@ -10,7 +10,7 @@
 %
 %*****************************************************************************
 
-#define dictionary-version-number 5.9.0;
+#define dictionary-version-number 5.11.0;
 #define dictionary-locale         fa_IR.UTF-8;
 
 % PERSIAN SYNTAX NOTES:
@@ -835,11 +835,11 @@ or
 
 ;
 
+% Unhandled, but getting split according to 4.0.affix.
+")" "%" ";" "'": ();                      % RPUNC
+"(" "$": ();                              % LPUNC
 
 
-%":" ";" :	% Colon and Semicolon
-%;
-
 %NUMBERS:   ;
 LEFT-WALL: ( Wcc+ or [Wi+] or [[Wd+]] ) & {Xp+}; % Connects to the Subject in declarative sentence, or Verb in subjectless or imperative sentence
 RIGHT-WALL: RW- ;

data/he/4.0.affix

@@ -1,13 +1,6 @@
-% need to add Hebrew punctuations
+")" """ "," "." "–" "‐" ... ":" ";" "?" "!": RPUNC+;
 
-")" "}" "]" ">" » 〉 ） 〕 》 】 ］ 』 」 "’’" "’" ''.y '.y
-"%" "," "." 。 ":" ";" "?" "!" ‽ ؟ ？！ ….y ....y "”" "–" "‐" 、 ～
-¢ ₵ ™ ℠ : RPUNC+;
-
-"(" "{" "[" "<" « 〈 （ 〔 《 【 ［ 『 「 `` „ “ ‘ ''.x '.x ….x ....x
-¿ ¡ "$" 
-£ ₤ € ¤ ₳ ฿ ₡ ₢ ₠ ₫ ৳ ƒ ₣ ₲ ₴ ₭ ₺  ℳ  ₥ ₦ ₧ ₱ ₰ ₹ ₨ ₪ ﷼  ₸ ₮ ₩ ¥ ៛ 호점
-† †† ‡ § ¶ © ® ℗ № "#": LPUNC+;
+"(" „ “ """ ... ₪: LPUNC+;
 
 =: INFIXMARK+;
 % The order is preserved.

data/he/4.0.dict

@@ -40,7 +40,7 @@
 % Among numerous other things, changes to handle count/uncountable changes
 % have not been done yet. The created infrastructure for that may still need changes.
 
-#define dictionary-version-number 5.9.0;
+#define dictionary-version-number 5.11.0;
 #define dictionary-locale         he_IL.UTF-8;
 
 % For now.
@@ -562,3 +562,7 @@ or Ds+ or (R- & (C+ or RS+)) or SIs- or (Ss+ &
 % skip over (null-link) unknown words. If you remove it, the parser
 % will output an error for any unknown words.
 <UNKNOWN-WORD>: XXX+;
+
+% Punctuations that get strip but are yet unhandled.
+ "." "–" "‐" ")" "".y" "....y" ":" ";" "?" "!" ₪: <UNKNOWN-WORD>;    % RPUNC
+"(" „ “ "".x" ....x: <UNKNOWN-WORD>;                                 % LPUNC

data/he/4.0.dict.m4

@@ -45,9 +45,8 @@ changecom(`%')dnl
 % Among numerous other things, changes to handle count/uncountable changes
 % have not been done yet. The created infrastructure for that may still need changes.
 
-% Dictionary version number is 5.3.15 (formatted as V5v3v15+)
-<dictionary-version-number>: V5v3v15+;
-<dictionary-locale>: HE4il+;
+#define dictionary-version-number 5.11.0;
+#define dictionary-locale         he_IL.UTF-8;
 
 % For now.
 LEFT-WALL: {Wa+} or {Wd+} or ();
@@ -458,4 +457,8 @@ or Ds+ or (R- & (C+ or RS+)) or SIs- or (Ss+ &
 % With the following line in the dictionary, the parser will simply
 % skip over (null-link) unknown words. If you remove it, the parser
 % will output an error for any unknown words.
-UNKNOWN-WORD: XXX+;
+<UNKNOWN-WORD>: XXX+;
+
+% Punctuations that get strip but are yet unhandled.
+ "." "–" "‐" ")" "".y" "....y" ":" ";" "?" "!" ₪: <UNKNOWN-WORD>;    % RPUNC
+"(" „ “ "".x" ....x: <UNKNOWN-WORD>;                                 % LPUNC

data/id/4.0.affix

@@ -1,30 +1,11 @@
-% 
 % Affixes get stripped off the left and right side of words 
 % i.e. spaces are inserted between the affix and the word itself.
-%
-% Some of the funky UTF-8 parenthesis are used in Asian texts.
-% In order to allow single straight quote ' and double straight quote ''
-% to be stripped off from both the left and the right, they are
-% distinguished by the suffix .x and .y (as as Mr.x Mrs.x or Jr.y Sr.y)
-% 
-% 。is an end-of-sentence marker used in Japanese texts.
 
 % Punctuation appearing on the right-side of words.
-")" "}" "]" ">" » 〉 ） 〕 》 】 ］ 』 」 "’’" "’" ''.y '.y ' `
-"%" "," "." 。.y ‧ ":" ";" "?" "!" ‽ ؟ ？ ！ ….y ....y "”" ━.y –.y ー.y ‐.y 、.y
-～ ¢ ₵ ™ ℠ : RPUNC+; 
+"," : RPUNC+; % Only this punctuation appears in the dict.
 
 % Punctuation appearing on the left-side of words.
-"(" "{" "[" "<" « 〈 （ 〔 《 【 ［ 『 「 、.x ` `` „ “ ‘ ''.x '.x ….x ....x
-¿ ¡ "$" US$ USD C$ 
-£ ₤ € ¤ ₳ ฿ ₡ ₢ ₠ ₫ ৳ ƒ ₣ ₲ ₴ ₭ ₺  ℳ  ₥ ₦ ₧ ₱ ₰ ₹ ₨ ₪ ﷼  ₸ ₮ ₩ ¥ ៛ 호점
-† †† ‡ § ¶ © ® ℗ № "#"
-* • ⁂ ❧ ☞ ◊ ※  ○  。.x ゜ ✿ ☆ ＊ ◕ ● ∇ □ ◇ ＠ ◎ 
-–.x ━.x ー.x -- - ‧.x
-  : LPUNC+;
-
-% Suffixes
-'s 're 've 'd 'll 'm ’s ’re ’ve ’d ’ll ’m: SUF+;
+%  : LPUNC+; % none.
 
 % The below is a quoted list, used during tokenization. Do NOT put
 % spaces in between the various quotation marks!!
@@ -33,14 +14,3 @@
 % The below is a quoted list, used during tokenization. Do NOT put
 % spaces in between the various symbols!!
 "()¿¡†‡§¶©®℗№#*•⁂❧☞◊※○。゜✿☆＊◕●∇□◇＠◎–━ー---‧": BULLETS+;
-
-/en/words/units.1: UNITS+;
-/en/words/units.1.dot: UNITS+;
-/en/words/units.3: UNITS+;
-/en/words/units.4: UNITS+;
-/en/words/units.4.dot: UNITS+;
-/en/words/units.5: UNITS+;
-%
-% units.6 contains just a single, sole slash in it. This allows units
-% such as mL/s to be split at the slash.
-/en/words/units.6: UNITS+;

data/id/4.0.dict

@@ -7,7 +7,7 @@
  %                                                                           %
  %***************************************************************************%
 
-#define dictionary-version-number 5.9.0;
+#define dictionary-version-number 5.11.0;
 #define dictionary-locale         id_ID.UTF-8;
 
 anjing kucing wanita cewek pria cowok lelaki laki-laki taman lapangan tulang

data/kz/4.0.affix

@@ -1,3 +1,4 @@
+% From these affixes, only "," is handled in the dictionary.
 ")" "%" "," "." ":" ";" "?" "!" "''" "'" : RPUNC+;
 "(" "$" "``": LPUNC+;
 

data/kz/4.0.dict

@@ -2,7 +2,7 @@
 % Kazakh dictionary -- experimental -- 2016
 % -- Tatiana Batura, Aigerim Bakiyeva, Aigerim Yerimbetova
 %
-#define dictionary-version-number 5.9.0;
+#define dictionary-version-number 5.11.0;
 #define dictionary-locale         kk_KZ.UTF-8;
 
 мен.pron: {W-} & S1s+;
@@ -133,3 +133,7 @@ LATIN-ADJ-P-NOUN-WORDS: XXX-;
 LATIN-ADJ-S-NOUN-WORDS: XXX-;
 HYPHENATED-WORDS: XXX-;
 <UNKNOWN-WORD>: XXX-;
+
+% Unhandled, but getting split according to 4.0.affix.
+")" "%" "." ":" ";" "?" "!" "''" "'" : <UNKNOWN-WORD>;    % RPUNC
+"(" "$" "``": <UNKNOWN-WORD>;                             % LPUNC

data/ru/4.0.affix

@@ -1,5 +1,5 @@
-")" "%" "," "." ":" ";" "?" "!" "''" "'": RPUNC+;
-"(" "$" "``": LPUNC+;
+")" "%" "," "." ":" ";" "?" "!" "'": RPUNC+;
+"(" "$": LPUNC+;
 
 ""«»《》【】『』`„“": QUOTES+;
 

data/ru/4.0.dict

@@ -7,10 +7,10 @@
 %%
 %% This file uses the utf8 encoding
 
-#define dictionary-version-number 5.9.0;
+#define dictionary-version-number 5.11.0;
 #define dictionary-locale         ru_RU.UTF-8;
 #define max-disjunct-cost         2.7;
-#define panic_max-disjunct-cost   4.0;
+#define panic-max-disjunct-cost   4.0;
 
 <costly-null>: [[[[]]]];
 
@@ -1096,3 +1096,7 @@ LENGTH-LIMIT-1: LL*+;
 % skip over (null-link) unknown words. If you remove it, the parser
 % will output an error for any unknown words.
 <UNKNOWN-WORD>: XXX+;
+
+% Unhandled, but getting split according to 4.0.affix.
+"%" "'": <UNKNOWN-WORD>;        % RPUNC
+"$": <UNKNOWN-WORD>;            % LPUNC

data/tr/4.0.affix

@@ -1,5 +1,6 @@
-")" "%" "," "." ":" ";" "?" "!" "''" "'": RPUNC+;
-"(" "$" "``": LPUNC+;
+% From these affixes, only "," is handled in the dictionary.
+")" "%" "," "." ":" ";" "?" "!" "'": RPUNC+;
+"(" "$" : LPUNC+;
 
 dakileri ki ler leri kileri düm m di lar diler um yor mi ni diğ diğini ini yorum miyorum ın in un ün nın nin nun nün a e ya ye i ı u ü yi yı yu yü yla ile yle da de ta te den ten dan tan sız: SUF+;
 

data/tr/4.0.dict

@@ -1,7 +1,7 @@
 %
 % Turkish dictionary -- experimental -- 2016 -- Tatiana Batura, Maria Mitkovskaya, Natalya Semenova
 %
-#define dictionary-version-number 5.9.0;
+#define dictionary-version-number 5.11.0;
 #define dictionary-locale         tr_TR.UTF-8;
 
 % adjectives
@@ -119,3 +119,7 @@ PL-GREEK-LETTER-AND-NUMBER: XXX-;
 CAPITALIZED-WORDS: XXX-;
 HYPHENATED-WORDS: XXX-;
 <UNKNOWN-WORD>: XXX-;
+
+% Unhandled, but getting split according to 4.0.affix.
+")" "%" "." ":" ";" "?" "!" "'": <UNKNOWN-WORD>;      % RPUNC
+"(" "$": <UNKNOWN-WORD>;                              % LPUNC

data/vn/4.0.affix

@@ -1,31 +1,11 @@
-% 
 % Affixes get stripped off the left and right side of words 
 % i.e. spaces are inserted between the affix and the word itself.
-%
-% Some of the funky UTF-8 parenthesis are used in Asian texts.
-% In order to allow single straight quote ' and double straight quote ''
-% to be stripped off from both the left and the right, they are
-% distinguished by the suffix .x and .y (as as Mr.x Mrs.x or Jr.y Sr.y)
-% 
-% 。is an end-of-sentence marker used in Japanese texts.
 
 % Punctuation appearing on the right-side of words.
-% Note: the ellipsis ....y must appear *before* the dot ".", else the
-% splitting won't work right.
-")" "}" "]" ">" "".y" » 〉 ） 〕 》 】 ］ 』 」 "’’" "’" “ ''.y '.y `.y
-"%" "," ....y "." 。.y ‧ ":" ";" "?" "!" ‽ ؟ ？ ！ ….y "”" ━.y –.y ー.y ‐.y 、.y
-～ ¢ ₵ ™ ℠ 
-  : RPUNC+; 
+"," : RPUNC+; % Only this punctuation appears in the dict.
 
 % Punctuation appearing on the left-side of words.
-"(" "{" "[" "<" "".x" « 〈 （ 〔 《 【 ［ 『 「 、.x `.x `` „ ‘ ''.x '.x ….x ....x
-¿ ¡ "$" US$ USD C$ 
-£ ₤ € ¤ ₳ ฿ ₡ ₢ ₠ ₫ ৳ ƒ ₣ ₲ ₴ ₭ ₺  ℳ  ₥ ₦ ₧ ₱ ₰ ₹ ₨ ₪ ﷼  ₸ ₮ ₩ ¥ ៛ 호점
-† †† ‡ § ¶ © ® ℗ № "#"
-* • ⁂ ❧ ☞ ◊ ※  ○  。.x ゜ ✿ ☆ ＊ ◕ ● ∇ □ ◇ ＠ ◎ 
-–.x ━.x ー.x -- - ‧.x
-  : LPUNC+;
-
+%  : LPUNC+; % None.
 
 % The below is a quoted list, used during tokenization. Do NOT put
 % spaces in between the various quotation marks!!

data/vn/4.0.dict

@@ -1,6 +1,6 @@
 % Vietnamese Dictionary
 
-#define dictionary-version-number 5.9.2;
+#define dictionary-version-number 5.11.0;
 #define dictionary-locale         vi_VN.UTF-8;
 
 % See https://bitbucket.org/ngocminh/lienkate for the master