Normalization with Chocomufin

.github/workflows/chocomufin.yml

@@ -0,0 +1,25 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: ChocoMufin
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install chocomufin
+    - name: Run ChocoMufin
+      run: |
+        chocomufin convert characters.csv ./data/**/*.xml --suffix ".normalized.xml" --sample-verbose 1
+        chocomufin generate characters.csv ./data/**/*.xml
+        chocomufin control characters.csv ./data/**/*.xml

characters.csv

@@ -1,152 +1,155 @@
-char,name,replacement,codepoint,mufidecode
-ɔ,LATIN SMALL LETTER OPEN O,,0254,o
-è,LATIN SMALL LETTER E WITH GRAVE,,00E8,e
->,GREATER-THAN SIGN,,003E,>
-ā,LATIN SMALL LETTER A WITH MACRON,,0101,a
-3,DIGIT THREE,,0033,3
-W,LATIN CAPITAL LETTER W,,0057,W
-Æ,LATIN CAPITAL LETTER AE,,00C6,AE
-H,LATIN CAPITAL LETTER H,,0048,H
-t,LATIN SMALL LETTER T,,0074,t
-ç,LATIN SMALL LETTER C WITH CEDILLA,,00E7,c
-î,LATIN SMALL LETTER I WITH CIRCUMFLEX,,00EE,i
-ğ,LATIN SMALL LETTER G WITH BREVE,,011F,g
-b,LATIN SMALL LETTER B,,0062,b
-M,LATIN CAPITAL LETTER M,,004D,M
-d,LATIN SMALL LETTER D,,0064,d
-ä,LATIN SMALL LETTER A WITH DIAERESIS,,00E4,a
-ī,LATIN SMALL LETTER I WITH MACRON,,012B,i
-l,LATIN SMALL LETTER L,,006C,l
-A,LATIN CAPITAL LETTER A,,0041,A
-Y,LATIN CAPITAL LETTER Y,,0059,Y
-1,DIGIT ONE,,0031,1
-€,EURO SIGN,,20AC,EUR
-’,RIGHT SINGLE QUOTATION MARK,,2019,'
-ě,LATIN SMALL LETTER E WITH CARON,,011B,e
-í,LATIN SMALL LETTER I WITH ACUTE,,00ED,i
-),RIGHT PARENTHESIS,,0029,)
-ž,LATIN SMALL LETTER Z WITH CARON,,017E,z
-È,LATIN CAPITAL LETTER E WITH GRAVE,,00C8,E
-ȩ,LATIN SMALL LETTER E WITH CEDILLA,,0229,e
-Ç,LATIN CAPITAL LETTER C WITH CEDILLA,,00C7,C
-!,EXCLAMATION MARK,,0021,!
-&,AMPERSAND,,0026,&
-C,LATIN CAPITAL LETTER C,,0043,C
-6,DIGIT SIX,,0036,6
-²,SUPERSCRIPT TWO,,00B2,2
-S,LATIN CAPITAL LETTER S,,0053,S
-R,LATIN CAPITAL LETTER R,,0052,R
-f,LATIN SMALL LETTER F,,0066,f
-Z,LATIN CAPITAL LETTER Z,,005A,Z
-T,LATIN CAPITAL LETTER T,,0054,T
-â,LATIN SMALL LETTER A WITH CIRCUMFLEX,,00E2,a
-ú,LATIN SMALL LETTER U WITH ACUTE,,00FA,u
-g,LATIN SMALL LETTER G,,0067,g
-2,DIGIT TWO,,0032,2
-Á,LATIN CAPITAL LETTER A WITH ACUTE,,00C1,A
-e,LATIN SMALL LETTER E,,0065,e
-œ,LATIN SMALL LIGATURE OE,,0153,oe
-ļ,LATIN SMALL LETTER L WITH CEDILLA,,013C,l
-"""",QUOTATION MARK,,0022,""""
-y,LATIN SMALL LETTER Y,,0079,y
-o,LATIN SMALL LETTER O,,006F,o
-«,LEFT-POINTING DOUBLE ANGLE QUOTATION MARK,,00AB,
-i,LATIN SMALL LETTER I,,0069,i
-c,LATIN SMALL LETTER C,,0063,c
-Q,LATIN CAPITAL LETTER Q,,0051,Q
-ʁ,LATIN LETTER SMALL CAPITAL INVERTED R,,0281,R
-u,LATIN SMALL LETTER U,,0075,u
-ø,LATIN SMALL LETTER O WITH STROKE,,00F8,o
-ã,LATIN SMALL LETTER A WITH TILDE,,00E3,a
-q,LATIN SMALL LETTER Q,,0071,q
-ə,LATIN SMALL LETTER SCHWA,,0259,e
-v,LATIN SMALL LETTER V,,0076,v
-/,SOLIDUS,,002F,/
-(,LEFT PARENTHESIS,,0028,(
-:,COLON,,003A,:
-],RIGHT SQUARE BRACKET,,005D,]
-#,NUMBER SIGN,,0023,#
-ů,LATIN SMALL LETTER U WITH RING ABOVE,,016F,u
-D,LATIN CAPITAL LETTER D,,0044,D
-ᑕ,CANADIAN SYLLABICS TA,,1455,ta
-O,LATIN CAPITAL LETTER O,,004F,O
-%,PERCENT SIGN,,0025,%
-8,DIGIT EIGHT,,0038,8
-G,LATIN CAPITAL LETTER G,,0047,G
-ì,LATIN SMALL LETTER I WITH GRAVE,,00EC,i
-Š,LATIN CAPITAL LETTER S WITH CARON,,0160,S
-4,DIGIT FOUR,,0034,4
-ß,LATIN SMALL LETTER SHARP S,,00DF,s
-ê,LATIN SMALL LETTER E WITH CIRCUMFLEX,,00EA,e
-x,LATIN SMALL LETTER X,,0078,x
-Ö,LATIN CAPITAL LETTER O WITH DIAERESIS,,00D6,O
-ë,LATIN SMALL LETTER E WITH DIAERESIS,,00EB,e
-À,LATIN CAPITAL LETTER A WITH GRAVE,,00C0,A
-L,LATIN CAPITAL LETTER L,,004C,L
-Â,LATIN CAPITAL LETTER A WITH CIRCUMFLEX,,00C2,A
-=,EQUALS SIGN,,003D,=
-ö,LATIN SMALL LETTER O WITH DIAERESIS,,00F6,o
-ù,LATIN SMALL LETTER U WITH GRAVE,,00F9,u
-0,DIGIT ZERO,,0030,0
-ɑ,LATIN SMALL LETTER ALPHA,,0251,a
-—,EM DASH,,2014,
-n,LATIN SMALL LETTER N,,006E,n
-r,LATIN SMALL LETTER R,,0072,r
-å,LATIN SMALL LETTER A WITH RING ABOVE,,00E5,a
-7,DIGIT SEVEN,,0037,7
-ū,LATIN SMALL LETTER U WITH MACRON,,016B,u
-",",COMMA,,002C,","
-k,LATIN SMALL LETTER K,,006B,k
-z,LATIN SMALL LETTER Z,,007A,z
-É,LATIN CAPITAL LETTER E WITH ACUTE,,00C9,E
-p,LATIN SMALL LETTER P,,0070,p
-–,EN DASH,,2013,
-û,LATIN SMALL LETTER U WITH CIRCUMFLEX,,00FB,u
-5,DIGIT FIVE,,0035,5
-h,LATIN SMALL LETTER H,,0068,h
-á,LATIN SMALL LETTER A WITH ACUTE,,00E1,a
-ñ,LATIN SMALL LETTER N WITH TILDE,,00F1,n
-Û,LATIN CAPITAL LETTER U WITH CIRCUMFLEX,,00DB,U
-^,CIRCUMFLEX ACCENT,,005E,^
-F,LATIN CAPITAL LETTER F,,0046,F
-m,LATIN SMALL LETTER M,,006D,m
-9,DIGIT NINE,,0039,9
-U,LATIN CAPITAL LETTER U,,0055,U
-<,LESS-THAN SIGN,,003C,<
-ſ,LATIN SMALL LETTER LONG S,,017F,s
-ï,LATIN SMALL LETTER I WITH DIAERESIS,,00EF,i
-a,LATIN SMALL LETTER A,,0061,a
-ɛ,LATIN SMALL LETTER OPEN E,,025B,e
-w,LATIN SMALL LETTER W,,0077,w
-s,LATIN SMALL LETTER S,,0073,s
-E,LATIN CAPITAL LETTER E,,0045,E
-B,LATIN CAPITAL LETTER B,,0042,B
-ɹ,LATIN SMALL LETTER TURNED R,,0279,r
-ô,LATIN SMALL LETTER O WITH CIRCUMFLEX,,00F4,o
--,HYPHEN-MINUS,,002D,-
-;,SEMICOLON,,003B,;
-.,FULL STOP,,002E,.
-P,LATIN CAPITAL LETTER P,,0050,P
-ó,LATIN SMALL LETTER O WITH ACUTE,,00F3,o
-ń,LATIN SMALL LETTER N WITH ACUTE,,0144,n
-ō,LATIN SMALL LETTER O WITH MACRON,,014D,o
-I,LATIN CAPITAL LETTER I,,0049,I
-ř,LATIN SMALL LETTER R WITH CARON,,0159,r
-J,LATIN CAPITAL LETTER J,,004A,J
-N,LATIN CAPITAL LETTER N,,004E,N
-à,LATIN SMALL LETTER A WITH GRAVE,,00E0,a
-V,LATIN CAPITAL LETTER V,,0056,V
-ü,LATIN SMALL LETTER U WITH DIAERESIS,,00FC,u
-K,LATIN CAPITAL LETTER K,,004B,K
-é,LATIN SMALL LETTER E WITH ACUTE,,00E9,e
-»,RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK,,00BB,
-',APOSTROPHE,,0027,'
-j,LATIN SMALL LETTER J,,006A,j
-X,LATIN CAPITAL LETTER X,,0058,X
-[,LEFT SQUARE BRACKET,,005B,[
-Î,LATIN CAPITAL LETTER I WITH CIRCUMFLEX,,00CE,I
-Ü,LATIN CAPITAL LETTER U WITH DIAERESIS,,00DC,U
-ł,LATIN SMALL LETTER L WITH STROKE,,0142,l
-š,LATIN SMALL LETTER S WITH CARON,,0161,s
-…,HORIZONTAL ELLIPSIS,,2026,
-°,DEGREE SIGN,,00B0,*
+char,name,replacement,codepoint,mufidecode,order
+#r#« ,Repl extra space before LEFT-POINTING DOUBLE ANGLE QUOTATION MARK,"""",00AB,,0
+#r# »,Repl extra space before RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK,"""",00BB,,0
+<,LESS-THAN SIGN,⟧,003C,<,0
+>,GREATER-THAN SIGN,⟦,003E,>,0
+è,LATIN SMALL LETTER E WITH GRAVE,è,00E8,e,
+ā,LATIN SMALL LETTER A WITH MACRON,a,0101,a,
+3,DIGIT THREE,3,0033,3,
+W,LATIN CAPITAL LETTER W,W,0057,W,
+Æ,LATIN CAPITAL LETTER AE,AE,00C6,AE,
+H,LATIN CAPITAL LETTER H,H,0048,H,
+t,LATIN SMALL LETTER T,t,0074,t,
+ç,LATIN SMALL LETTER C WITH CEDILLA,ç,00E7,c,
+î,LATIN SMALL LETTER I WITH CIRCUMFLEX,î,00EE,i,
+ğ,LATIN SMALL LETTER G WITH BREVE,g,011F,g,
+b,LATIN SMALL LETTER B,b,0062,b,
+M,LATIN CAPITAL LETTER M,M,004D,M,
+d,LATIN SMALL LETTER D,d,0064,d,
+ä,LATIN SMALL LETTER A WITH DIAERESIS,ä,00E4,a,
+ī,LATIN SMALL LETTER I WITH MACRON,i,012B,i,
+l,LATIN SMALL LETTER L,l,006C,l,
+A,LATIN CAPITAL LETTER A,A,0041,A,
+Y,LATIN CAPITAL LETTER Y,Y,0059,Y,
+1,DIGIT ONE,1,0031,1,
+€,EURO SIGN,€,20AC,EUR,
+’,RIGHT SINGLE QUOTATION MARK,',2019,',
+ě,LATIN SMALL LETTER E WITH CARON,e,011B,e,
+í,LATIN SMALL LETTER I WITH ACUTE,i,00ED,i,
+),RIGHT PARENTHESIS,),0029,),
+ž,LATIN SMALL LETTER Z WITH CARON,z,017E,z,
+È,LATIN CAPITAL LETTER E WITH GRAVE,È,00C8,E,
+ȩ,LATIN SMALL LETTER E WITH CEDILLA,e,0229,e,
+Ç,LATIN CAPITAL LETTER C WITH CEDILLA,Ç,00C7,C,
+!,EXCLAMATION MARK,!,0021,!,
+&,AMPERSAND,&,0026,&,
+C,LATIN CAPITAL LETTER C,C,0043,C,
+6,DIGIT SIX,6,0036,6,
+²,SUPERSCRIPT TWO,2,00B2,2,
+S,LATIN CAPITAL LETTER S,S,0053,S,
+R,LATIN CAPITAL LETTER R,R,0052,R,
+f,LATIN SMALL LETTER F,f,0066,f,
+Z,LATIN CAPITAL LETTER Z,Z,005A,Z,
+T,LATIN CAPITAL LETTER T,T,0054,T,
+â,LATIN SMALL LETTER A WITH CIRCUMFLEX,â,00E2,a,
+ú,LATIN SMALL LETTER U WITH ACUTE,ù,00FA,u,
+g,LATIN SMALL LETTER G,g,0067,g,
+2,DIGIT TWO,2,0032,2,
+Á,LATIN CAPITAL LETTER A WITH ACUTE,À,00C1,A,
+e,LATIN SMALL LETTER E,e,0065,e,
+œ,LATIN SMALL LIGATURE OE,oe,0153,oe,
+ļ,LATIN SMALL LETTER L WITH CEDILLA,l,013C,l,
+"""",QUOTATION MARK,"""",0022,"""",
+«,LEFT-POINTING DOUBLE ANGLE QUOTATION MARK,"""",00AB,,
+»,RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK,"""",00BB,,
+y,LATIN SMALL LETTER Y,y,0079,y,
+o,LATIN SMALL LETTER O,o,006F,o,
+i,LATIN SMALL LETTER I,i,0069,i,
+c,LATIN SMALL LETTER C,c,0063,c,
+Q,LATIN CAPITAL LETTER Q,Q,0051,Q,
+u,LATIN SMALL LETTER U,u,0075,u,
+ø,LATIN SMALL LETTER O WITH STROKE,o,00F8,o,
+ã,LATIN SMALL LETTER A WITH TILDE,à,00E3,a,
+q,LATIN SMALL LETTER Q,q,0071,q,
+v,LATIN SMALL LETTER V,v,0076,v,
+/,SOLIDUS,/,002F,/,
+(,LEFT PARENTHESIS,(,0028,(,
+:,COLON,;,003A,:,
+],RIGHT SQUARE BRACKET,],005D,],
+#,NUMBER SIGN,#,0023,#,
+ů,LATIN SMALL LETTER U WITH RING ABOVE,ù,016F,u,
+D,LATIN CAPITAL LETTER D,D,0044,D,
+ᑕ,CANADIAN SYLLABICS TA,C,1455,ta,
+O,LATIN CAPITAL LETTER O,O,004F,O,
+%,PERCENT SIGN,%,0025,%,
+8,DIGIT EIGHT,8,0038,8,
+G,LATIN CAPITAL LETTER G,G,0047,G,
+ì,LATIN SMALL LETTER I WITH GRAVE,î,00EC,i,
+Š,LATIN CAPITAL LETTER S WITH CARON,S,0160,S,
+4,DIGIT FOUR,4,0034,4,
+ß,LATIN SMALL LETTER SHARP S,ß,00DF,s,
+ê,LATIN SMALL LETTER E WITH CIRCUMFLEX,ê,00EA,e,
+x,LATIN SMALL LETTER X,x,0078,x,
+Ö,LATIN CAPITAL LETTER O WITH DIAERESIS,Ö,00D6,O,
+ë,LATIN SMALL LETTER E WITH DIAERESIS,ë,00EB,e,
+À,LATIN CAPITAL LETTER A WITH GRAVE,À,00C0,A,
+L,LATIN CAPITAL LETTER L,L,004C,L,
+Â,LATIN CAPITAL LETTER A WITH CIRCUMFLEX,Â,00C2,A,
+=,EQUALS SIGN,=,003D,=,
+ö,LATIN SMALL LETTER O WITH DIAERESIS,ö,00F6,o,
+ù,LATIN SMALL LETTER U WITH GRAVE,ù,00F9,u,
+0,DIGIT ZERO,0,0030,0,
+ɑ,LATIN SMALL LETTER ALPHA,a,0251,a,
+—,EM DASH,-,2014,,
+n,LATIN SMALL LETTER N,n,006E,n,
+r,LATIN SMALL LETTER R,r,0072,r,
+å,LATIN SMALL LETTER A WITH RING ABOVE,à,00E5,a,
+7,DIGIT SEVEN,7,0037,7,
+ū,LATIN SMALL LETTER U WITH MACRON,û,016B,u,
+",",COMMA,",",002C,",",
+k,LATIN SMALL LETTER K,k,006B,k,
+z,LATIN SMALL LETTER Z,z,007A,z,
+É,LATIN CAPITAL LETTER E WITH ACUTE,É,00C9,E,
+p,LATIN SMALL LETTER P,p,0070,p,
+–,EN DASH,-,2013,,
+û,LATIN SMALL LETTER U WITH CIRCUMFLEX,û,00FB,u,
+5,DIGIT FIVE,5,0035,5,
+h,LATIN SMALL LETTER H,h,0068,h,
+á,LATIN SMALL LETTER A WITH ACUTE,à,00E1,a,
+ñ,LATIN SMALL LETTER N WITH TILDE,n,00F1,n,
+Û,LATIN CAPITAL LETTER U WITH CIRCUMFLEX,Û,00DB,U,
+^,CIRCUMFLEX ACCENT,,005E,^,
+F,LATIN CAPITAL LETTER F,F,0046,F,
+m,LATIN SMALL LETTER M,m,006D,m,
+9,DIGIT NINE,9,0039,9,
+U,LATIN CAPITAL LETTER U,U,0055,U,
+ſ,LATIN SMALL LETTER LONG S,s,017F,s,
+ï,LATIN SMALL LETTER I WITH DIAERESIS,ï,00EF,i,
+a,LATIN SMALL LETTER A,a,0061,a,
+w,LATIN SMALL LETTER W,w,0077,w,
+s,LATIN SMALL LETTER S,s,0073,s,
+E,LATIN CAPITAL LETTER E,E,0045,E,
+B,LATIN CAPITAL LETTER B,B,0042,B,
+ô,LATIN SMALL LETTER O WITH CIRCUMFLEX,ô,00F4,o,
+-,HYPHEN-MINUS,-,002D,-,
+;,SEMICOLON,;,003B,;,
+.,FULL STOP,.,002E,.,
+P,LATIN CAPITAL LETTER P,P,0050,P,
+ó,LATIN SMALL LETTER O WITH ACUTE,ô,00F3,o,
+ń,LATIN SMALL LETTER N WITH ACUTE,n,0144,n,
+ō,LATIN SMALL LETTER O WITH MACRON,ô,014D,o,
+I,LATIN CAPITAL LETTER I,I,0049,I,
+ř,LATIN SMALL LETTER R WITH CARON,r,0159,r,
+J,LATIN CAPITAL LETTER J,J,004A,J,
+N,LATIN CAPITAL LETTER N,N,004E,N,
+à,LATIN SMALL LETTER A WITH GRAVE,à,00E0,a,
+V,LATIN CAPITAL LETTER V,V,0056,V,
+ü,LATIN SMALL LETTER U WITH DIAERESIS,ü,00FC,u,
+K,LATIN CAPITAL LETTER K,K,004B,K,
+é,LATIN SMALL LETTER E WITH ACUTE,é,00E9,e,
+',APOSTROPHE,',0027,',
+j,LATIN SMALL LETTER J,j,006A,j,
+X,LATIN CAPITAL LETTER X,X,0058,X,
+[,LEFT SQUARE BRACKET,[,005B,[,
+Î,LATIN CAPITAL LETTER I WITH CIRCUMFLEX,Î,00CE,I,
+Ü,LATIN CAPITAL LETTER U WITH DIAERESIS,Ü,00DC,U,
+ł,LATIN SMALL LETTER L WITH STROKE,l,0142,l,
+š,LATIN SMALL LETTER S WITH CARON,s,0161,s,
+…,HORIZONTAL ELLIPSIS,...,2026,,
+°,DEGREE SIGN,°,00B0,*,
+ɔ,LATIN SMALL LETTER OPEN O,,0254,o,
+ʁ,LATIN LETTER SMALL CAPITAL INVERTED R,,0281,R,
+ə,LATIN SMALL LETTER SCHWA,,0259,e,
+ɛ,LATIN SMALL LETTER OPEN E,,025B,e,
+ɹ,LATIN SMALL LETTER TURNED R,,0279,r,
+#r#⟧⟦,Repl Inverted pair of white square brackets,⟦⟧,,0,