Skip to content

Commit

Permalink
updated all NUMBER rules to also include class Mn, NON-spacing marker…
Browse files Browse the repository at this point in the history 
…s as part of a number
  • Loading branch information
@kosloot
kosloot committed Apr 2, 2024
1 parent 01c557c commit a59b77c
Show file tree
Hide file tree
Showing 17 changed files with 34 additions and 35 deletions.
4 changes: 2 additions & 2 deletions config/tokconfig-deu
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers anf non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-eng
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ NUMBER-YEAR=('\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:a\.?m\.?|p\.?m\.?)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-eng-twitter
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ NUMBER-YEAR=('\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:a\.?m\.?|p\.?m\.?)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers anf non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-fra
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers anf non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-fry
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers anf non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-generic
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:a\.?m\.?|p\.?m\.?)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers anf non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-ita
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-nld
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N}{1,2})?(am|pm|AM|PM)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

#'s-Gravenhage, 's-Hertogenbosch
GENITIVEPLACENAME=['`’‘´][sS]-\p{Lu}\p{L}+
Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-nld-historical
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N}{1,2})?(am|pm|AM|PM)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers anf non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

#'s-Gravenhage, 's-Hertogenbosch
GENITIVEPLACENAME=['`’‘´][sS]-\p{Lu}\p{L}+
Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-nld-sonarchat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
5 changes: 2 additions & 3 deletions config/tokconfig-nld-twitter
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N}{1,2})?(am|pm|AM|PM)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+

#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-nld-withplaceholder
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N}{1,2})?(am|pm|AM|PM)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-por
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-rus
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:a\.?m\.?|p\.?m\.?)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-spa
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-swe
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)
#Times
TIME=\p{N}{1,2}\.\p{N}{1,2}(?:\.\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

CURRENCY=\p{Sc}

Expand Down
4 changes: 2 additions & 2 deletions config/tokconfig-tur
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ FRACNUMBER=\p{N}+(?:/\p{N}+)+
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:a\.?m\.?|p\.?m\.?)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#retain digits, including those starting with initial period (.22), and negative numbers and non-spacing markers
NUMBER=-?(?:[\.,]?[\p{N}+|\p{Mn}])+

ROMAN-NUMERALS=\b[IVXivx]{1,2}\.

Expand Down

0 comments on commit a59b77c

Please sign in to comment.