tokenize 0.2.0 → 0.2.2

raw patch · 3 files changed

+42/−7 lines, 3 files

Files

src/NLP/Tokenize/String.hs +18/−1
src/NLP/Tokenize/Text.hs +21/−4
tokenize.cabal +3/−2

src/NLP/Tokenize/String.hs view

@@ -9,6 +9,7 @@     , punctuation     , finalPunctuation     , initialPunctuation+    , allPunctuation     , contractions     , negatives     )@@ -26,7 +27,13 @@ --  to tokenizers down --  the pipeline. Left Strings will be passed through the pipeline unchanged. --  Use a Left String in a tokenizer to protect certain tokens from further ---  processing (e.g. see the 'uris' tokenizer).+--  processing (e.g. see the 'uris' tokenizer). +--  You can define your own custom tokenizer pipelines by chaining tokenizers together:+---+-- > myTokenizer :: Tokenizer +-- > myTokenizer = whitespace >=> allPunctuation+---+ type Tokenizer =  String -> EitherList String String  -- | The EitherList is a newtype-wrapped list of Eithers.@@ -71,6 +78,14 @@       ([],w) -> [Right w]       (ps,w) -> [Right ps, Right w] +-- | Split tokens on transitions between punctuation and+-- non-punctuation characters. This tokenizer is not included in+-- defaultTokenizer pipeline because dealing with word-internal+-- punctuation is quite application specific.+allPunctuation :: Tokenizer+allPunctuation = E . map Right +                 . groupBy (\a b -> Char.isPunctuation a == Char.isPunctuation b) +                  -- | Split words ending in n't, and freeze n't  negatives :: Tokenizer negatives x | "n't" `isSuffixOf` x = E [ Right . reverse . drop 3 . reverse $ x@@ -111,5 +126,7 @@     ,"ReferenceError #1065 broke my debugger!"     ,"I would've gone."     ,"They've been there."+    ,"Hyphen-words"+    ,"Yes/No questions"     ]

src/NLP/Tokenize/Text.hs view

@@ -13,6 +13,7 @@     , punctuation     , finalPunctuation     , initialPunctuation+    , allPunctuation     , contractions     , negatives     )@@ -27,11 +28,17 @@ import qualified Data.Text as T  -- | A Tokenizer is function which takes a list and returns a list of Eithers---  (wrapped in a newtype). Right Strings will be passed on for processing+--  (wrapped in a newtype). Right Texts will be passed on for processing --  to tokenizers down---  the pipeline. Left Strings will be passed through the pipeline unchanged.---  Use a Left String in a tokenizer to protect certain tokens from further ---  processing (e.g. see the 'uris' tokenizer).+--  the pipeline. Left Texts will be passed through the pipeline unchanged.+--  Use a Left Texts in a tokenizer to protect certain tokens from further +--  processing (e.g. see the 'uris' tokenizer). +--  You can define your own custom tokenizer pipelines by chaining tokenizers together:+---+-- > myTokenizer :: Tokenizer +-- > myTokenizer = whitespace >=> allPunctuation+---+ type Tokenizer =  Text -> EitherList Text Text  -- | The EitherList is a newtype-wrapped list of Eithers.@@ -85,6 +92,14 @@              | otherwise -> [ Right ps                             , Right w ] +-- | Split tokens on transitions between punctuation and+-- non-punctuation characters. This tokenizer is not included in+-- defaultTokenizer pipeline because dealing with word-internal+-- punctuation is quite application specific.+allPunctuation :: Tokenizer+allPunctuation = E . map Right +                 . T.groupBy (\a b -> Char.isPunctuation a == Char.isPunctuation b) + -- | Split words ending in n't, and freeze n't  negatives :: Tokenizer negatives x | "n't" `T.isSuffixOf` x = E [ Right . T.reverse . T.drop 3 . T.reverse $ x@@ -127,5 +142,7 @@     ,"ReferenceError #1065 broke my debugger!"     ,"I would've gone."     ,"They've been there."+    ,"Hyphen-words"+    ,"Yes/No questions"     ]

tokenize.cabal view

@@ -7,7 +7,7 @@ -- The package version. See the Haskell package versioning policy -- (http://www.haskell.org/haskellwiki/Package_versioning_policy) for -- standards guiding when and how versions should be incremented.-Version:             0.2.0+Version:             0.2.2  -- A short (one-line) description of the package. Synopsis:            Simple tokenizer for English text.@@ -58,7 +58,8 @@                 split >= 0.1,                 text -Executable bench+Benchmark bench+   type: exitcode-stdio-1.0    default-language: Haskell2010    Main-Is:          Bench.hs    hs-source-dirs:   tests/src