tokenize 0.1.0 → 0.1.1

raw patch · 2 files changed

+6/−2 lines, 2 files

Files

NLP/Tokenize.hs +5/−1
tokenize.cabal +1/−1

NLP/Tokenize.hs view

@@ -3,6 +3,7 @@     , Tokenizer     , tokenize     , run+    , defaultTokenizer     , whitespace     , uris     , punctuation@@ -31,11 +32,14 @@  -- | Split string into words using the default tokenizer pipeline  tokenize :: String -> [String]-tokenize  = run (whitespace >=> uris >=> punctuation >=> negatives)+tokenize  = run defaultTokenizer  -- | Run a tokenizer run :: Tokenizer -> (String -> [String]) run f = map unwrap . unE . f++defaultTokenizer :: Tokenizer+defaultTokenizer = whitespace >=> uris >=> punctuation >=> negatives  -- | Detect common uris and freeze them uris :: Tokenizer

tokenize.cabal view

@@ -7,7 +7,7 @@ -- The package version. See the Haskell package versioning policy -- (http://www.haskell.org/haskellwiki/Package_versioning_policy) for -- standards guiding when and how versions should be incremented.-Version:             0.1.0+Version:             0.1.1  -- A short (one-line) description of the package. Synopsis:            Simple tokenizer for English text.