tokenize 0.3.0 → 0.3.0.1
raw patch · 5 files changed
+106/−122 lines, 5 filesdep ~basedep ~criterionnew-uploader
Dependency ranges changed: base, criterion
Files
- CHANGELOG.md +5/−0
- src/NLP/Tokenize/String.hs +19/−22
- src/NLP/Tokenize/Text.hs +17/−20
- tests/src/Bench.hs +8/−16
- tokenize.cabal +57/−64
+ CHANGELOG.md view
@@ -0,0 +1,5 @@+0.3.0.1 -- 2024-08-04+---------------------++* Update benchmark to `criterion >= 1`.+* Tested with GHC 7.10 - 9.10 (Cabal) and 8.2 - 9.8 (Stack).
src/NLP/Tokenize/String.hs view
@@ -1,4 +1,4 @@-module NLP.Tokenize.String +module NLP.Tokenize.String ( EitherList(..) , Tokenizer , tokenize@@ -18,7 +18,6 @@ import qualified Data.Char as Char import Data.List import Data.Maybe-import Control.Monad.Instances import Control.Applicative import Data.List.Split import Control.Monad@@ -27,11 +26,11 @@ -- (wrapped in a newtype). Right Strings will be passed on for processing -- to tokenizers down -- the pipeline. Left Strings will be passed through the pipeline unchanged.--- Use a Left String in a tokenizer to protect certain tokens from further --- processing (e.g. see the 'uris' tokenizer). +-- Use a Left String in a tokenizer to protect certain tokens from further+-- processing (e.g. see the 'uris' tokenizer). -- You can define your own custom tokenizer pipelines by chaining tokenizers together: ------ > myTokenizer :: Tokenizer +-- > myTokenizer :: Tokenizer -- > myTokenizer = whitespace >=> allPunctuation --- @@ -40,7 +39,7 @@ -- | The EitherList is a newtype-wrapped list of Eithers. newtype EitherList a b = E { unE :: [Either a b] } --- | Split string into words using the default tokenizer pipeline +-- | Split string into words using the default tokenizer pipeline tokenize :: String -> [String] tokenize = run defaultTokenizer @@ -49,11 +48,11 @@ run f = map unwrap . unE . f defaultTokenizer :: Tokenizer-defaultTokenizer = whitespace - >=> uris - >=> punctuation - >=> contractions - >=> negatives +defaultTokenizer = whitespace+ >=> uris+ >=> punctuation+ >=> contractions+ >=> negatives -- | Detect common uris and freeze them uris :: Tokenizer@@ -62,7 +61,7 @@ where isUri x = any (`isPrefixOf` x) ["http://","ftp://","mailto:"] -- | Split off initial and final punctuation-punctuation :: Tokenizer +punctuation :: Tokenizer punctuation = finalPunctuation >=> initialPunctuation -- | Split off word-final punctuation@@ -84,10 +83,10 @@ -- defaultTokenizer pipeline because dealing with word-internal -- punctuation is quite application specific. allPunctuation :: Tokenizer-allPunctuation = E . map Right - . groupBy (\a b -> Char.isPunctuation a == Char.isPunctuation b) - --- | Split words ending in n't, and freeze n't +allPunctuation = E . map Right+ . groupBy (\a b -> Char.isPunctuation a == Char.isPunctuation b)++-- | Split words ending in n't, and freeze n't negatives :: Tokenizer negatives x | "n't" `isSuffixOf` x = E [ Right . reverse . drop 3 . reverse $ x , Left "n't" ]@@ -100,10 +99,10 @@ [] -> return x ((w,s):_) -> E [ Right w,Left s] where cts = ["'m","'s","'d","'ve","'ll"]- splitSuffix w sfx = + splitSuffix w sfx = let w' = reverse w len = length sfx- in if sfx `isSuffixOf` w + in if sfx `isSuffixOf` w then Just (take (length w - len) w, reverse . take len $ w') else Nothing @@ -113,11 +112,10 @@ whitespace xs = E [Right w | w <- words xs ] instance Monad (EitherList a) where- return x = E [Right x] E xs >>= f = E $ concatMap (either (return . Left) (unE . f)) xs instance Applicative (EitherList a) where- pure x = return x+ pure x = E [Right x] f <*> x = f `ap` x instance Functor (EitherList a) where@@ -126,7 +124,7 @@ unwrap (Left x) = x unwrap (Right x) = x -examples = +examples = ["This shouldn't happen." ,"Some 'quoted' stuff" ,"This is a URL: http://example.org."@@ -137,4 +135,3 @@ ,"Hyphen-words" ,"Yes/No questions" ]-
src/NLP/Tokenize/Text.hs view
@@ -21,7 +21,6 @@ import qualified Data.Char as Char import Data.Maybe-import Control.Monad.Instances () import Control.Applicative import Control.Monad @@ -32,11 +31,11 @@ -- (wrapped in a newtype). Right Texts will be passed on for processing -- to tokenizers down -- the pipeline. Left Texts will be passed through the pipeline unchanged.--- Use a Left Texts in a tokenizer to protect certain tokens from further --- processing (e.g. see the 'uris' tokenizer). +-- Use a Left Texts in a tokenizer to protect certain tokens from further+-- processing (e.g. see the 'uris' tokenizer). -- You can define your own custom tokenizer pipelines by chaining tokenizers together: ------ > myTokenizer :: Tokenizer +-- > myTokenizer :: Tokenizer -- > myTokenizer = whitespace >=> allPunctuation --- @@ -45,7 +44,7 @@ -- | The EitherList is a newtype-wrapped list of Eithers. newtype EitherList a b = E { unE :: [Either a b] } --- | Split string into words using the default tokenizer pipeline +-- | Split string into words using the default tokenizer pipeline tokenize :: Text -> [Text] tokenize = run defaultTokenizer @@ -54,11 +53,11 @@ run f = \txt -> map T.copy $ (map unwrap . unE . f) txt defaultTokenizer :: Tokenizer-defaultTokenizer = whitespace - >=> uris - >=> punctuation - >=> contractions - >=> negatives +defaultTokenizer = whitespace+ >=> uris+ >=> punctuation+ >=> contractions+ >=> negatives -- | Detect common uris and freeze them uris :: Tokenizer@@ -67,7 +66,7 @@ where isUri u = any (`T.isPrefixOf` u) ["http://","ftp://","mailto:"] -- | Split off initial and final punctuation-punctuation :: Tokenizer +punctuation :: Tokenizer punctuation = finalPunctuation >=> initialPunctuation hyphens :: Tokenizer@@ -98,10 +97,10 @@ -- defaultTokenizer pipeline because dealing with word-internal -- punctuation is quite application specific. allPunctuation :: Tokenizer-allPunctuation = E . map Right - . T.groupBy (\a b -> Char.isPunctuation a == Char.isPunctuation b) +allPunctuation = E . map Right+ . T.groupBy (\a b -> Char.isPunctuation a == Char.isPunctuation b) --- | Split words ending in n't, and freeze n't +-- | Split words ending in n't, and freeze n't negatives :: Tokenizer negatives x | "n't" `T.isSuffixOf` x = E [ Right . T.reverse . T.drop 3 . T.reverse $ x , Left "n't" ]@@ -114,10 +113,10 @@ [] -> return x ((w,s):_) -> E [ Right w,Left s] where cts = ["'m","'s","'d","'ve","'ll"]- splitSuffix w sfx = + splitSuffix w sfx = let w' = T.reverse w len = T.length sfx- in if sfx `T.isSuffixOf` w + in if sfx `T.isSuffixOf` w then Just (T.take (T.length w - len) w, T.reverse . T.take len $ w') else Nothing @@ -127,11 +126,10 @@ whitespace xs = E [Right w | w <- T.words xs ] instance Monad (EitherList a) where- return x = E [Right x] E xs >>= f = E $ concatMap (either (return . Left) (unE . f)) xs instance Applicative (EitherList a) where- pure x = return x+ pure x = E [Right x] f <*> x = f `ap` x instance Functor (EitherList a) where@@ -142,7 +140,7 @@ unwrap (Right x) = x examples :: [Text]-examples = +examples = ["This shouldn't happen." ,"Some 'quoted' stuff" ,"This is a URL: http://example.org."@@ -153,4 +151,3 @@ ,"Hyphen-words" ,"Yes/No questions" ]-
tests/src/Bench.hs view
@@ -9,21 +9,13 @@ import Control.DeepSeq import Criterion.Main-import Criterion.Config (defaultConfig, Config(..), ljust) - import System.Environment (getArgs, withArgs) import qualified NLP.Tokenize.String as StrTok import qualified NLP.Tokenize.Text as TextTok -myConfig :: Config-myConfig = defaultConfig {- -- Always GC between runs.- cfgPerformGC = ljust True- }- readF :: FilePath -> IO Text readF file = do bs <- BS.readFile file@@ -37,14 +29,14 @@ (f:rest) -> do plugTxt <- mapM readF [f] let plugStr = map T.unpack plugTxt- deepseq plugStr $ withArgs rest $ defaultMainWith myConfig (return ())- [ bgroup "tokenizing"- [ bench "Native String Tokenizer" $ nf (map StrTok.tokenize) plugStr- , bench "Native Text Tokenizer" $ nf (map TextTok.tokenize) plugTxt- , bench "Text->Text based on String Tokenizer" $ nf (map strTokenizer) plugTxt- , bench "String->String based on Text Tokenizer" $ nf (map txtTokenizer) plugStr- ]- ]+ deepseq plugStr $ withArgs rest $ defaultMain+ [ bgroup "tokenizing"+ [ bench "Native String Tokenizer" $ nf (map StrTok.tokenize) plugStr+ , bench "Native Text Tokenizer" $ nf (map TextTok.tokenize) plugTxt+ , bench "Text->Text based on String Tokenizer" $ nf (map strTokenizer) plugTxt+ , bench "String->String based on Text Tokenizer" $ nf (map txtTokenizer) plugStr+ ]+ ] strTokenizer :: Text -> [Text] strTokenizer txt = map T.pack (StrTok.tokenize $ T.unpack txt)
tokenize.cabal view
@@ -1,76 +1,69 @@--- tokenize.cabal auto-generated by cabal init. For additional--- options, see--- http://www.haskell.org/cabal/release/cabal-latest/doc/users-guide/authors.html#pkg-descr.--- The name of the package.-Name: tokenize---- The package version. See the Haskell package versioning policy--- (http://www.haskell.org/haskellwiki/Package_versioning_policy) for--- standards guiding when and how versions should be incremented.-Version: 0.3.0---- A short (one-line) description of the package.-Synopsis: Simple tokenizer for English text.---- A longer description of the package.-Description: Simple tokenizer for English text.---- The license under which the package is released.-License: BSD3---- The file containing the license text.-License-file: LICENSE---- The package author(s).-Author: Grzegorz Chrupała---- An email address to which users can send suggestions, bug reports,--- and patches.-Maintainer: grzegorz.chrupala@gmail.com--Homepage: https://bitbucket.org/gchrupala/lingo/overview---- A copyright notice.--- Copyright: --Category: Natural Language Processing+cabal-version: >=1.10+name: tokenize+version: 0.3.0.1 -Build-type: Simple+synopsis: Simple tokenizer for English text+description: Simple tokenizer for English text.+license: BSD3+license-file: LICENSE+author: Grzegorz Chrupała+maintainer: Andreas Abel+homepage: https://github.com/haskell/tokenize+bug-reports: https://github.com/haskell/tokenize/issues+category: Natural Language Processing+build-type: Simple --- Extra files to be distributed with the package, such as examples or--- a README.--- Extra-source-files: +tested-with:+ GHC == 9.10.0+ GHC == 9.8.2+ GHC == 9.6.4+ GHC == 9.4.8+ GHC == 9.2.8+ GHC == 9.0.2+ GHC == 8.10.7+ GHC == 8.8.4+ GHC == 8.6.5+ GHC == 8.4.4+ GHC == 8.2.2+ GHC == 8.0.2+ GHC == 7.10.3 --- Constraint on the version of Cabal needed to build this package.-Cabal-version: >=1.10+extra-source-files:+ CHANGELOG.md +source-repository head+ type: git+ location: https://github.com/haskell/tokenize -Library- default-language: Haskell2010- hs-source-dirs: src- -- Modules exported by the library.- Exposed-modules: NLP.Tokenize- NLP.Tokenize.Text- NLP.Tokenize.String+library+ hs-source-dirs: src+ exposed-modules:+ NLP.Tokenize+ NLP.Tokenize.Text+ NLP.Tokenize.String -- Packages needed in order to build this package.- Build-depends: base >= 3 && < 5,- split >= 0.1,- text+ build-depends:+ base >= 4 && < 5+ , split >= 0.1+ , text -Benchmark bench- type: exitcode-stdio-1.0- default-language: Haskell2010- Main-Is: Bench.hs+ default-language: Haskell2010++benchmark bench+ type: exitcode-stdio-1.0+ main-is: Bench.hs hs-source-dirs: tests/src - Build-depends: tokenize,- criterion,- filepath >= 1.3.0.1,- text >= 0.11.3.0,- base >= 4 && <= 6,- deepseq,- split >= 0.1.2.3,- bytestring+ build-depends:+ tokenize+ , base+ , bytestring+ , criterion >= 1+ , deepseq+ , filepath >= 1.3.0.1+ , split >= 0.1.2.3+ , text >= 0.11.3.0 + default-language: Haskell2010 ghc-options: -Wall -main-is Bench