tokenize 0.1.3 → 0.3.0.1
raw patch · 7 files changed
Files
- CHANGELOG.md +5/−0
- NLP/Tokenize.hs +0/−115
- src/NLP/Tokenize.hs +8/−0
- src/NLP/Tokenize/String.hs +137/−0
- src/NLP/Tokenize/Text.hs +153/−0
- tests/src/Bench.hs +45/−0
- tokenize.cabal +59/−45
+ CHANGELOG.md view
@@ -0,0 +1,5 @@+0.3.0.1 -- 2024-08-04+---------------------++* Update benchmark to `criterion >= 1`.+* Tested with GHC 7.10 - 9.10 (Cabal) and 8.2 - 9.8 (Stack).
− NLP/Tokenize.hs
@@ -1,115 +0,0 @@-module NLP.Tokenize - ( EitherList(..)- , Tokenizer- , tokenize- , run- , defaultTokenizer- , whitespace- , uris- , punctuation- , finalPunctuation- , initialPunctuation- , contractions- , negatives- )-where--import qualified Data.Char as Char-import Data.List-import Data.Maybe-import Control.Monad.Instances-import Data.List.Split-import Control.Monad---- | A Tokenizer is function which takes a list and returns a list of Eithers--- (wrapped in a newtype). Right Strings will be passed on for processing--- to tokenizers down--- the pipeline. Left Strings will be passed through the pipeline unchanged.--- Use a Left String in a tokenizer to protect certain tokens from further --- processing (e.g. see the 'uris' tokenizer).-type Tokenizer = String -> EitherList String String---- | The EitherList is a newtype-wrapped list of Eithers.-newtype EitherList a b = E { unE :: [Either a b] }---- | Split string into words using the default tokenizer pipeline -tokenize :: String -> [String]-tokenize = run defaultTokenizer---- | Run a tokenizer-run :: Tokenizer -> (String -> [String])-run f = map unwrap . unE . f--defaultTokenizer :: Tokenizer-defaultTokenizer = whitespace - >=> uris - >=> punctuation - >=> contractions - >=> negatives ---- | Detect common uris and freeze them-uris :: Tokenizer-uris x | isUri x = E [Left x]- | True = E [Right x]- where isUri x = any (`isPrefixOf` x) ["http://","ftp://","mailto:"]---- | Split off initial and final punctuation-punctuation :: Tokenizer -punctuation = finalPunctuation >=> initialPunctuation---- | Split off word-final punctuation-finalPunctuation :: Tokenizer-finalPunctuation x = E . filter (not . null . unwrap) $- case span Char.isPunctuation . reverse $ x of- ([],w) -> [Right . reverse $ w]- (ps,w) -> [Right . reverse $ w, Right . reverse $ ps]---- | Split off word-initial punctuation-initialPunctuation :: Tokenizer-initialPunctuation x = E . filter (not . null . unwrap) $- case span Char.isPunctuation$ x of- ([],w) -> [Right w]- (ps,w) -> [Right ps, Right w]---- | Split words ending in n't, and freeze n't -negatives :: Tokenizer-negatives x | "n't" `isSuffixOf` x = E [ Right . reverse . drop 3 . reverse $ x- , Left "n't" ]- | True = E [Right x]---- | Split common contractions off and freeze them.--- | Currently deals with: 'm, 's, 'd, 've, 'll-contractions :: Tokenizer-contractions x = case catMaybes . map (splitSuffix x) $ cts of- [] -> return x- ((w,s):_) -> E [ Right w,Left s]- where cts = ["'m","'s","'d","'ve","'ll"]- splitSuffix w sfx = - let w' = reverse w- len = length sfx- in if sfx `isSuffixOf` w - then Just (take (length w - len) w, reverse . take len $ w')- else Nothing----- | Split string on whitespace. This is just a wrapper for Data.List.words-whitespace :: Tokenizer-whitespace xs = E [Right w | w <- words xs ]--instance Monad (EitherList a) where- return x = E [Right x]- E xs >>= f = E $ concatMap (either (return . Left) (unE . f)) xs--unwrap (Left x) = x-unwrap (Right x) = x--examples = - ["This shouldn't happen."- ,"Some 'quoted' stuff"- ,"This is a URL: http://example.org."- ,"How about an email@example.com"- ,"ReferenceError #1065 broke my debugger!"- ,"I would've gone."- ,"They've been there."- ]-
+ src/NLP/Tokenize.hs view
@@ -0,0 +1,8 @@+{-# LANGUAGE OverloadedStrings #-}+-- | NLP Tokenizer+module NLP.Tokenize+ ( module NLP.Tokenize.String+ )+where++import NLP.Tokenize.String
+ src/NLP/Tokenize/String.hs view
@@ -0,0 +1,137 @@+module NLP.Tokenize.String+ ( EitherList(..)+ , Tokenizer+ , tokenize+ , run+ , defaultTokenizer+ , whitespace+ , uris+ , punctuation+ , finalPunctuation+ , initialPunctuation+ , allPunctuation+ , contractions+ , negatives+ )+where++import qualified Data.Char as Char+import Data.List+import Data.Maybe+import Control.Applicative+import Data.List.Split+import Control.Monad++-- | A Tokenizer is function which takes a list and returns a list of Eithers+-- (wrapped in a newtype). Right Strings will be passed on for processing+-- to tokenizers down+-- the pipeline. Left Strings will be passed through the pipeline unchanged.+-- Use a Left String in a tokenizer to protect certain tokens from further+-- processing (e.g. see the 'uris' tokenizer).+-- You can define your own custom tokenizer pipelines by chaining tokenizers together:+---+-- > myTokenizer :: Tokenizer+-- > myTokenizer = whitespace >=> allPunctuation+---++type Tokenizer = String -> EitherList String String++-- | The EitherList is a newtype-wrapped list of Eithers.+newtype EitherList a b = E { unE :: [Either a b] }++-- | Split string into words using the default tokenizer pipeline+tokenize :: String -> [String]+tokenize = run defaultTokenizer++-- | Run a tokenizer+run :: Tokenizer -> (String -> [String])+run f = map unwrap . unE . f++defaultTokenizer :: Tokenizer+defaultTokenizer = whitespace+ >=> uris+ >=> punctuation+ >=> contractions+ >=> negatives++-- | Detect common uris and freeze them+uris :: Tokenizer+uris x | isUri x = E [Left x]+ | True = E [Right x]+ where isUri x = any (`isPrefixOf` x) ["http://","ftp://","mailto:"]++-- | Split off initial and final punctuation+punctuation :: Tokenizer+punctuation = finalPunctuation >=> initialPunctuation++-- | Split off word-final punctuation+finalPunctuation :: Tokenizer+finalPunctuation x = E . filter (not . null . unwrap) $+ case span Char.isPunctuation . reverse $ x of+ ([],w) -> [Right . reverse $ w]+ (ps,w) -> [Right . reverse $ w, Right . reverse $ ps]++-- | Split off word-initial punctuation+initialPunctuation :: Tokenizer+initialPunctuation x = E . filter (not . null . unwrap) $+ case span Char.isPunctuation$ x of+ ([],w) -> [Right w]+ (ps,w) -> [Right ps, Right w]++-- | Split tokens on transitions between punctuation and+-- non-punctuation characters. This tokenizer is not included in+-- defaultTokenizer pipeline because dealing with word-internal+-- punctuation is quite application specific.+allPunctuation :: Tokenizer+allPunctuation = E . map Right+ . groupBy (\a b -> Char.isPunctuation a == Char.isPunctuation b)++-- | Split words ending in n't, and freeze n't+negatives :: Tokenizer+negatives x | "n't" `isSuffixOf` x = E [ Right . reverse . drop 3 . reverse $ x+ , Left "n't" ]+ | True = E [Right x]++-- | Split common contractions off and freeze them.+-- | Currently deals with: 'm, 's, 'd, 've, 'll+contractions :: Tokenizer+contractions x = case catMaybes . map (splitSuffix x) $ cts of+ [] -> return x+ ((w,s):_) -> E [ Right w,Left s]+ where cts = ["'m","'s","'d","'ve","'ll"]+ splitSuffix w sfx =+ let w' = reverse w+ len = length sfx+ in if sfx `isSuffixOf` w+ then Just (take (length w - len) w, reverse . take len $ w')+ else Nothing+++-- | Split string on whitespace. This is just a wrapper for Data.List.words+whitespace :: Tokenizer+whitespace xs = E [Right w | w <- words xs ]++instance Monad (EitherList a) where+ E xs >>= f = E $ concatMap (either (return . Left) (unE . f)) xs++instance Applicative (EitherList a) where+ pure x = E [Right x]+ f <*> x = f `ap` x++instance Functor (EitherList a) where+ fmap f (E xs) = E $ (fmap . fmap) f xs++unwrap (Left x) = x+unwrap (Right x) = x++examples =+ ["This shouldn't happen."+ ,"Some 'quoted' stuff"+ ,"This is a URL: http://example.org."+ ,"How about an email@example.com"+ ,"ReferenceError #1065 broke my debugger!"+ ,"I would've gone."+ ,"They've been there."+ ,"Hyphen-words"+ ,"Yes/No questions"+ ]
+ src/NLP/Tokenize/Text.hs view
@@ -0,0 +1,153 @@+{-# LANGUAGE OverloadedStrings #-}+-- | NLP Tokenizer, adapted to use Text instead of Strings from the+-- `tokenize` package.++module NLP.Tokenize.Text+ ( EitherList(..)+ , Tokenizer+ , tokenize+ , run+ , defaultTokenizer+ , whitespace+ , uris+ , punctuation+ , finalPunctuation+ , initialPunctuation+ , allPunctuation+ , contractions+ , negatives+ )+where++import qualified Data.Char as Char+import Data.Maybe+import Control.Applicative+import Control.Monad++import Data.Text (Text)+import qualified Data.Text as T++-- | A Tokenizer is function which takes a list and returns a list of Eithers+-- (wrapped in a newtype). Right Texts will be passed on for processing+-- to tokenizers down+-- the pipeline. Left Texts will be passed through the pipeline unchanged.+-- Use a Left Texts in a tokenizer to protect certain tokens from further+-- processing (e.g. see the 'uris' tokenizer).+-- You can define your own custom tokenizer pipelines by chaining tokenizers together:+---+-- > myTokenizer :: Tokenizer+-- > myTokenizer = whitespace >=> allPunctuation+---++type Tokenizer = Text -> EitherList Text Text++-- | The EitherList is a newtype-wrapped list of Eithers.+newtype EitherList a b = E { unE :: [Either a b] }++-- | Split string into words using the default tokenizer pipeline+tokenize :: Text -> [Text]+tokenize = run defaultTokenizer++-- | Run a tokenizer+run :: Tokenizer -> (Text -> [Text])+run f = \txt -> map T.copy $ (map unwrap . unE . f) txt++defaultTokenizer :: Tokenizer+defaultTokenizer = whitespace+ >=> uris+ >=> punctuation+ >=> contractions+ >=> negatives++-- | Detect common uris and freeze them+uris :: Tokenizer+uris x | isUri x = E [Left x]+ | True = E [Right x]+ where isUri u = any (`T.isPrefixOf` u) ["http://","ftp://","mailto:"]++-- | Split off initial and final punctuation+punctuation :: Tokenizer+punctuation = finalPunctuation >=> initialPunctuation++hyphens :: Tokenizer+hyphens xs = E [Right w | w <- T.split (=='-') xs ]++-- | Split off word-final punctuation+finalPunctuation :: Tokenizer+finalPunctuation x = E $ filter (not . T.null . unwrap) res+ where+ res :: [Either Text Text]+ res = case T.span Char.isPunctuation (T.reverse x) of+ (ps, w) | T.null ps -> [ Right $ T.reverse w ]+ | otherwise -> [ Right $ T.reverse w+ , Right $ T.reverse ps]+ -- ([],w) -> [Right . T.reverse $ w]+ -- (ps,w) -> [Right . T.reverse $ w, Right . T.reverse $ ps]++-- | Split off word-initial punctuation+initialPunctuation :: Tokenizer+initialPunctuation x = E $ filter (not . T.null . unwrap) $+ case T.span Char.isPunctuation x of+ (ps,w) | T.null ps -> [ Right w ]+ | otherwise -> [ Right ps+ , Right w ]++-- | Split tokens on transitions between punctuation and+-- non-punctuation characters. This tokenizer is not included in+-- defaultTokenizer pipeline because dealing with word-internal+-- punctuation is quite application specific.+allPunctuation :: Tokenizer+allPunctuation = E . map Right+ . T.groupBy (\a b -> Char.isPunctuation a == Char.isPunctuation b)++-- | Split words ending in n't, and freeze n't+negatives :: Tokenizer+negatives x | "n't" `T.isSuffixOf` x = E [ Right . T.reverse . T.drop 3 . T.reverse $ x+ , Left "n't" ]+ | True = E [ Right x ]++-- | Split common contractions off and freeze them.+-- | Currently deals with: 'm, 's, 'd, 've, 'll+contractions :: Tokenizer+contractions x = case catMaybes . map (splitSuffix x) $ cts of+ [] -> return x+ ((w,s):_) -> E [ Right w,Left s]+ where cts = ["'m","'s","'d","'ve","'ll"]+ splitSuffix w sfx =+ let w' = T.reverse w+ len = T.length sfx+ in if sfx `T.isSuffixOf` w+ then Just (T.take (T.length w - len) w, T.reverse . T.take len $ w')+ else Nothing+++-- | Split string on whitespace. This is just a wrapper for Data.List.words+whitespace :: Tokenizer+whitespace xs = E [Right w | w <- T.words xs ]++instance Monad (EitherList a) where+ E xs >>= f = E $ concatMap (either (return . Left) (unE . f)) xs++instance Applicative (EitherList a) where+ pure x = E [Right x]+ f <*> x = f `ap` x++instance Functor (EitherList a) where+ fmap f (E xs) = E $ (fmap . fmap) f xs++unwrap :: Either a a -> a+unwrap (Left x) = x+unwrap (Right x) = x++examples :: [Text]+examples =+ ["This shouldn't happen."+ ,"Some 'quoted' stuff"+ ,"This is a URL: http://example.org."+ ,"How about an email@example.com"+ ,"ReferenceError #1065 broke my debugger!"+ ,"I would've gone."+ ,"They've been there."+ ,"Hyphen-words"+ ,"Yes/No questions"+ ]
+ tests/src/Bench.hs view
@@ -0,0 +1,45 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE PackageImports #-}+module Bench where++import qualified Data.ByteString as BS+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE++import Control.DeepSeq+import Criterion.Main++import System.Environment (getArgs, withArgs)++import qualified NLP.Tokenize.String as StrTok+import qualified NLP.Tokenize.Text as TextTok+++readF :: FilePath -> IO Text+readF file = do+ bs <- BS.readFile file+ return $ TE.decodeLatin1 bs++main :: IO ()+main = do+ args <- getArgs+ case args of+ [] -> putStrLn "Usage: bench <input_text_file>"+ (f:rest) -> do+ plugTxt <- mapM readF [f]+ let plugStr = map T.unpack plugTxt+ deepseq plugStr $ withArgs rest $ defaultMain+ [ bgroup "tokenizing"+ [ bench "Native String Tokenizer" $ nf (map StrTok.tokenize) plugStr+ , bench "Native Text Tokenizer" $ nf (map TextTok.tokenize) plugTxt+ , bench "Text->Text based on String Tokenizer" $ nf (map strTokenizer) plugTxt+ , bench "String->String based on Text Tokenizer" $ nf (map txtTokenizer) plugStr+ ]+ ]++strTokenizer :: Text -> [Text]+strTokenizer txt = map T.pack (StrTok.tokenize $ T.unpack txt)++txtTokenizer :: String -> [String]+txtTokenizer str = map T.unpack (TextTok.tokenize $ T.pack str)
tokenize.cabal view
@@ -1,55 +1,69 @@--- tokenize.cabal auto-generated by cabal init. For additional--- options, see--- http://www.haskell.org/cabal/release/cabal-latest/doc/users-guide/authors.html#pkg-descr.--- The name of the package.-Name: tokenize---- The package version. See the Haskell package versioning policy--- (http://www.haskell.org/haskellwiki/Package_versioning_policy) for--- standards guiding when and how versions should be incremented.-Version: 0.1.3---- A short (one-line) description of the package.-Synopsis: Simple tokenizer for English text.---- A longer description of the package.-Description: Simple tokenizer for English text.---- The license under which the package is released.-License: BSD3---- The file containing the license text.-License-file: LICENSE+cabal-version: >=1.10+name: tokenize+version: 0.3.0.1 --- The package author(s).-Author: Grzegorz Chrupała+synopsis: Simple tokenizer for English text+description: Simple tokenizer for English text.+license: BSD3+license-file: LICENSE+author: Grzegorz Chrupała+maintainer: Andreas Abel+homepage: https://github.com/haskell/tokenize+bug-reports: https://github.com/haskell/tokenize/issues+category: Natural Language Processing+build-type: Simple --- An email address to which users can send suggestions, bug reports,--- and patches.-Maintainer: gchrupala@lsv.uni-saarland.de+tested-with:+ GHC == 9.10.0+ GHC == 9.8.2+ GHC == 9.6.4+ GHC == 9.4.8+ GHC == 9.2.8+ GHC == 9.0.2+ GHC == 8.10.7+ GHC == 8.8.4+ GHC == 8.6.5+ GHC == 8.4.4+ GHC == 8.2.2+ GHC == 8.0.2+ GHC == 7.10.3 -Homepage: https://bitbucket.org/gchrupala/lingo/overview+extra-source-files:+ CHANGELOG.md --- A copyright notice.--- Copyright: +source-repository head+ type: git+ location: https://github.com/haskell/tokenize -Category: Natural Language Processing+library+ hs-source-dirs: src+ exposed-modules:+ NLP.Tokenize+ NLP.Tokenize.Text+ NLP.Tokenize.String -Build-type: Simple+ -- Packages needed in order to build this package.+ build-depends:+ base >= 4 && < 5+ , split >= 0.1+ , text --- Extra files to be distributed with the package, such as examples or--- a README.--- Extra-source-files: + default-language: Haskell2010 --- Constraint on the version of Cabal needed to build this package.-Cabal-version: >=1.2+benchmark bench+ type: exitcode-stdio-1.0+ main-is: Bench.hs+ hs-source-dirs: tests/src + build-depends:+ tokenize+ , base+ , bytestring+ , criterion >= 1+ , deepseq+ , filepath >= 1.3.0.1+ , split >= 0.1.2.3+ , text >= 0.11.3.0 -Library- -- Modules exported by the library.- Exposed-modules: NLP.Tokenize- - -- Packages needed in order to build this package.- Build-depends: base >= 3 && < 5, split >= 0.1 - - + default-language: Haskell2010+ ghc-options: -Wall -main-is Bench