tokenize 0.1.1 → 0.1.2

raw patch · 2 files changed

+33/−8 lines, 2 files

Files

• NLP/Tokenize.hs +32/−7
• tokenize.cabal +1/−1

@@ -9,12 +9,14 @@     , punctuation     , finalPunctuation     , initialPunctuation+    , contractions     , negatives     ) where  import qualified Data.Char as Char import Data.List+import Data.Maybe import Control.Monad.Instances import Data.List.Split import Control.Monad@@ -39,7 +41,11 @@ run f = map unwrap . unE . f  defaultTokenizer :: Tokenizer-defaultTokenizer = whitespace >=> uris >=> punctuation >=> negatives+defaultTokenizer =     whitespace +                   >=> uris +                   >=> punctuation +                   >=> contractions +                   >=> negatives   -- | Detect common uris and freeze them uris :: Tokenizer@@ -53,25 +59,40 @@  -- | Split off word-final punctuation finalPunctuation :: Tokenizer-finalPunctuation x = E $+finalPunctuation x = E . filter (not . null . unwrap) $     case span Char.isPunctuation . reverse $ x of       ([],w) -> [Right . reverse $ w]       (ps,w) -> [Right . reverse $ w, Right . reverse $ ps]  -- | Split off word-initial punctuation initialPunctuation :: Tokenizer-initialPunctuation x = E $+initialPunctuation x = E . filter (not . null . unwrap) $     case span Char.isPunctuation$ x of       ([],w) -> [Right w]       (ps,w) -> [Right ps, Right w] --- | Split words ending in "n't", and freeze "n't" +-- | Split words ending in n't, and freeze n't  negatives :: Tokenizer negatives x | "n't" `isSuffixOf` x = E [ Right . reverse . drop 3 . reverse $ x                                        , Left "n't" ]             | True                 = E [Right x] --- | Split string on whitespace. This is just a wrapper for 'words'+-- | Split common contractions off and freeze them.+-- | Currently deals with: 's, 'd, 've, 'll+contractions :: Tokenizer+contractions x = case catMaybes . map (splitSuffix x) $ cts of+                   [] -> return x+                   ((w,s):_) -> E [ Right w,Left s]+    where cts = ["'s","'d","'ve","'ll"]+          splitSuffix w sfx = +              let w' = reverse w+                  len = length sfx+              in if sfx `isSuffixOf` w +                 then Just (take (length w - len) w, reverse . take len $ w')+                 else Nothing+++-- | Split string on whitespace. This is just a wrapper for Data.List.words whitespace :: Tokenizer whitespace xs = E [Right w | w <- words xs ] @@ -84,7 +105,11 @@  examples =      ["This shouldn't happen."-    ,"Some 'quoted' struff"+    ,"Some 'quoted' stuff"     ,"This is a URL: http://example.org."-    ,"How about an email@example.com"]+    ,"How about an email@example.com"+    ,"ReferenceError #1065 broke my debugger!"+    ,"I would've gone."+    ,"They've been there."+    ] 

@@ -7,7 +7,7 @@ -- The package version. See the Haskell package versioning policy -- (http://www.haskell.org/haskellwiki/Package_versioning_policy) for -- standards guiding when and how versions should be incremented.-Version:             0.1.1+Version:             0.1.2  -- A short (one-line) description of the package. Synopsis:            Simple tokenizer for English text.