tokenize (empty) → 0.1.0

raw patch · 4 files changed

+173/−0 lines, 4 filesdep +basedep +splitsetup-changed

Dependencies added: base, split

Files

• LICENSE +30/−0
• NLP/Tokenize.hs +86/−0
• Setup.hs +2/−0
• tokenize.cabal +55/−0

@@ -0,0 +1,30 @@+Copyright (c)2011, Grzegorz Chrupała++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of Grzegorz Chrupała nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

@@ -0,0 +1,86 @@+module NLP.Tokenize +    ( EitherList(..)+    , Tokenizer+    , tokenize+    , run+    , whitespace+    , uris+    , punctuation+    , finalPunctuation+    , initialPunctuation+    , negatives+    )+where++import qualified Data.Char as Char+import Data.List+import Control.Monad.Instances+import Data.List.Split+import Control.Monad++-- | A Tokenizer is function which takes a list and returns a list of Eithers+--  (wrapped in a newtype). Right Strings will be passed on for processing+--  to tokenizers down+--  the pipeline. Left Strings will be passed through the pipeline unchanged.+--  Use a Left String in a tokenizer to protect certain tokens from further +--  processing (e.g. see the 'uris' tokenizer).+type Tokenizer =  String -> EitherList String String++-- | The EitherList is a newtype-wrapped list of Eithers.+newtype EitherList a b =  E { unE :: [Either a b] }++-- | Split string into words using the default tokenizer pipeline +tokenize :: String -> [String]+tokenize  = run (whitespace >=> uris >=> punctuation >=> negatives)++-- | Run a tokenizer+run :: Tokenizer -> (String -> [String])+run f = map unwrap . unE . f++-- | Detect common uris and freeze them+uris :: Tokenizer+uris x | isUri x = E [Left x]+       | True    = E [Right x]+    where isUri x = any (`isPrefixOf` x) ["http://","ftp://","mailto:"]++-- | Split off initial and final punctuation+punctuation :: Tokenizer +punctuation = finalPunctuation >=> initialPunctuation++-- | Split off word-final punctuation+finalPunctuation :: Tokenizer+finalPunctuation x = E $+    case span Char.isPunctuation . reverse $ x of+      ([],w) -> [Right . reverse $ w]+      (ps,w) -> [Right . reverse $ w, Right . reverse $ ps]++-- | Split off word-initial punctuation+initialPunctuation :: Tokenizer+initialPunctuation x = E $+    case span Char.isPunctuation$ x of+      ([],w) -> [Right w]+      (ps,w) -> [Right ps, Right w]++-- | Split words ending in "n't", and freeze "n't" +negatives :: Tokenizer+negatives x | "n't" `isSuffixOf` x = E [ Right . reverse . drop 3 . reverse $ x+                                       , Left "n't" ]+            | True                 = E [Right x]++-- | Split string on whitespace. This is just a wrapper for 'words'+whitespace :: Tokenizer+whitespace xs = E [Right w | w <- words xs ]++instance Monad (EitherList a) where+    return x = E [Right x]+    E xs >>= f = E $ concatMap (either (return . Left) (unE . f)) xs++unwrap (Left x) = x+unwrap (Right x) = x++examples = +    ["This shouldn't happen."+    ,"Some 'quoted' struff"+    ,"This is a URL: http://example.org."+    ,"How about an email@example.com"]+

@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain

@@ -0,0 +1,55 @@+-- tokenize.cabal auto-generated by cabal init. For additional+-- options, see+-- http://www.haskell.org/cabal/release/cabal-latest/doc/users-guide/authors.html#pkg-descr.+-- The name of the package.+Name:                tokenize++-- The package version. See the Haskell package versioning policy+-- (http://www.haskell.org/haskellwiki/Package_versioning_policy) for+-- standards guiding when and how versions should be incremented.+Version:             0.1.0++-- A short (one-line) description of the package.+Synopsis:            Simple tokenizer for English text.++-- A longer description of the package.+Description:         Simple tokenizer for English text.++-- The license under which the package is released.+License:             BSD3++-- The file containing the license text.+License-file:        LICENSE++-- The package author(s).+Author:              Grzegorz Chrupała++-- An email address to which users can send suggestions, bug reports,+-- and patches.+Maintainer:          gchrupala@lsv.uni-saarland.de++Homepage:            https://bitbucket.org/gchrupala/lingo/overview++-- A copyright notice.+-- Copyright:           ++Category:            Natural Language Processing++Build-type:          Simple++-- Extra files to be distributed with the package, such as examples or+-- a README.+-- Extra-source-files:  ++-- Constraint on the version of Cabal needed to build this package.+Cabal-version:       >=1.2+++Library+  -- Modules exported by the library.+ Exposed-modules:     NLP.Tokenize+  +  -- Packages needed in order to build this package.+ Build-depends: base >= 3 && < 5, split >= 0.1      +  +