diff --git a/chatter.cabal b/chatter.cabal --- a/chatter.cabal +++ b/chatter.cabal @@ -1,5 +1,5 @@ name: chatter -version: 0.0.0.2 +version: 0.0.0.3 synopsis: A library of simple NLP algorithms. description: chatter is a collection of simple Natural Language Processing algorithms. @@ -15,7 +15,7 @@ are available in the 'NLP.Similarity.VectorSim' module. homepage: http://github.com/creswick/chatter Bug-Reports: http://github.com/creswick/chatter/issues -category: Tools +category: Natural language processing license: BSD3 License-file: LICENSE author: Rogan Creswick @@ -44,21 +44,25 @@ NLP.Types NLP.Tokenize NLP.Corpora.Parsing + NLP.Corpora.Email NLP.Similarity.VectorSim Data.DefaultMap Build-depends: base >= 4 && <= 6, - text, - containers, - safe, - random-shuffle, - MonadRandom, - cereal, - fullstop, - split, - bytestring, - zlib, - filepath + text >= 0.11.3.0, + containers >= 0.5.0.0, + safe >= 0.3.3, + random-shuffle >= 0.0.4, + MonadRandom >= 0.1.2, + cereal >= 0.4.0.1, + fullstop >= 0.1.3.1, + split >= 0.1.2.3, + bytestring >= 0.10.0.0, + directory, + mbox, + zlib >= 0.5.4.1, + filepath >= 1.3.0.1, + ghc-prim ghc-options: -Wall @@ -69,11 +73,11 @@ hs-source-dirs: appsrc Build-depends: chatter, - filepath, - text, - base >= 4 && <= 6, - bytestring, - cereal + filepath >= 1.3.0.1, + text >= 0.11.3.0, + base >= 4 && <= 6, + bytestring >= 0.10.0.0, + cereal >= 0.4.0.1 ghc-options: -Wall -main-is Tagger -rtsopts @@ -83,12 +87,12 @@ hs-source-dirs: appsrc Build-depends: chatter, - filepath, - text, - base >= 4 && <= 6, - bytestring, - cereal, - containers + filepath >= 1.3.0.1, + text >= 0.11.3.0, + base >= 4 && <= 6, + bytestring >= 0.10.0.0, + cereal >= 0.4.0.1, + containers >= 0.5.0.0 ghc-options: -Wall -main-is Trainer -rtsopts @@ -98,12 +102,12 @@ hs-source-dirs: appsrc Build-depends: chatter, - filepath, - text, - base >= 4 && <= 6, - bytestring, - cereal, - containers + filepath >= 1.3.0.1, + text >= 0.11.3.0, + base >= 4 && <= 6, + bytestring >= 0.10.0.0, + cereal >= 0.4.0.1, + containers >= 0.5.0.0 ghc-options: -Wall -main-is Evaluate -rtsopts @@ -117,10 +121,12 @@ Build-depends: chatter, criterion, - filepath, - text, + filepath >= 1.3.0.1, + text >= 0.11.3.0, base >= 4 && <= 6, - split + tokenize, + deepseq, + split >= 0.1.2.3 ghc-options: -Wall -main-is Bench diff --git a/src/NLP/Corpora/Email.hs b/src/NLP/Corpora/Email.hs new file mode 100644 --- /dev/null +++ b/src/NLP/Corpora/Email.hs @@ -0,0 +1,44 @@ +{-# LANGUAGE OverloadedStrings #-} +-- | Utilities for reading mailman-style email archives. +module NLP.Corpora.Email where + +import qualified Data.ByteString as BS +import Data.List (isSuffixOf) +import Data.List.Split (splitWhen) +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.IO as T +import qualified Data.Text.Encoding as TE + +import Data.MBox + +import System.Directory (getDirectoryContents) +import System.FilePath (()) + +import NLP.Tokenize (tokenize) + +-- | Path to the directory containing all the PLUG archives. +plugDataPath :: FilePath +plugDataPath = "./data/corpora/PLUG/" + +plugArchiveText :: IO [Text] +plugArchiveText = do + archive <- fullPlugArchive + return $ map body archive + +plugArchiveTokens :: IO [[Text]] +plugArchiveTokens = do + archive <- fullPlugArchive + return $ map (tokenize . body) archive + +fullPlugArchive :: IO [Message] +fullPlugArchive = do + files <- getDirectoryContents plugDataPath + let archiveFiles = filter (".txt" `isSuffixOf`) files + contents <- mapM (\f->readF (plugDataPath f)) archiveFiles + return $ concatMap parseMBox contents + +readF :: FilePath -> IO Text +readF file = do + bs <- BS.readFile file + return $ TE.decodeLatin1 bs diff --git a/src/NLP/Tokenize.hs b/src/NLP/Tokenize.hs --- a/src/NLP/Tokenize.hs +++ b/src/NLP/Tokenize.hs @@ -48,7 +48,7 @@ defaultTokenizer :: Tokenizer defaultTokenizer = whitespace >=> uris - >=> hyphens +-- >=> hyphens >=> punctuation >=> contractions >=> negatives diff --git a/tests/src/Bench.hs b/tests/src/Bench.hs --- a/tests/src/Bench.hs +++ b/tests/src/Bench.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE PackageImports #-} {-# LANGUAGE OverloadedStrings #-} module Bench where @@ -5,13 +6,17 @@ import qualified Data.Text as T import qualified Data.Text.IO as T +import Control.DeepSeq import Criterion.Main import Criterion.Config (defaultConfig, Config(..), ljust) import Criterion (bench, bgroup, Benchmark) +import qualified "tokenize" NLP.Tokenize as StrTok +import qualified "chatter" NLP.Tokenize as TextTok import NLP.POS (tagText) import NLP.POS.AvgPerceptronTagger (trainNew, mkTagger) import Corpora +import NLP.Corpora.Email import qualified NLP.Similarity.VectorSimBench as VS @@ -27,10 +32,23 @@ muc3_1 <- VS.muc3_01 muc3_2 <- VS.muc3_02 muc3_3 <- VS.muc3_03 - defaultMainWith myConfig (return ()) - [ bgroup "POS Tagging" [] -- postagBench - , bgroup "Similarity" $ VS.benchmarks (muc3_1++muc3_2) muc3_3 + pTxt <- plugArchiveText + let len = length pTxt + plugTxt = take (len `div` 4) pTxt + plugStr = map T.unpack plugTxt + deepseq plugStr $ defaultMainWith myConfig (return ()) + [ bgroup "tokenizing" + [ bench "String Tokenizer" $ nf (map StrTok.tokenize) plugStr + , bench "Text Tokenizer" $ nf (map TextTok.tokenize) plugTxt + , bench "String Tokenizer" $ nf (map strTokenizer) plugTxt + ] + + -- bgroup "POS Tagging" [] -- postagBench + -- , bgroup "Similarity" $ VS.benchmarks (muc3_1++muc3_2) muc3_3 ] + +strTokenizer :: Text -> [Text] +strTokenizer txt = map T.pack (StrTok.tokenize $ T.unpack txt) -- posTagging :: IO [Benchmark] -- posTagging = do diff --git a/tests/src/NLP/Similarity/VectorSimBench.hs b/tests/src/NLP/Similarity/VectorSimBench.hs --- a/tests/src/NLP/Similarity/VectorSimBench.hs +++ b/tests/src/NLP/Similarity/VectorSimBench.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE PackageImports #-} {-# LANGUAGE OverloadedStrings #-} module NLP.Similarity.VectorSimBench where @@ -7,7 +8,7 @@ import qualified Data.Text.IO as T import Criterion (bench, whnf, Benchmark) -import NLP.Tokenize (tokenize) +import "chatter" NLP.Tokenize (tokenize) import NLP.Similarity.VectorSim import NLP.Types (mkCorpus, Corpus)