chatter 0.0.0.2 → 0.0.0.3
raw patch · 5 files changed
+107/−38 lines, 5 filesdep +deepseqdep +directorydep +ghc-primdep ~MonadRandomdep ~bytestringdep ~cereal
Dependencies added: deepseq, directory, ghc-prim, mbox, tokenize
Dependency ranges changed: MonadRandom, bytestring, cereal, containers, filepath, fullstop, random-shuffle, safe, split, text, zlib
Files
- chatter.cabal +39/−33
- src/NLP/Corpora/Email.hs +44/−0
- src/NLP/Tokenize.hs +1/−1
- tests/src/Bench.hs +21/−3
- tests/src/NLP/Similarity/VectorSimBench.hs +2/−1
chatter.cabal view
@@ -1,5 +1,5 @@ name: chatter-version: 0.0.0.2+version: 0.0.0.3 synopsis: A library of simple NLP algorithms. description: chatter is a collection of simple Natural Language Processing algorithms.@@ -15,7 +15,7 @@ are available in the 'NLP.Similarity.VectorSim' module. homepage: http://github.com/creswick/chatter Bug-Reports: http://github.com/creswick/chatter/issues-category: Tools+category: Natural language processing license: BSD3 License-file: LICENSE author: Rogan Creswick@@ -44,21 +44,25 @@ NLP.Types NLP.Tokenize NLP.Corpora.Parsing+ NLP.Corpora.Email NLP.Similarity.VectorSim Data.DefaultMap Build-depends: base >= 4 && <= 6,- text,- containers,- safe,- random-shuffle,- MonadRandom,- cereal,- fullstop,- split,- bytestring,- zlib,- filepath+ text >= 0.11.3.0,+ containers >= 0.5.0.0,+ safe >= 0.3.3,+ random-shuffle >= 0.0.4,+ MonadRandom >= 0.1.2,+ cereal >= 0.4.0.1,+ fullstop >= 0.1.3.1,+ split >= 0.1.2.3,+ bytestring >= 0.10.0.0,+ directory,+ mbox,+ zlib >= 0.5.4.1,+ filepath >= 1.3.0.1,+ ghc-prim ghc-options: -Wall @@ -69,11 +73,11 @@ hs-source-dirs: appsrc Build-depends: chatter,- filepath,- text,- base >= 4 && <= 6,- bytestring,- cereal+ filepath >= 1.3.0.1,+ text >= 0.11.3.0,+ base >= 4 && <= 6,+ bytestring >= 0.10.0.0,+ cereal >= 0.4.0.1 ghc-options: -Wall -main-is Tagger -rtsopts @@ -83,12 +87,12 @@ hs-source-dirs: appsrc Build-depends: chatter,- filepath,- text,- base >= 4 && <= 6,- bytestring,- cereal,- containers+ filepath >= 1.3.0.1,+ text >= 0.11.3.0,+ base >= 4 && <= 6,+ bytestring >= 0.10.0.0,+ cereal >= 0.4.0.1,+ containers >= 0.5.0.0 ghc-options: -Wall -main-is Trainer -rtsopts @@ -98,12 +102,12 @@ hs-source-dirs: appsrc Build-depends: chatter,- filepath,- text,- base >= 4 && <= 6,- bytestring,- cereal,- containers+ filepath >= 1.3.0.1,+ text >= 0.11.3.0,+ base >= 4 && <= 6,+ bytestring >= 0.10.0.0,+ cereal >= 0.4.0.1,+ containers >= 0.5.0.0 ghc-options: -Wall -main-is Evaluate -rtsopts @@ -117,10 +121,12 @@ Build-depends: chatter, criterion,- filepath,- text,+ filepath >= 1.3.0.1,+ text >= 0.11.3.0, base >= 4 && <= 6,- split+ tokenize,+ deepseq,+ split >= 0.1.2.3 ghc-options: -Wall -main-is Bench
+ src/NLP/Corpora/Email.hs view
@@ -0,0 +1,44 @@+{-# LANGUAGE OverloadedStrings #-}+-- | Utilities for reading mailman-style email archives.+module NLP.Corpora.Email where++import qualified Data.ByteString as BS+import Data.List (isSuffixOf)+import Data.List.Split (splitWhen)+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.IO as T+import qualified Data.Text.Encoding as TE++import Data.MBox++import System.Directory (getDirectoryContents)+import System.FilePath ((</>))++import NLP.Tokenize (tokenize)++-- | Path to the directory containing all the PLUG archives.+plugDataPath :: FilePath+plugDataPath = "./data/corpora/PLUG/"++plugArchiveText :: IO [Text]+plugArchiveText = do+ archive <- fullPlugArchive+ return $ map body archive++plugArchiveTokens :: IO [[Text]]+plugArchiveTokens = do+ archive <- fullPlugArchive+ return $ map (tokenize . body) archive++fullPlugArchive :: IO [Message]+fullPlugArchive = do+ files <- getDirectoryContents plugDataPath+ let archiveFiles = filter (".txt" `isSuffixOf`) files+ contents <- mapM (\f->readF (plugDataPath </> f)) archiveFiles+ return $ concatMap parseMBox contents++readF :: FilePath -> IO Text+readF file = do+ bs <- BS.readFile file+ return $ TE.decodeLatin1 bs
src/NLP/Tokenize.hs view
@@ -48,7 +48,7 @@ defaultTokenizer :: Tokenizer defaultTokenizer = whitespace >=> uris - >=> hyphens+-- >=> hyphens >=> punctuation >=> contractions >=> negatives
tests/src/Bench.hs view
@@ -1,3 +1,4 @@+{-# LANGUAGE PackageImports #-} {-# LANGUAGE OverloadedStrings #-} module Bench where @@ -5,13 +6,17 @@ import qualified Data.Text as T import qualified Data.Text.IO as T +import Control.DeepSeq import Criterion.Main import Criterion.Config (defaultConfig, Config(..), ljust) import Criterion (bench, bgroup, Benchmark) +import qualified "tokenize" NLP.Tokenize as StrTok+import qualified "chatter" NLP.Tokenize as TextTok import NLP.POS (tagText) import NLP.POS.AvgPerceptronTagger (trainNew, mkTagger) import Corpora+import NLP.Corpora.Email import qualified NLP.Similarity.VectorSimBench as VS @@ -27,10 +32,23 @@ muc3_1 <- VS.muc3_01 muc3_2 <- VS.muc3_02 muc3_3 <- VS.muc3_03- defaultMainWith myConfig (return ())- [ bgroup "POS Tagging" [] -- postagBench- , bgroup "Similarity" $ VS.benchmarks (muc3_1++muc3_2) muc3_3+ pTxt <- plugArchiveText+ let len = length pTxt+ plugTxt = take (len `div` 4) pTxt+ plugStr = map T.unpack plugTxt+ deepseq plugStr $ defaultMainWith myConfig (return ())+ [ bgroup "tokenizing"+ [ bench "String Tokenizer" $ nf (map StrTok.tokenize) plugStr+ , bench "Text Tokenizer" $ nf (map TextTok.tokenize) plugTxt+ , bench "String Tokenizer" $ nf (map strTokenizer) plugTxt+ ]++ -- bgroup "POS Tagging" [] -- postagBench+ -- , bgroup "Similarity" $ VS.benchmarks (muc3_1++muc3_2) muc3_3 ]++strTokenizer :: Text -> [Text]+strTokenizer txt = map T.pack (StrTok.tokenize $ T.unpack txt) -- posTagging :: IO [Benchmark] -- posTagging = do
tests/src/NLP/Similarity/VectorSimBench.hs view
@@ -1,3 +1,4 @@+{-# LANGUAGE PackageImports #-} {-# LANGUAGE OverloadedStrings #-} module NLP.Similarity.VectorSimBench where @@ -7,7 +8,7 @@ import qualified Data.Text.IO as T import Criterion (bench, whnf, Benchmark) -import NLP.Tokenize (tokenize)+import "chatter" NLP.Tokenize (tokenize) import NLP.Similarity.VectorSim import NLP.Types (mkCorpus, Corpus)