chatter 0.8.0.2 → 0.9.0.0

raw patch · 11 files changed

+52/−28 lines, 11 filesdep ~cerealPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: cereal

API changes (from Hackage documentation)

Files

• appsrc/POSTrainer.hs +7/−10
• changelog.md +8/−0
• chatter.cabal +8/−8
• data/models/README +8/−8
• data/models/brown.pos.model.gz too large to diff
• data/models/conll2000.chunk.model.gz binary
• data/models/conll2000.pos.model.gz too large to diff
• data/models/wikiner.ner.model.gz too large to diff
• src/NLP/Corpora/Brown.hs +8/−0
• src/NLP/Corpora/Conll.hs +11/−0
• src/NLP/POS.hs +2/−2

@@ -12,6 +12,7 @@ import NLP.Types (POSTagger, Error) import NLP.Types.IOB +import qualified NLP.Corpora.Brown as B import qualified NLP.Corpora.Conll as C  main :: IO ()@@ -20,18 +21,14 @@   let output = last args       corpora = init args -      avgPerTagger :: POSTagger C.Tag+      avgPerTagger :: POSTagger B.Tag       avgPerTagger = Avg.mkTagger Avg.emptyPerceptron Nothing -      initTagger :: POSTagger C.Tag+      initTagger :: POSTagger B.Tag       initTagger   = UT.mkTagger Map.empty (Just avgPerTagger)+   rawCorpus <- mapM T.readFile corpora-  let eCorpora :: Either Error [[IOBChunk C.Chunk C.Tag]]-      eCorpora = parseIOB $ T.concat rawCorpus-  case eCorpora of-    Left err -> T.putStrLn err-    Right taggedCorpora -> do-     let taggedSentences = map toTaggedSentence taggedCorpora-     tagger <- train initTagger taggedSentences-     saveTagger tagger output+  let taggedSentences = B.parseTaggedSentences $ T.concat rawCorpus+  tagger <- train initTagger taggedSentences+  saveTagger tagger output 

@@ -1,5 +1,13 @@ = HEAD = += 0.9.0.0 =++ - Regenerated the model files; changes to cereal caused the old+   Average Perceptron models to throw exceptions on load, and as such,+   are no longer compatible.  The packaged models have been updated.++ - Minor updates to slack build script.+ = 0.8.0.2 =   - Bounded the version of cereal to < 0.5 because of breaking changes

@@ -1,5 +1,5 @@ name:                chatter-version:             0.8.0.2+version:             0.9.0.0 synopsis:            A library of simple NLP algorithms. description:         chatter is a collection of simple Natural Language                      Processing algorithms.@@ -77,7 +77,7 @@                      containers     >= 0.5.0.0,                      random-shuffle >= 0.0.4,                      MonadRandom    >= 0.1.2,-                     cereal         >= 0.4.0.1 && < 0.5,+                     cereal         >= 0.4.0.1 && < 0.5.3.0,                      cereal-text    >= 0.1 && < 0.2,                      fullstop       >= 0.1.3.1,                      bytestring     >= 0.10.0.0,@@ -112,7 +112,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5+                     cereal >= 0.4.0.1 && < 0.5.3.0     ghc-options:      -Wall -main-is Tagger -rtsopts @@ -126,7 +126,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5,+                     cereal >= 0.4.0.1 && < 0.5.3.0,                      containers >= 0.5.0.0     ghc-options:      -Wall -main-is POSTrainer -rtsopts@@ -141,7 +141,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5,+                     cereal >= 0.4.0.1 && < 0.5.3.0,                      containers >= 0.5.0.0     ghc-options:      -Wall -main-is ChunkTrainer -rtsopts@@ -156,7 +156,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5,+                     cereal >= 0.4.0.1 && < 0.5.3.0,                      containers >= 0.5.0.0     ghc-options:      -Wall -main-is NERTrainer -rtsopts@@ -172,7 +172,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5,+                     cereal >= 0.4.0.1 && < 0.5.3.0,                      containers >= 0.5.0.0     ghc-options:      -Wall -main-is Evaluate -rtsopts@@ -226,7 +226,7 @@                      tokenize,                      QuickCheck,                      filepath,-                     cereal >= 0.4.0.1 && < 0.5,+                     cereal >= 0.4.0.1 && < 0.5.3.0,                      quickcheck-instances,                      containers,                      tasty,

@@ -2,17 +2,16 @@ brown.pos.model.gz --------------------------------------------------------------------- -Averaged Perceptron tagger and Unambiguous Tagger trained on most of-the Brown corpus; the following files were held out for testing:+Averaged Perceptron tagger and Unambiguous Tagger trained on the Brown corpus. -  ca01 ca03 cb02 cc01 cc03 cd02 ce01 ce03 cf02 cg01 cg03 ch02 cj01 cj03-  ck02 cl01 cl03 cm02 cn01 cn03 cp02 cr01 cr03 ca02 cb01 cb03 cc02 cd01-  cd03 ce02 cf01 cf03 cg02 ch01 ch03 cj02 ck01 ck03 cl02 cm01 cm03 cn02-  cp01 cp03 cr02+Generated by: -The remainder of the Brown corpus was used to train this model.+$ time ./dist/build/trainPOS/trainPOS full_brown brown.pos.model +where full_brown is the result of concatenating all the Brown corpus files in ~/nltk_data/corpora/brown: +$ cat ~/nltk_data/corpora/brown/c??? > full_brown+ --------------------------------------------------------------------- conll2000.pos.model.gz ---------------------------------------------------------------------@@ -45,4 +44,5 @@ wikiner.ner.model.gz --------------------------------------------------------------------- -A Chunker that chunks to NER tags; trained on aij-wikiner-en-wp2+A Chunker that chunks to NER tags; trained on aij-wikiner-en-wp2 (see: http://schwa.org/projects/resources/wiki/Wikiner)+

file too large to diff

binary file changed (630879 → 593326 bytes)

file too large to diff

file too large to diff

@@ -5,6 +5,8 @@ module NLP.Corpora.Brown  ( Tag(..)  , Chunk(..)+ , parseTaggedSentences+     ) where @@ -19,6 +21,12 @@  import qualified NLP.Types.Tags as T import NLP.Types.General+import NLP.Types.Tree hiding (Chunk)+import NLP.Corpora.Parsing (readPOS)++-- | Parse a Brown corpus into TagagedSentences.+parseTaggedSentences :: Text -> [TaggedSentence Tag]+parseTaggedSentences rawCorpus = map readPOS $ T.lines rawCorpus  data Chunk = C_NP -- ^ Noun Phrase.            | C_VP -- ^ Verb Phrase.

@@ -15,6 +15,17 @@  import qualified NLP.Types.Tags as T import NLP.Types.General+import NLP.Types.Tree hiding (Chunk)+import NLP.Types.IOB++-- | Parse an IOB-formatted Conll corpus into TagagedSentences.+parseTaggedSentences :: Text -> [TaggedSentence Tag]+parseTaggedSentences rawCorpus =+  let res :: Either Error [[IOBChunk Chunk Tag]]+      res = parseIOB rawCorpus+  in case res of+       Left            err -> []+       Right taggedCorpora -> map toTaggedSentence taggedCorpora  -- | Named entity categories defined for the Conll 2003 task. data NERTag = PER

@@ -46,7 +46,7 @@   ) where -+import qualified Control.Exception           as X import           Codec.Compression.GZip      (decompress) import           Data.ByteString             (ByteString) import qualified Data.ByteString             as BS@@ -118,7 +118,7 @@ loadTagger file = do   content <- getContent file   case deserialize taggerTable content of-    Left err -> error err+    Left err -> error ("Could not load tagger from file: "++ file ++" due to error: \n" ++ err)     Right tgr -> return tgr   where     getContent :: FilePath -> IO ByteString