chatter 0.3.0.0 → 0.3.0.1

raw patch · 18 files changed

+125/−112 lines, 18 filesdep ~basePVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: base

API changes (from Hackage documentation)

Files

• appsrc/Evaluate.hs +1/−1
• appsrc/Tagger.hs +1/−1
• appsrc/Trainer.hs +1/−1
• changelog.md +8/−0
• chatter.cabal +7/−7
• src/Data/DefaultMap.hs +6/−0
• src/NLP/Extraction/Examples/ParsecExamples.hs +21/−11
• src/NLP/Extraction/Parsec.hs +1/−0
• src/NLP/POS.hs +1/−1
• src/NLP/POS/LiteralTagger.hs +1/−1
• src/NLP/Types.hs +8/−5
• src/NLP/Types/General.hs +4/−1
• src/NLP/Types/Tags.hs +22/−0
• src/NLP/Types/Tree.hs +24/−41
• tests/src/Data/DefaultMapTests.hs +1/−8
• tests/src/IntegrationTests.hs +1/−10
• tests/src/NLP/Extraction/ParsecTests.hs +16/−17
• tests/src/NLP/TypesTests.hs +1/−7

@@ -8,7 +8,7 @@  import NLP.Corpora.Parsing import NLP.POS (eval, loadTagger)-import NLP.Types (POSTagger, RawTag, unTS)+import NLP.Types (POSTagger, unTS) import qualified NLP.Corpora.Brown as B  main :: IO ()

@@ -7,7 +7,7 @@ import System.Environment (getArgs)  import NLP.POS (tagText, loadTagger)-import NLP.Types (RawTag, POSTagger)+import NLP.Types (POSTagger)  import qualified NLP.Corpora.Brown as B 

@@ -10,7 +10,7 @@ import qualified NLP.POS.UnambiguousTagger as UT import NLP.POS (saveTagger, train) import NLP.Corpora.Parsing-import NLP.Types (POSTagger, RawTag)+import NLP.Types (POSTagger)  import qualified NLP.Corpora.Brown as B 

@@ -1,3 +1,11 @@+= 0.3.0.1 =++ - Bumped dependency on base to >= 4.6 (from >= 4) because of+   Text.Read.readMaybe.  This drops support for HP2012, and requires+   GHC 7.6 or newer.++ - fixed lots of warnings and added documentation.+ = 0.3.0.0 =   - Changed the Sentence and TaggedSentence data types to be actual

@@ -1,5 +1,5 @@ name:                chatter-version:             0.3.0.0+version:             0.3.0.1 synopsis:            A library of simple NLP algorithms. description:         chatter is a collection of simple Natural Language                      Processing algorithms.@@ -57,7 +57,7 @@                      NLP.Extraction.Examples.ParsecExamples                      Data.DefaultMap -   Build-depends:    base >= 4 && <= 6,+   Build-depends:    base >= 4.6 && <= 6,                      text >= 0.11.3.0,                      containers >= 0.5.0.0,                      safe >= 0.3.3,@@ -96,7 +96,7 @@    Build-depends:    chatter,                      filepath >= 1.3.0.1,                      text >= 0.11.3.0,-                     base >= 4 && <= 6,+                     base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,                      cereal >= 0.4.0.1 @@ -110,7 +110,7 @@    Build-depends:    chatter,                      filepath >= 1.3.0.1,                      text >= 0.11.3.0,-                     base >= 4 && <= 6,+                     base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,                      cereal >= 0.4.0.1,                      containers >= 0.5.0.0@@ -125,7 +125,7 @@    Build-depends:    chatter,                      filepath >= 1.3.0.1,                      text >= 0.11.3.0,-                     base >= 4 && <= 6,+                     base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,                      cereal >= 0.4.0.1,                      containers >= 0.5.0.0@@ -145,7 +145,7 @@                      criterion >= 0.8.0.1,                      filepath >= 1.3.0.1,                      text >= 0.11.3.0,-                     base       >= 4 && <= 6,+                     base       >= 4.6 && <= 6,                      deepseq,                      split >= 0.1.2.3,                      tokenize >= 0.2.0@@ -173,7 +173,7 @@                      IntegrationTests     Build-depends:    chatter,-                     base       >= 4 && <= 6,+                     base       >= 4.6 && <= 6,                      text,                      HUnit,                      parsec >= 3.1.5,

@@ -2,6 +2,7 @@ module Data.DefaultMap where +import Test.QuickCheck (Arbitrary(..)) import Control.DeepSeq (NFData) import Data.Map (Map) import qualified Data.Map as Map@@ -45,3 +46,8 @@ foldl :: (a -> b -> a) -> a -> DefaultMap k b -> a foldl fn acc m = Map.foldl fn acc (defMap m) +instance (Arbitrary k, Arbitrary v, Ord k) => Arbitrary (DefaultMap k v) where+  arbitrary = do+      def <- arbitrary+      entries <- arbitrary+      return $ fromList def entries

@@ -1,8 +1,25 @@ {-# LANGUAGE OverloadedStrings    #-}+-- | Example parsing with Parsec.+--+-- This example shows how the following grammar, from NLTK, can be+-- implemented in Chatter, using Parsec-based Information Extraction+-- patterns:+--+-- > grammar = r"""+-- >  NP: {<DT|JJ|NN.*>+}          # Chunk sequences of DT, JJ, NN+-- >  PP: {<IN><NP>}               # Chunk prepositions followed by NP+-- >  VP: {<VB.*><NP|PP|CLAUSE>+$} # Chunk verbs and their arguments+-- >  CLAUSE: {<NP><VP>}           # Chunk NP, VP+-- >  """+--+-- > > import NLP.Extraction.Examples.ParsecExamples+-- > > import Text.Parsec.Prim+-- > > tgr <- defaultTagger+-- > > map (parse findClause "interactive") $ tag tgr "Mary saw the cat sit on the mat."+-- > [Right (Chunk_CN (Chunk C_CL [Chunk_CN (Chunk C_NP [POS_CN (POS AT (Token "the")),POS_CN (POS NN (Token "cat"))]),Chunk_CN (Chunk C_VP [POS_CN (POS VB (Token "sit")),Chunk_CN (Chunk C_PP [POS_CN (POS IN (Token "on")),Chunk_CN (Chunk C_NP [POS_CN (POS AT (Token "the")),POS_CN (POS NN (Token "mat"))])])])]))]+-- module NLP.Extraction.Examples.ParsecExamples where -import qualified Data.Text as T- import Text.Parsec.Prim ( (<|>), try) import qualified Text.Parsec.Combinator as PC @@ -11,15 +28,8 @@  import qualified NLP.Corpora.Brown as B --- grammar = r"""---   NP: {<DT|JJ|NN.*>+}          # Chunk sequences of DT, JJ, NN---   PP: {<IN><NP>}               # Chunk prepositions followed by NP---   VP: {<VB.*><NP|PP|CLAUSE>+$} # Chunk verbs and their arguments---   CLAUSE: {<NP><VP>}           # Chunk NP, VP---   """--- cp = nltk.RegexpParser(grammar)--- sentence = [("Mary", Tag "NN"), ("saw", Tag "VBD"), ("the", Tag "DT"), ("cat", Tag "NN"), ("sit", Tag "VB"), ("on", Tag "IN"), ("the", Tag "DT"), ("mat", Tag "NN")]--- sentence = [TaggedSent [POS NP (Token "Mary"),POS VBD (Token "saw"),POS DT (Token "the"),POS NN (Token "cat"),POS VB (Token "sit"),POS IN (Token "on"),POS DT (Token "the"),POS NN (Token "mat"),POS Term (Token ".")]]++ -- | Find a clause in a larger collection of text. -- -- findClause skips over leading tokens, if needed, to locate a

@@ -1,6 +1,7 @@ {-# LANGUAGE OverloadedStrings, RankNTypes, FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE MultiParamTypeClasses #-}+{-# OPTIONS_GHC -fno-warn-orphans #-}  -- | This is a very simple wrapper around Parsec for writing -- Information Extraction patterns.

@@ -55,7 +55,7 @@ import           NLP.Types                   ( POSTagger(..), Sentence, POS(..)                                              , combine, Tag (..), unTS, tsLength                                              , TaggedSentence(..), stripTags-                                             , tagUNK, printTS)+                                             , printTS)  import qualified NLP.POS.AvgPerceptronTagger as Avg import qualified NLP.POS.LiteralTagger       as LT

@@ -24,7 +24,7 @@ import NLP.Tokenize.Text (Tokenizer, EitherList(..), defaultTokenizer) import NLP.Tokenize.Chatter (runTokenizer) import NLP.FullStop (segment)-import NLP.Types ( tagUNK, Sentence, TaggedSentence(..), POS(..), applyTags+import NLP.Types ( tagUNK, Sentence, TaggedSentence(..), applyTags                  , Tag, POSTagger(..), CaseSensitive(..), tokens, showTok) import Text.Regex.TDFA import Text.Regex.TDFA.Text (compile)

@@ -12,21 +12,19 @@ import Data.ByteString (ByteString) import Data.Map (Map) import qualified Data.Map as Map-import Data.Serialize (Serialize, put, get)+import Data.Serialize (Serialize) import Data.Set (Set) import qualified Data.Set as Set import Data.Text (Text)-import qualified Data.Text as T  import GHC.Generics +import Test.QuickCheck.Arbitrary (Arbitrary(..))+ import NLP.Types.General import NLP.Types.Tags import NLP.Types.Tree --- data Tag t => TaggedSentence t = TS [(Text, t)]---   deriving (Eq, Ord, Read, Show)- -- | Part of Speech tagger, with back-off tagger. -- -- A sequence of pos taggers can be assembled by using backoff@@ -87,6 +85,11 @@  instance NFData Corpus instance Serialize Corpus++instance Arbitrary Corpus where+  arbitrary = do+      docs <- arbitrary+      return $ mkCorpus docs  -- | Get the number of documents that a term occurred in. termCounts :: Corpus -> Text -> Int

@@ -3,10 +3,11 @@ module NLP.Types.General where -import Data.Serialize (Serialize, put, get)+import Data.Serialize (Serialize) import Data.Text (Text) import GHC.Generics +import Test.QuickCheck (Arbitrary(..), elements)  -- | Just a handy alias for Text type Error = Text@@ -17,3 +18,5 @@   deriving (Read, Show, Generic)  instance Serialize CaseSensitive+instance Arbitrary CaseSensitive where+  arbitrary = elements [Sensitive, Insensitive]

@@ -13,15 +13,36 @@ import Test.QuickCheck (Arbitrary(..), NonEmptyList(..)) import Test.QuickCheck.Instances () +-- | The class of things that can be regarded as 'chunks'; Chunk tags+-- are much like POS tags, but should not be confused. Generally,+-- chunks distinguish between different phrasal categories (e.g.; Noun+-- Phrases, Verb Phrases, Prepositional Phrases, etc..) class (Ord a, Eq a, Read a, Show a, Generic a, Serialize a) => ChunkTag a where   fromChunk :: a -> Text +-- | The class of POS Tags.+--+-- We use a typeclass here because POS tags just need a few things in+-- excess of equality (they also need to be serializable and human+-- readable).  Passing around all the constraints everywhere becomes a+-- hassle, and it's handy to have a uniform interface to the diferent+-- kinds of tag types.+--+-- This typeclass also allows for corpus-specific tags to be+-- distinguished; They have different semantics, so they should not be+-- merged.  That said, if you wish to create a unifying POS Tag set,+-- and mappings into that set, you can use the type system to ensure+-- that that is done correctly.+--+-- This /may/ get renamed to POSTag at some later date. class (Ord a, Eq a, Read a, Show a, Generic a, Serialize a) => Tag a where   fromTag :: a -> Text   parseTag :: Text -> a   tagUNK :: a   tagTerm :: a -> Text ++-- | A fall-back 'ChunkTag' instance, analogous to 'RawTag' newtype RawChunk = RawChunk Text   deriving (Ord, Eq, Read, Show, Generic) @@ -30,6 +51,7 @@ instance ChunkTag RawChunk where   fromChunk (RawChunk ch) = ch +-- | A fallback POS tag instance. newtype RawTag = RawTag Text   deriving (Ord, Eq, Read, Show, Generic) 

@@ -6,14 +6,11 @@ import Data.String (IsString(..)) import Data.Text (Text) import qualified Data.Text as T-import Data.List (intercalate)  import Test.QuickCheck (Arbitrary(..), listOf, elements, NonEmptyList(..)) import Test.QuickCheck.Instances ()  import NLP.Types.Tags-import NLP.Types.General-import qualified NLP.Corpora.Brown as B  -- | A sentence of tokens without tags.  Generated by the tokenizer. -- (tokenizer :: Text -> Sentence)@@ -23,9 +20,11 @@ instance Arbitrary Sentence where   arbitrary = Sent <$> arbitrary +-- | Extract the token list from a 'Sentence' tokens :: Sentence -> [Token] tokens (Sent ts) = ts +-- | Apply a parallel list of 'Tag's to a 'Sentence'. applyTags :: Tag t => Sentence -> [t] -> TaggedSentence t applyTags (Sent ts) tags = TaggedSent $ zipWith POS tags ts @@ -36,7 +35,8 @@ data ChunkedSentence chunk tag = ChunkedSent [ChunkOr chunk tag]   deriving (Read, Show, Eq) -instance (ChunkTag c, Arbitrary c, Arbitrary t, Tag t) => Arbitrary (ChunkedSentence c t) where+instance (ChunkTag c, Arbitrary c, Arbitrary t, Tag t) =>+  Arbitrary (ChunkedSentence c t) where   arbitrary = ChunkedSent <$> arbitrary  -- | A tagged sentence has POS Tags.  Generated by a part-of-speech@@ -72,6 +72,8 @@ combine :: Tag t => [TaggedSentence t] -> [TaggedSentence t] -> [TaggedSentence t] combine xs ys = zipWith combineSentences xs ys +-- | Merge 'TaggedSentence' values, preffering the tags in the first 'TaggedSentence'.+-- Delegates to 'pickTag'. combineSentences :: Tag t => TaggedSentence t -> TaggedSentence t -> TaggedSentence t combineSentences (TaggedSent xs) (TaggedSent ys) = TaggedSent $ zipWith pickTag xs ys @@ -83,10 +85,9 @@   | t1 /= tagUNK = POS t1 txt1   | otherwise    = POS t2 txt1 --- | This type seem redundant, it just exists to support the--- differences in TaggedSentence and ChunkedSentence.------ See the t3 example below to see how verbose this becomes.+-- | A data type to represent the portions of a parse tree for Chunks.+-- Note that this part of the parse tree could be a POS tag with no+-- chunk. data ChunkOr chunk tag = Chunk_CN (Chunk chunk tag)                        | POS_CN   (POS tag)                          deriving (Read, Show, Eq)@@ -97,19 +98,22 @@                 chink <- mkChink <$> arbitrary <*> arbitrary                 return [chunk, chink] +-- | Helper to create 'ChunkOr' types. mkChunk :: (ChunkTag chunk, Tag tag) => chunk -> [ChunkOr chunk tag] -> ChunkOr chunk tag mkChunk chunk children = Chunk_CN (Chunk chunk children) +-- | Helper to create 'ChunkOr' types that just hold POS tagged data. mkChink :: (ChunkTag chunk, Tag tag) => tag -> Token -> ChunkOr chunk tag mkChink tag token      = POS_CN (POS tag token) -+-- | A Chunk that strictly contains chunks or POS tags. data Chunk chunk tag = Chunk chunk [ChunkOr chunk tag]   deriving (Read, Show, Eq)  instance (ChunkTag c, Arbitrary c, Arbitrary t, Tag t) => Arbitrary (Chunk c t) where   arbitrary = Chunk <$> arbitrary <*> arbitrary +-- | A POS-tagged token. data POS tag = POS tag Token   deriving (Read, Show, Eq) @@ -124,6 +128,10 @@ printPOS :: Tag tag => POS tag -> Text printPOS (POS tag (Token txt)) = T.intercalate "" [txt, "/", tagTerm tag] ++-- | Raw tokenized text.+--+-- 'Token' has a 'IsString' instance to simplify use. data Token = Token Text   deriving (Read, Show, Eq) @@ -134,25 +142,29 @@ instance IsString Token where   fromString = Token . T.pack +-- | Extract the text of a 'Token' showTok :: Token -> Text showTok (Token txt) = txt +-- | Extract the last three characters of a 'Token', if the token is+-- long enough, otherwise returns the full token text. suffix :: Token -> Text suffix (Token str) | T.length str <= 3 = str                    | otherwise         = T.drop (T.length str - 3) str +-- | Extract the list of 'POS' tags from a 'TaggedSentence' unTS :: Tag t => TaggedSentence t -> [POS t] unTS (TaggedSent ts) = ts +-- | Calculate the length of a 'TaggedSentence' (in terms of the+-- number of tokens). tsLength :: Tag t => TaggedSentence t -> Int tsLength (TaggedSent ts) = length ts +-- | Brutally concatenate two 'TaggedSentence's tsConcat :: Tag t => [TaggedSentence t] -> TaggedSentence t tsConcat tss = TaggedSent (concatMap unTS tss) --- flattenText :: Tag t => TaggedSentence t -> Text--- flattenText (TS ts) = T.unwords $ map fst ts- -- | True if the input sentence contains the given text token.  Does -- not do partial or approximate matching, and compares details in a -- fully case-sensitive manner.@@ -175,32 +187,3 @@ -- | Compare a token with a text string. tokenMatches :: Text -> Token -> Bool tokenMatches txt (Token tok) = txt == tok------ (S (NP (NN I)) (VP (V saw) (NP (NN him))))-t1 :: Sentence-t1 = Sent-     [ Token "I"-     , Token "saw"-     , Token "him"-     , Token "."-     ]--t2 :: TaggedSentence B.Tag-t2 = TaggedSent-     [ POS B.NN    (Token "I")-     , POS B.VB    (Token "saw")-     , POS B.NN    (Token "him")-     , POS B.Term  (Token ".")-     ]--t3 :: ChunkedSentence B.Chunk B.Tag-t3 = ChunkedSent-     [ mkChunk B.C_NP [ mkChink B.NN (Token "I")  ]-     , mkChunk B.C_VP [ mkChink B.VB (Token "saw")-                      , mkChink B.NN (Token "him")-                      ]-     , mkChink B.Term (Token ".")-     ]-

@@ -1,13 +1,12 @@ module Data.DefaultMapTests where -import Test.QuickCheck (Arbitrary, arbitrary) import Test.QuickCheck.Instances () import Test.Framework.Providers.QuickCheck2 (testProperty) import Test.Framework ( testGroup, Test )  import Data.Serialize (decode, encode) -import Data.DefaultMap (DefaultMap(..), fromList)+import Data.DefaultMap (DefaultMap(..))  tests :: Test tests = testGroup "NLP.Data.DefaultMapTests"@@ -15,12 +14,6 @@           [ testProperty "DefaultMap round-trips" prop_defMapSerialize           ]         ]--instance (Arbitrary k, Arbitrary v, Ord k) => Arbitrary (DefaultMap k v) where-  arbitrary = do-      def <- arbitrary-      entries <- arbitrary-      return $ fromList def entries  prop_defMapSerialize :: DefaultMap String String -> Bool prop_defMapSerialize c = case (decode . encode) c of

@@ -4,26 +4,17 @@ where  -----------------------------------------------------------------------import Test.QuickCheck ( Arbitrary, arbitrary, (==>), Property-                       , NonEmptyList(..), listOf) import Test.QuickCheck.Instances ()-import Test.Framework.Providers.QuickCheck2 (testProperty) import Test.Framework.Providers.HUnit (testCase) import Test.Framework ( testGroup, Test, buildTest )-import Test.HUnit      ( (@=?), Assertion, assertFailure, assertEqual )+import Test.HUnit      (Assertion, assertFailure, assertEqual ) -----------------------------------------------------------------------import qualified Data.Text as T import Data.Text (Text) import qualified Data.Map as Map-import Text.Parsec.Prim (parse, (<|>), try)-import Text.Parsec.Pos-import qualified Text.Parsec.Combinator as PC ---------------------------------------------------------------------- import NLP.Types import NLP.POS import NLP.Corpora.Parsing-import NLP.Extraction.Parsec-import NLP.Extraction.Examples.ParsecExamples  import qualified NLP.POS.AvgPerceptronTagger as Avg import qualified NLP.POS.LiteralTagger       as LT

@@ -5,21 +5,17 @@ module NLP.Extraction.ParsecTests where  -----------------------------------------------------------------------import Test.QuickCheck ( Arbitrary, arbitrary, (==>), Property-                       , NonEmptyList(..), listOf)+import Test.QuickCheck ((==>), Property) import Test.QuickCheck.Instances () import Test.Framework.Providers.QuickCheck2 (testProperty) import Test.Framework ( testGroup, Test ) -----------------------------------------------------------------------import qualified Data.Text as T-import Data.Text (Text)-import Text.Parsec.Prim (parse, (<|>), try)-import Text.Parsec.Pos-import qualified Text.Parsec.Combinator as PC+import Text.Parsec.Prim (parse) ---------------------------------------------------------------------- import NLP.Types import NLP.Extraction.Parsec import NLP.Extraction.Examples.ParsecExamples+import qualified NLP.Corpora.Brown as B  import TestUtils @@ -30,15 +26,18 @@         , testProperty "followedBy" prop_followedBy         , testGroup "Noun Phrase extractor" $             map (genTest parseNounPhrase)-             [ ("Just NN", TaggedSent [(POS (RawTag "NN") "Dog")]-                         , Just (POS (RawTag "n-phr") "Dog"))-             , ("DT NN", TaggedSent [(POS (RawTag "DT") "The"), (POS (RawTag "NN") "dog")]-                       , Just (POS (RawTag "n-phr") "The dog"))-             , ("NN NN", TaggedSent [(POS (RawTag "NN") "Sunday"), (POS (RawTag "NN") "night")]-                       , Just (POS (RawTag "n-phr") "Sunday night"))-             , ("JJ NN", TaggedSent [(POS (RawTag "JJ") "beautiful"), (POS (RawTag "NN") "game")]-                       , Just (POS (RawTag "n-phr") "beautiful game"))-             , ("None - VB", TaggedSent [(POS (RawTag "VB") "jump")]+             [ ("Just NN", TaggedSent [POS B.NN "Dog"]+                         , Just (mkChunk B.C_NP [mkChink B.NN "Dog"]))+             , ("DT NN", TaggedSent [POS B.DT "The", POS B.NN "dog"]+                       , Just (mkChunk B.C_NP [ mkChink B.DT "The"+                                              , mkChink B.NN "dog"]))+             , ("NN NN", TaggedSent [POS B.NN "Sunday", POS B.NN "night"]+                       , Just (mkChunk B.C_NP [ mkChink B.NN "Sunday"+                                              , mkChink B.NN "night"]))+             , ("JJ NN", TaggedSent [POS B.JJ "beautiful", POS B.NN "game"]+                       , Just (mkChunk B.C_NP [ mkChink B.JJ "beautiful"+                                              , mkChink B.NN"game"]))+             , ("None - VB", TaggedSent [POS B.VB "jump"]                            , Nothing)              ]         ]@@ -64,7 +63,7 @@   in (POS theTag theToken) == actual  -parseNounPhrase :: TaggedSentence RawTag -> Maybe (POS RawTag)+parseNounPhrase :: TaggedSentence B.Tag -> Maybe (ChunkOr B.Chunk B.Tag) parseNounPhrase sent =   case parse nounPhrase "parseNounPhrase Test" sent of     Left  _ -> Nothing

@@ -1,12 +1,11 @@ module NLP.TypesTests where -import Test.QuickCheck (Arbitrary, arbitrary) import Test.QuickCheck.Instances () import Test.Framework.Providers.QuickCheck2 (testProperty) import Test.Framework ( testGroup, Test )  import Data.Serialize (decode, encode)-import NLP.Types (mkCorpus, Corpus(..))+import NLP.Types (Corpus(..))   tests :: Test@@ -15,11 +14,6 @@           [ testProperty "Corpus round-trips" prop_corpusSerialize           ]         ]--instance Arbitrary Corpus where-  arbitrary = do-      docs <- arbitrary-      return $ mkCorpus docs  prop_corpusSerialize :: Corpus -> Bool prop_corpusSerialize c = case (decode . encode) c of