chatter 0.0.0.3 → 0.0.0.4

raw patch · 7 files changed

+82/−7 lines, 7 files

Files

• chatter.cabal +3/−1
• src/Data/DefaultMap.hs +6/−0
• src/NLP/POS.hs +9/−5
• src/NLP/Types.hs +5/−1
• tests/src/Data/DefaultMapTests.hs +28/−0
• tests/src/Main.hs +4/−0
• tests/src/NLP/TypesTests.hs +27/−0

@@ -1,5 +1,5 @@ name:                chatter-version:             0.0.0.3+version:             0.0.0.4 synopsis:            A library of simple NLP algorithms. description:         chatter is a collection of simple Natural Language                      Processing algorithms.@@ -143,6 +143,8 @@                      NLP.Similarity.VectorSimTests                      NLP.POSTests                      NLP.POS.UnambiguousTaggerTests+                     NLP.TypesTests+                     Data.DefaultMapTests                      Corpora                      TestUtils 

@@ -3,6 +3,7 @@  import Data.Map (Map) import qualified Data.Map as Map+import Data.Serialize  -- | Defaulting Map; a Map that returns a default value when queried -- for a key that does not exist.@@ -37,3 +38,8 @@ -- standard `Data.Map.foldl` foldl :: (a -> b -> a) -> a -> DefaultMap k b -> a foldl fn acc m = Map.foldl fn acc (defMap m)++-- | Serialize instance+instance (Ord k, Serialize k, Serialize v) => Serialize (DefaultMap k v) where+  get   = fmap (uncurry DefMap) (getTwoOf get get)+  put m = (putTwoOf put put) (defDefault m, defMap m)

@@ -51,7 +51,7 @@  import NLP.Corpora.Parsing (readPOS) -import NLP.Types (TaggedSentence, Tag(..)+import NLP.Types (TaggedSentence, Tag(..), Sentence                  , POSTagger(..), tagUNK, stripTags)  import qualified NLP.POS.LiteralTagger as LT@@ -125,10 +125,14 @@ tag :: POSTagger -> Text -> [TaggedSentence] tag p txt = let sentences = (posSplitter p) txt                 tokens    = map (posTokenizer p) sentences-                priority  = (posTagger p) tokens-            in case posBackoff p of-                 Nothing  -> priority-                 Just tgr -> combine priority (tag tgr txt)+            in tagTokens p tokens++tagTokens :: POSTagger -> [Sentence] -> [TaggedSentence]+tagTokens p tokens = let priority = (posTagger p) tokens+                     in case posBackoff p of+                          Nothing  -> priority+                          Just tgr -> combine priority (tagTokens tgr tokens)+  -- | Combine the results of POS taggers, using the second param to -- fill in 'tagUNK' entries, where possible.

@@ -7,7 +7,7 @@ import Data.ByteString (ByteString) import Data.Map (Map) import qualified Data.Map as Map-import Data.Serialize (Serialize, put, get)+import Data.Serialize (Serialize, put, get, getTwoOf, putTwoOf) import Data.Set (Set) import qualified Data.Set as Set import Data.Text (Text)@@ -98,6 +98,10 @@                      , corpTermCounts :: Map Text Int                      -- ^ A count of the number of documents each term occurred in.                      } deriving (Read, Show, Eq, Ord)++instance Serialize Corpus where+  get   = fmap (uncurry Corpus) (getTwoOf get get)+  put c = (putTwoOf put put) (corpLength c, corpTermCounts c)  -- | Get the number of documents that a term occurred in. termCounts :: Corpus -> Text -> Int

@@ -0,0 +1,28 @@+module Data.DefaultMapTests where++import Test.QuickCheck (Arbitrary, arbitrary)+import Test.QuickCheck.Instances ()+import Test.Framework.Providers.QuickCheck2 (testProperty)+import Test.Framework ( testGroup, Test )++import Data.Serialize (decode, encode)++import Data.DefaultMap (DefaultMap(..), fromList)++tests :: Test+tests = testGroup "NLP.Data.DefaultMapTests"+        [ testGroup "Serialize / Deserialize Tests"+          [ testProperty "DefaultMap round-trips" prop_defMapSerialize+          ]+        ]++instance (Arbitrary k, Arbitrary v, Ord k) => Arbitrary (DefaultMap k v) where+  arbitrary = do+      def <- arbitrary+      entries <- arbitrary+      return $ fromList def entries++prop_defMapSerialize :: DefaultMap String String -> Bool+prop_defMapSerialize c = case (decode . encode) c of+                           Right c' -> c == c'+                           Left _ -> False

@@ -23,6 +23,8 @@ import qualified NLP.Similarity.VectorSimTests as Vec import qualified NLP.POSTests as POS import qualified NLP.POS.UnambiguousTaggerTests as UT+import qualified NLP.TypesTests as TypeTests+import qualified Data.DefaultMapTests as DefMap  import Corpora @@ -57,6 +59,8 @@         , Vec.tests         , POS.tests         , UT.tests+        , TypeTests.tests+        , DefMap.tests         ]  

@@ -0,0 +1,27 @@+module NLP.TypesTests where++import Test.QuickCheck (Arbitrary, arbitrary)+import Test.QuickCheck.Instances ()+import Test.Framework.Providers.QuickCheck2 (testProperty)+import Test.Framework ( testGroup, Test )++import Data.Serialize (decode, encode)+import NLP.Types (mkCorpus, Corpus(..))+++tests :: Test+tests = testGroup "NLP.Types"+        [ testGroup "Serialize / Deserialize Tests"+          [ testProperty "Corpus round-trips" prop_corpusSerialize+          ]+        ]++instance Arbitrary Corpus where+  arbitrary = do+      docs <- arbitrary+      return $ mkCorpus docs++prop_corpusSerialize :: Corpus -> Bool+prop_corpusSerialize c = case (decode . encode) c of+                           Right c' -> c == c'+                           Left _ -> False