chatter 0.9.0.0 → 0.9.1.0

raw patch · 3 files changed

+23/−13 lines, 3 filesdep ~cerealPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: cereal

API changes (from Hackage documentation)

Files

• changelog.md +6/−0
• chatter.cabal +8/−8
• src/NLP/Similarity/VectorSim.hs +9/−5

@@ -1,5 +1,11 @@ = HEAD = += 0.9.1.0 =++ - Updated dependency versions (cereal, specifically) to account for ghc-8.0.+ - Switched to using HashSet in VectorSim; gaining ~25% performance improvement (Thanks to @dgaw!)+ - Dropped support for ghc-7.6.+ = 0.9.0.0 =   - Regenerated the model files; changes to cereal caused the old

@@ -1,5 +1,5 @@ name:                chatter-version:             0.9.0.0+version:             0.9.1.0 synopsis:            A library of simple NLP algorithms. description:         chatter is a collection of simple Natural Language                      Processing algorithms.@@ -77,7 +77,7 @@                      containers     >= 0.5.0.0,                      random-shuffle >= 0.0.4,                      MonadRandom    >= 0.1.2,-                     cereal         >= 0.4.0.1 && < 0.5.3.0,+                     cereal         >= 0.4.0.1 && < 0.5.4.0,                      cereal-text    >= 0.1 && < 0.2,                      fullstop       >= 0.1.3.1,                      bytestring     >= 0.10.0.0,@@ -112,7 +112,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5.3.0+                     cereal >= 0.4.0.1 && < 0.5.4.0     ghc-options:      -Wall -main-is Tagger -rtsopts @@ -126,7 +126,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5.3.0,+                     cereal >= 0.4.0.1 && < 0.5.4.0,                      containers >= 0.5.0.0     ghc-options:      -Wall -main-is POSTrainer -rtsopts@@ -141,7 +141,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5.3.0,+                     cereal >= 0.4.0.1 && < 0.5.4.0,                      containers >= 0.5.0.0     ghc-options:      -Wall -main-is ChunkTrainer -rtsopts@@ -156,7 +156,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5.3.0,+                     cereal >= 0.4.0.1 && < 0.5.4.0,                      containers >= 0.5.0.0     ghc-options:      -Wall -main-is NERTrainer -rtsopts@@ -172,7 +172,7 @@                      text >= 0.11.3.0,                      base >= 4.6 && <= 6,                      bytestring >= 0.10.0.0,-                     cereal >= 0.4.0.1 && < 0.5.3.0,+                     cereal >= 0.4.0.1 && < 0.5.4.0,                      containers >= 0.5.0.0     ghc-options:      -Wall -main-is Evaluate -rtsopts@@ -226,7 +226,7 @@                      tokenize,                      QuickCheck,                      filepath,-                     cereal >= 0.4.0.1 && < 0.5.3.0,+                     cereal >= 0.4.0.1 && < 0.5.4.0,                      quickcheck-instances,                      containers,                      tasty,

@@ -8,7 +8,7 @@ import Test.QuickCheck (Arbitrary(..)) import qualified Data.HashMap.Strict as HM import qualified Data.DefaultMap as DM-import qualified Data.Set as Set+import qualified Data.HashSet as HSet import Data.Text (Text) import qualified Data.Text as T import Data.List (elemIndices)@@ -72,7 +72,7 @@ -- implementation.  If you need to run similarity against any single -- document more than once, then you should create `TermVector`s for -- each of your documents and use `tvSim` instead of `similarity`.--- +-- -- The return value will be in the range [0, 1]. -- -- There *must* be at least one document in the corpus.@@ -153,12 +153,16 @@ -- | find the dot product of two vectors. dotProd :: TermVector -> TermVector -> Double dotProd xs ys = let-  terms = Set.fromList (keys xs) `Set.union` Set.fromList (keys ys)-  in Set.foldl (+) 0 (Set.map (\t -> (lookup t xs) * (lookup t ys)) terms)+  -- Perhaps a bit more performance can be had if we ensure the first set+  -- in the union is the smaller one?+  -- https://hackage.haskell.org/package/unordered-containers-0.2.7.2/docs/Data-HashSet.html#v:union+  terms = vectorToSet xs `HSet.union` vectorToSet ys+  in HSet.foldl' (+) 0 (HSet.map (\t -> (lookup t xs) * (lookup t ys)) terms)+  where+    vectorToSet = HSet.fromMap . (HM.map (const ())) . DM.defMap . fromTV  keys :: TermVector -> [Text] keys tv = DM.keys $ fromTV tv  lookup :: Text -> TermVector -> Double lookup key tv = DM.lookup key $ fromTV tv-