word2vec-model (empty) → 0.1.0.0
raw patch · 9 files changed
+495/−0 lines, 9 filesdep +HUnitdep +attoparsecdep +basesetup-changed
Dependencies added: HUnit, attoparsec, base, binary, binary-ieee754, bytestring, conduit, conduit-combinators, conduit-extra, hspec, text, unordered-containers, vector, word2vec-model
Files
- ChangeLog.md +3/−0
- LICENSE +30/−0
- README.md +16/−0
- Setup.hs +2/−0
- app/Similarity.hs +25/−0
- app/WordAnalogy.hs +26/−0
- src/Data/Word2Vec/Model.hs +214/−0
- test/Spec.hs +67/−0
- word2vec-model.cabal +112/−0
+ ChangeLog.md view
@@ -0,0 +1,3 @@+# Changelog for word2vec-model++## Unreleased changes
+ LICENSE view
@@ -0,0 +1,30 @@+Copyright Filip Graliński (c) 2017++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.++ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following+ disclaimer in the documentation and/or other materials provided+ with the distribution.++ * Neither the name of Filip Graliński nor the names of other+ contributors may be used to endorse or promote products derived+ from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,16 @@+# word2vec-model++Reading word2vec binary models (generated with the original tool by Mikolov).++This simple module is only for *reading* word2vec models (it cannot be used+to *generate* a word2vec model, for this the original word2vec tools should be used).++Note that word2vec binary format is not a proper serialisation format (as it is mostly+a raw dump of C data. *Caveat emptor*, it might be risky to read a model generated+on a host with a different architecture.++Example:++ {-# LANGUAGE OverloadedStrings #-}+ model <- readWord2VecModel "binary.bin"+ let theMostSimilar = findKNearestToWord w2v 30 "polska"
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ app/Similarity.hs view
@@ -0,0 +1,25 @@+{-# LANGUAGE OverloadedStrings #-}++module Main where++import Data.Word2Vec.Model+import Data.Text as T++import qualified Data.Conduit.Binary as CB+import qualified Data.Conduit.Text as CT+import Data.Conduit+import Data.Conduit.Combinators as C++import System.Environment++main :: IO ()+main = do+ (arg:_) <- getArgs+ w2v <- readWord2VecModel arg+ runConduitRes $ stdin .| CT.decodeUtf8Lenient .| CT.lines .| C.map (getSimilarOnes w2v)+ .| CT.encode CT.utf8 .| stdout++getSimilarOnes :: Word2VecModel -> Text -> Text+getSimilarOnes w2v w = T.unlines $ Prelude.map (format w) similarOnes+ where similarOnes = findKNearestToWord w2v 30 w+ format w (w', d) = T.intercalate "\t" [w, w', (T.pack $ show d)]
+ app/WordAnalogy.hs view
@@ -0,0 +1,26 @@+{-# LANGUAGE OverloadedStrings #-}++module Main where++import Data.Word2Vec.Model+import Data.Text as T++import qualified Data.Conduit.Binary as CB+import qualified Data.Conduit.Text as CT+import Data.Conduit+import Data.Conduit.Combinators as C++import System.Environment++main :: IO ()+main = do+ (arg:_) <- getArgs+ w2v <- readWord2VecModel arg+ runConduitRes $ stdin .| CT.decodeUtf8Lenient .| CT.lines .| C.map (solveAnalogy w2v)+ .| CT.encode CT.utf8 .| stdout++solveAnalogy :: Word2VecModel -> Text -> Text+solveAnalogy w2v line = T.unlines $ Prelude.map (format a1 a2 b1) similarOnes+ where [a1, a2, b1] = T.words line+ similarOnes = solveWordAnalogy w2v 30 a1 a2 b1+ format a1 a2 b1 (w', d) = T.intercalate "\t" [a1, a2, b1, w', (T.pack $ show d)]
+ src/Data/Word2Vec/Model.hs view
@@ -0,0 +1,214 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE BangPatterns #-}++-----------------------------------------------------------------------------+-- |+-- Module : Data.Word2Vec.Model+-- Copyright : Filip Graliński 2017+-- License : BSD-style (see the LICENSE file in the distribution)+--+-- Maintainer : filipg@amu.edu.pl+-- Stability : experimental+-- Portability : ?+--+-- Reading word2vec binary models (generated with the original tool by Mikolov).+--+-- This simple module is only for /reading/ word2vec models (it cannot be used+-- to /generate/ a word2vec model, for this the original word2vec tools should be used).+--+-- Note that word2vec binary format is not a proper serialisation format (as it is mostly+-- a raw dump of C data. /Caveat emptor/, it might be risky to read a model generated+-- on a host with a different architecture.+--+-- Example:+--+-- @+-- model <- readWord2VecModel "binary.bin"+-- let theMostSimilar = findKNearestToWord w2v 30 "bar"+-- @+--+-----------------------------------------------------------------------------++module Data.Word2Vec.Model+ (+ -- * Main data structure+ Word2VecModel++ -- * Basic operations+ , readWord2VecModel++ , numberOfWords+ , numberOfDimensions++ -- * Operations on vectors+ , WVector+ , buildWVector+ , getVector+ , normalizeVector++ -- * Looking for the nearest vector+ --+ -- (Like the distance/word-analogy tools in the original word2vec.)+ , findNearestToWord+ , findKNearestToWord+ , findKNearestToVector+ , solveWordAnalogy++ -- * Helper functions+ , cosineSimilarity+ , dotProduct+ ) where++import qualified Data.HashMap.Strict as DHS+import qualified Data.Text as T+import Data.Text.Encoding+import qualified Data.Vector.Storable as V+import qualified Data.ByteString.Lazy as BL+import qualified Data.ByteString.Internal as BS+import qualified Data.Attoparsec.ByteString.Lazy as AL+import qualified Data.Attoparsec.ByteString.Char8 as AP+import Data.Binary+import Data.Binary.Get+import Data.Binary.IEEE754+import Data.List (maximumBy, foldl')+import Data.Maybe (catMaybes)++-- | Vector stored in a word2vec model with its norm (to speed up+-- calculating cosine similarities).+data WVector = WVector (V.Vector Float) -- vector itself+ Float -- norm+ deriving (Eq, Show)++-- | Word2vec Model+data Word2VecModel = Word2VecModel Int -- number of words+ Int -- number of dimensions+ !(DHS.HashMap T.Text WVector) -- word-to-vector map+ deriving (Eq, Show)++-- | Main function, reading a word2vec binary model into memory.+readWord2VecModel :: FilePath -> IO (Word2VecModel)+readWord2VecModel fileName = do+ contents <- BL.readFile fileName+ pure $ processWord2VecBinaryModel contents++-- | Get the number of words in the model.+numberOfWords :: Word2VecModel -> Int+numberOfWords (Word2VecModel n _ _) = n++-- | Get the number of dimensions.+numberOfDimensions :: Word2VecModel -> Int+numberOfDimensions (Word2VecModel _ d _) = d++-- | /(Practically) O(1)/ Get a vector for a given word.+getVector :: Word2VecModel -> T.Text -> Maybe WVector+getVector (Word2VecModel _ _ h) w = DHS.lookup w h++processWord2VecBinaryModel :: BL.ByteString -> Word2VecModel+processWord2VecBinaryModel contents =+ case AL.parse parseWord2VecBinaryModel contents of+ AL.Fail _ _ _ -> error "does not look like a word2vec binary model"+ AL.Done _ !m -> m++parseWord2VecBinaryModel :: AP.Parser Word2VecModel+parseWord2VecBinaryModel = do+ nbOfWords <- AP.decimal+ " "+ nbOfDimensions <- AP.decimal+ "\n"+ entries <- AP.many' (parseWord2VecEntry nbOfDimensions <* "\n")+ return $ Word2VecModel nbOfWords nbOfDimensions $ DHS.fromList entries++floatSize :: Int+floatSize = 4++parseWord2VecEntry :: Int -> AP.Parser (T.Text, WVector)+parseWord2VecEntry nbOfDimensions = do+ word <- AP.takeWhile1 (not . AP.isSpace)+ " "+ floatVectorRaw <- AP.take (floatSize * nbOfDimensions)+ return (decodeUtf8 word, buildWVector $ bytesToFloats floatVectorRaw)++-- see https://stackoverflow.com/questions/20912582/haskell-bytestring-to-float-array+bytesToFloats :: BS.ByteString -> V.Vector Float+bytesToFloats = V.unsafeCast . aux . BS.toForeignPtr+ where aux (fp,offset,len) = V.unsafeFromForeignPtr fp offset len++-- | Build a word2vec vector from a raw float vector+--+-- (Just calculates its norm.)+buildWVector :: V.Vector Float -> WVector+buildWVector v = WVector v (norm v)++-- | Normalise a vector with its norm+normalizeVector :: WVector -> WVector+normalizeVector (WVector v n) = buildWVector (V.map (/ n) v)++-- | Calculate cosine similarity between two word2vec vectors.+--+-- Note that it was called wrongly /cosine distance/ in the original word2vec.+cosineSimilarity :: WVector -> WVector -> Float+cosineSimilarity (WVector veca norma) (WVector vecb normb) = (dotProduct veca vecb) / (norma * normb)+ where norm v = V.sum $ V.map (\e -> e * e) v++-- | Calculate dot product between two word2vec vectors+dotProduct :: V.Vector Float -> V.Vector Float -> Float+dotProduct veca vecb = V.sum $ V.zipWith (*) veca vecb++norm :: V.Vector Float -> Float+norm = sqrt . V.sum . V.map (\e -> e * e)++-- | /O(n) where n is the number of words, assuming k is small/ Find+-- the top-k most similar words for a given word. (The queried word is+-- excluded.)+findKNearestToWord :: Word2VecModel -> Int -> T.Text -> [(T.Text, Float)]+findKNearestToWord m k w = case getVector m w of+ Just v -> filter (\(t,_) -> t /= w) $ findKNearestToVector m (k+1) v+ Nothing -> []++-- | /O(n) where n is the number of words, assuming k is small/ Find+-- the top-k most similar words for a given vector.+findKNearestToVector :: Word2VecModel -> Int -> WVector -> [(T.Text, Float)]+findKNearestToVector m@(Word2VecModel _ _ h) k v = reverse $ catMaybes+ $ foldl' step (replicate k Nothing)+ $ map (\(w',v') -> (w', cosineSimilarity v' v)) $ DHS.toList h+ where step [] v = [Just v] -- not really needed, for completeness+ step l@(lowest:theRest) v = if isBetter v lowest+ then+ insertInto v theRest+ else+ l+ insertInto v [] = [Just v]+ insertInto v l@(lowest:theRest) = if isBetter v lowest+ then+ (lowest:insertInto v theRest)+ else+ (Just v:l)+ isBetter _ Nothing = True+ isBetter (_, s1) (Just (_, s2)) = s1 > s2++-- | /O(n) where n is the number of words/ Find the word which is the+-- most similar to a given word. (The queried word is excluded.)+findNearestToWord :: Word2VecModel -> T.Text -> Maybe (T.Text, Float)+findNearestToWord m@(Word2VecModel _ _ h) w = findNearestToWord' h <$> (getVector m w)+ where findNearestToWord' h v = maximumBy (\(_,p) (_, q) -> p `compare` q)+ $ map (\(w',v') -> (w', cosineSimilarity v' v))+ $ filter (\(w',_) -> w' /= w) $ DHS.toList h++solveWordAnalogy :: Word2VecModel -> Int -> T.Text -> T.Text -> T.Text -> [(T.Text, Float)]+solveWordAnalogy m k a1 a2 b1 = case targetVector of+ Just v -> findKNearestToVector m k v+ Nothing -> []+ where targetVector = getVectorByAnalogy <$>+ (getVector m a1) <*> (getVector m a2) <*> (getVector m b1)+ getVectorByAnalogy v1 v2 u1 = vadd (normalizeVector u1) (vsubtract (normalizeVector v2)+ (normalizeVector v1))++pointWiseOperation :: (Float -> Float -> Float) -> WVector -> WVector -> WVector+pointWiseOperation fun (WVector v1 _) (WVector v2 _) =+ buildWVector $ V.zipWith fun v1 v2++vadd :: WVector -> WVector -> WVector+vadd = pointWiseOperation (+)++vsubtract :: WVector -> WVector -> WVector+vsubtract = pointWiseOperation (-)
+ test/Spec.hs view
@@ -0,0 +1,67 @@+{-# LANGUAGE OverloadedStrings #-}++import Test.Hspec++import Data.Word2Vec.Model++import qualified Data.Vector.Storable as V++import qualified Test.HUnit as HU++import Control.Monad++main :: IO ()+main = hspec $ do+ describe "basic facilities" $ do+ it "trivial dot product" $ do+ dotProduct (V.fromList [1.0, 2.0]) (V.fromList [1.0, 2.0]) `shouldBeAlmost` 5.0+ it "dot product" $ do+ dotProduct (V.fromList [0.5, 0.0, -2.0]) (V.fromList [3.0, 5.3, 1.0]) `shouldBeAlmost` (-0.5)+ it "cosine similarity (dissimilar)" $ do+ cosineSimilarity (getWVector [0.0, 4.3]) (getWVector [2.7, 0.0]) `shouldBeAlmost` 0.0+ it "cosine similarity (perfect similarity — trivial)" $ do+ cosineSimilarity (getWVector [1.0, 2.0]) (getWVector [1.0, 2.0]) `shouldBeAlmost` 1.0+ it "cosine similarity (perfect similarity)" $ do+ cosineSimilarity (getWVector [1.0, -3.7]) (getWVector [2.0, -7.4]) `shouldBeAlmost` 1.0+ describe "reading a small binary model" $ do+ it "find the most similar (cosine)" $ do+ model <- readWord2VecModel "test/sample1.bin"+ let Just (theWord, _) = findNearestToWord model "rębajło"+ numberOfWords model `shouldBe` 3997+ numberOfDimensions model `shouldBe` 20+ theWord `shouldBe` "asesor"+ it "find K most similar ones (cosine)" $ do+ model <- readWord2VecModel "test/sample1.bin"+ (map fst $ findKNearestToWord model 5 "umiał") `shouldBe` ["myśliłem",+ "myślili",+ "złości",+ "złość",+ "czas"]+ it "solve a word analogy puzzle" $ do+ model <- readWord2VecModel "test/sample1.bin"+ (map fst $ solveWordAnalogy model 5 "tadeusz" "polska" "zosia") `shouldBe` [+ "polska",+ "kochany",+ "polski",+ "dziewczyny",+ "naród" ]++getWVector :: [Float] -> WVector+getWVector = buildWVector . V.fromList++class AEq a where+ (=~) :: a -> a -> Bool++instance AEq Float where+ x =~ y = abs ( x - y ) < (1.0e-4 :: Float)++(@=~?) :: (Show a, AEq a) => a -> a -> HU.Assertion+(@=~?) actual expected = expected =~ actual HU.@? assertionMsg+ where+ assertionMsg = "Expected : " ++ show expected +++ "\nActual : " ++ show actual++shouldBeAlmost got expected = got @=~? expected++shouldReturnAlmost :: (AEq a, Show a, Eq a) => IO a -> a -> Expectation+shouldReturnAlmost action expected = action >>= (@=~? expected)
+ word2vec-model.cabal view
@@ -0,0 +1,112 @@+-- This file has been generated from package.yaml by hpack version 0.21.2.+--+-- see: https://github.com/sol/hpack+--+-- hash: 89967af707c702f80ea2fdcecf346aeae1f656bba7c4e0957f907f564751b783++name: word2vec-model+version: 0.1.0.0+synopsis: Reading word2vec binary models+description: Please see the README on Github at <https://gonito.net/gitlist/word2vec-model.git/blob/master/README.md>+homepage: https://gonito.net/gitlist/word2vec-model.git+author: Filip Graliński+maintainer: filipg@amu.edu.pl+copyright: BSD3+license: BSD3+license-file: LICENSE+build-type: Simple+cabal-version: >= 1.10++extra-source-files:+ ChangeLog.md+ README.md++source-repository head+ type: git+ location: git://gonito.net/word2vec-model.git++library+ exposed-modules:+ Data.Word2Vec.Model+ other-modules:+ Paths_word2vec_model+ hs-source-dirs:+ src+ build-depends:+ attoparsec+ , base >=4.7 && <5+ , binary+ , binary-ieee754+ , bytestring+ , text+ , unordered-containers+ , vector+ default-language: Haskell2010++executable word2vec-model-similarity+ main-is: Similarity.hs+ other-modules:+ WordAnalogy+ Paths_word2vec_model+ hs-source-dirs:+ app+ ghc-options: -threaded -rtsopts -with-rtsopts=-N+ build-depends:+ attoparsec+ , base >=4.7 && <5+ , binary+ , binary-ieee754+ , bytestring+ , conduit+ , conduit-combinators+ , conduit-extra+ , text+ , unordered-containers+ , vector+ , word2vec-model+ default-language: Haskell2010++executable word2vec-model-word-analogy+ main-is: WordAnalogy.hs+ other-modules:+ Similarity+ Paths_word2vec_model+ hs-source-dirs:+ app+ ghc-options: -threaded -rtsopts -with-rtsopts=-N+ build-depends:+ attoparsec+ , base >=4.7 && <5+ , binary+ , binary-ieee754+ , bytestring+ , conduit+ , conduit-combinators+ , conduit-extra+ , text+ , unordered-containers+ , vector+ , word2vec-model+ default-language: Haskell2010++test-suite word2vec-model-test+ type: exitcode-stdio-1.0+ main-is: Spec.hs+ other-modules:+ Paths_word2vec_model+ hs-source-dirs:+ test+ ghc-options: -threaded -rtsopts -with-rtsopts=-N+ build-depends:+ HUnit+ , attoparsec+ , base >=4.7 && <5+ , binary+ , binary-ieee754+ , bytestring+ , hspec+ , text+ , unordered-containers+ , vector+ , word2vec-model+ default-language: Haskell2010