cndict 0.4.8 → 0.5.0

raw patch · 3 files changed

+94/−44 lines, 3 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• cndict.cabal +1/−1
• src/Data/Chinese/CCDict.hs +4/−2
• src/Data/Chinese/Frequency.hs +89/−41

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                cndict-version:             0.4.8+version:             0.5.0 synopsis:            Chinese/Mandarin <-> English dictionary, Chinese lexer. -- description: license:             PublicDomain

@@ -32,8 +32,10 @@ import Data.Tree  import           Data.Chinese.Pinyin-import           Data.Chinese.Frequency+import qualified Data.Chinese.Frequency as Frequency +import Data.Chinese.Frequency hiding (lookup)+ -------------------------------------------------- -- Dictionary @@ -283,7 +285,7 @@     geoMean n = product $ map fromIntegral n     -- assocs = [ (node, geoMean (filter (/=0) (nodeSum node)))     --          | node <- forest ]-    wordCount word = maybe 1 subtlexWCount (M.lookup word subtlex)+    wordCount word = maybe 1 subtlexWCount (Frequency.lookup word subtlex)     entryCount (KnownWord entry) = wordCount (entrySimplified entry)     entryCount UnknownWord{} = 1     nodeSum (Node entries _) = map entryCount entries

@@ -4,64 +4,104 @@   ( SubtlexMap   , SubtlexEntry(..)   , subtlex+  , Data.Chinese.Frequency.lookup   ) where -import           Control.Applicative+-- import           Control.Applicative import qualified Data.ByteString.Lazy as L-import           Data.Csv             as Csv+import qualified Data.ByteString      as B+import qualified Data.ByteString.Char8 as B8+-- import           Data.Csv             as Csv import           Data.FileEmbed import           Data.Map             (Map) import qualified Data.Map.Strict            as M import           Data.Text            (Text)+import           Data.Text.Encoding import qualified Data.Text            as T-import           Data.Vector          (Vector)-import qualified Data.Vector          as V+-- import           Data.Vector          (Vector)+-- import qualified Data.Vector          as V  -- import           Data.Chinese.Pinyin -type SubtlexMap = Map Text SubtlexEntry+type SubtlexMap = Map B.ByteString RawEntry +data RawEntry = RawEntry+  { rawEntryIndex    :: {-# UNPACK #-} !Int+  , rawEntryWCount   :: {-# UNPACK #-} !Int+  , rawEntryWMillion :: {-# UNPACK #-} !Double+  }+ data SubtlexEntry = SubtlexEntry-  { subtlexIndex    :: {-# UNPACK #-} !Int-  , subtlexWord     :: {-# UNPACK #-} !T.Text-  -- , subtlexPinyin   :: [T.Text]-  , subtlexWCount   :: {-# UNPACK #-} !Int-  , subtlexWMillion :: {-# UNPACK #-} !Double-  -- , subtlexEnglish  :: T.Text+  { subtlexIndex    :: !Int+  , subtlexWord     :: !T.Text+  , subtlexWCount   :: !Int+  , subtlexWMillion :: !Double   } deriving ( Show ) -instance FromRecord SubtlexEntry where-  parseRecord rec = SubtlexEntry-    <$> pure 0-    <*> fmap T.copy (index rec 0)-    -- <*> fmap (map toToneMarks . T.splitOn "/") (index rec 2)-    <*> index rec 4-    <*> index rec 5-    -- <*> index rec 14+toEntry :: Int -> B.ByteString -> RawEntry+toEntry idx row = RawEntry+    { rawEntryIndex    = idx+    , rawEntryWCount   = asInt (chunks!!4)+    , rawEntryWMillion = read (B8.unpack $ chunks!!5) }+  where+    chunks = B.split 9 row+    asInt str =+      case B8.readInt str of+        Nothing        -> -1+        Just (n,_rest) -> n -_loadSubtlexEntries :: FilePath -> IO (Vector SubtlexEntry)-_loadSubtlexEntries path = do-  inp <- L.readFile path-  case Csv.decodeWith (Csv.DecodeOptions 9) HasHeader inp of-    Left msg   -> error msg-    Right rows -> return rows+lookup :: Text -> SubtlexMap -> Maybe SubtlexEntry+lookup key m = do+    RawEntry n wcount wmillion <- M.lookup (encodeUtf8 key) m+    return SubtlexEntry+      { subtlexIndex = n+      , subtlexWord  = key+      , subtlexWCount = wcount+      , subtlexWMillion = wmillion }+-- instance FromRecord SubtlexEntry where+--   parseRecord rec = SubtlexEntry+--     <$> pure 0+--     <*> fmap T.copy (index rec 0)+--     -- <*> fmap (map toToneMarks . T.splitOn "/") (index rec 2)+--     <*> index rec 4+--     <*> index rec 5+--     -- <*> index rec 14 -mkSubtlexMap :: Vector SubtlexEntry -> SubtlexMap+-- _loadSubtlexEntries :: FilePath -> IO (Vector SubtlexEntry)+-- _loadSubtlexEntries path = do+--   inp <- L.readFile path+--   case Csv.decodeWith (Csv.DecodeOptions 9) HasHeader inp of+--     Left msg   -> error msg+--     Right rows -> return rows++-- mkSubtlexMap :: Vector SubtlexEntry -> SubtlexMap+-- mkSubtlexMap rows = M.fromListWith join+--   [ (subtlexWord row, row{subtlexIndex = n})+--   | (n,row) <- zip [0..] (V.toList rows)+--   -- , subtlexEnglish row /= "#"+--   ]+--   where+--     join e1 e2 = SubtlexEntry+--       { subtlexIndex = min (subtlexIndex e1) (subtlexIndex e2)+--       , subtlexWord  = subtlexWord e1+--       -- , subtlexPinyin = subtlexPinyin e1+--       , subtlexWCount = subtlexWCount e1 + subtlexWCount e2+--       , subtlexWMillion = subtlexWMillion e1 + subtlexWMillion e2+--       -- , subtlexEnglish = subtlexEnglish e1+--       }++mkSubtlexMap :: [B.ByteString] -> SubtlexMap mkSubtlexMap rows = M.fromListWith join-  [ (subtlexWord row, row{subtlexIndex = n})-  | (n,row) <- zip [0..] (V.toList rows)+  [ (word, toEntry n row)+  | (n,row) <- zip [0..] rows+  , let chunks = B.split 9 row+        word = head chunks+  , not (null chunks)   -- , subtlexEnglish row /= "#"   ]   where-    join e1 e2 = SubtlexEntry-      { subtlexIndex = min (subtlexIndex e1) (subtlexIndex e2)-      , subtlexWord  = subtlexWord e1-      -- , subtlexPinyin = subtlexPinyin e1-      , subtlexWCount = subtlexWCount e1 + subtlexWCount e2-      , subtlexWMillion = subtlexWMillion e1 + subtlexWMillion e2-      -- , subtlexEnglish = subtlexEnglish e1-      }-+    join (RawEntry n1 c1 m1) (RawEntry n2 c2 m2) =+      RawEntry (min n1 n2) (c1+c2) (m1+m2)   @@ -71,8 +111,16 @@  subtlex :: SubtlexMap subtlex = mkSubtlexMap $-  case Csv.decodeWith (Csv.DecodeOptions 9) HasHeader inp of-    Left msg -> error msg-    Right rows -> rows+    rows   where-    inp = L.fromStrict $(embedFile "data/SUBTLEX_CH_131210_CE.utf8")+    utfData = $(embedFile "data/SUBTLEX_CH_131210_CE.utf8")+    rows = drop 1 (B.split 0xa utfData)++-- subtlex :: SubtlexMap+-- subtlex = mkSubtlexMap $+--   case Csv.decodeWith (Csv.DecodeOptions 9) HasHeader inp of+--     Left msg -> error msg+--     Right rows -> rows+--   where+--     inp = L.fromStrict $(embedFile "data/SUBTLEX_CH_131210_CE.utf8")+