cndict 0.1.0 → 0.1.1

raw patch · 4 files changed

+26/−11 lines, 4 files

Files

• cndict.cabal +1/−1
• src/Data/Chinese/CCDict.hs +16/−5
• src/Data/Chinese/Frequency.hs +7/−3
• src/Data/Chinese/Pinyin.hs +2/−2

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                cndict-version:             0.1.0+version:             0.1.1 synopsis:            Chinese/Mandarin <-> English dictionary, Chinese lexer. -- description: license:             PublicDomain

@@ -12,6 +12,7 @@   , tokenizer   ) where +import           Control.Monad       (mplus) import           Data.Char import           Data.FileEmbed import           Data.List           (foldl', nub)@@ -57,15 +58,22 @@       (x:xs) -> go xs =<< M.lookup x trie   where     go [] (CCTrieEntry es _) = es-    go (x:xs) (CCTrieEntry es m) = maybe es (go xs) (M.lookup x m)+    go (x:xs) (CCTrieEntry es m) = (go xs =<< M.lookup x m) `mplus` es  ++ -------------------------------------------------- -- Tokenizer  data Token = KnownWord Entry | UnknownWord Text   deriving ( Read, Show, Eq, Ord ) +-- Interesting case: 他的话 tokenizes to [他,的话] by both google translate and+-- MDGB. The correct tokenization is [他,的,话]. Not sure if it can be fixed without+-- adding an entry for 他的 in the dictionary.+-- FIXME: 多工作 should tokenize to [多,工作], not [多工,作].+-- TODO: Mark text inclosed in curly brackets as unknown words. -- | Break a string of simplified chinese down to a list of tokens. tokenizer :: CCDict -> Text -> [Token] tokenizer trie inp = filter isValid $ go 0 inp inp@@ -99,8 +107,8 @@ joinEntry Nothing (Just e)    = Just e joinEntry (Just e) Nothing    = Just e joinEntry (Just e1) (Just e2) = Just Entry-  { entryChinese = entryChinese e1-  , entryPinyin     = nub $ entryPinyin e1 ++ entryPinyin e2+  { entryChinese    = entryChinese e1+  , entryPinyin     = entryPinyin e1 -- nub $ entryPinyin e1 ++ entryPinyin e2   , entryDefinition = nub $ entryDefinition e1 ++ entryDefinition e2 }  unions :: [CCDict] -> CCDict@@ -121,12 +129,15 @@ parseLine line =     Just Entry     { entryChinese = chinese-    , entryPinyin     = map toToneMarks $ T.words $ T.tail $ T.init $ T.unwords (pinyin ++ [pin])-    , entryDefinition = [T.unwords english] }+    , entryPinyin     = [T.unwords $ map toToneMarks $ T.words $ T.tail $ T.init $ T.unwords (pinyin ++ [pin])]+    , entryDefinition = splitDefinition (T.unwords english) }   where     (_traditional : chinese : rest) = T.words line     (pinyin, (pin : english)) = break (\word -> T.count "]" word > 0) rest +-- /first/second/third/ -> [first, second, third]+splitDefinition :: Text -> [Text]+splitDefinition = filter (not . T.null) . T.splitOn "/"   --------------------------------------------------

@@ -22,7 +22,8 @@ type SubtlexMap = Map Text SubtlexEntry  data SubtlexEntry = SubtlexEntry-  { subtlexWord     :: T.Text+  { subtlexIndex    :: Int+  , subtlexWord     :: T.Text   , subtlexPinyin   :: [T.Text]   , subtlexWCount   :: Int   , subtlexWMillion :: Double@@ -31,7 +32,8 @@  instance FromRecord SubtlexEntry where   parseRecord rec = SubtlexEntry-    <$> index rec 0+    <$> pure 0+    <*> index rec 0     <*> fmap (map toToneMarks . T.splitOn "/") (index rec 2)     <*> index rec 4     <*> index rec 5 <*> index rec 14@@ -44,7 +46,9 @@     Right rows -> return rows  mkSubtlexMap :: Vector SubtlexEntry -> SubtlexMap-mkSubtlexMap rows = M.fromList [ (subtlexWord row, row) | row <- V.toList rows ]+mkSubtlexMap rows = M.fromList+  [ (subtlexWord row, row{subtlexIndex = n})+  | (n,row) <- zip [0..] (V.toList rows) ]   

@@ -15,7 +15,7 @@ toToneMarks = modToneNumber toTonal  fromToneMarks :: Text -> Text-fromToneMarks = undefined+fromToneMarks = error "Data.Chinese.Pinyin.fromToneMarks: undefined."  modToneNumber :: (Int -> Char -> Char) -> Text -> Text modToneNumber fn txt@@ -24,7 +24,7 @@   | Just n <- findStrIndex "ou" txt'         = modify n   | Just n <- findSecondVowel txt'           = modify n   | Just n <- T.findIndex (`elem` "aoeiu") txt' = modify n-  | otherwise = txt+  | otherwise = T.init txt   where     tone = digitToInt (T.last txt)     modify n = T.pack [ if n==i then fn tone c else c | (i,c) <- zip [0..] (T.unpack txt') ]