cndict 0.4.1 → 0.4.2

raw patch · 2 files changed

+29/−25 lines, 2 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• cndict.cabal +1/−1
• src/Data/Chinese/CCDict.hs +28/−24

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                cndict-version:             0.4.1+version:             0.4.2 synopsis:            Chinese/Mandarin <-> English dictionary, Chinese lexer. -- description: license:             PublicDomain

@@ -14,7 +14,6 @@   , tokenizer   ) where -import           Control.Monad       (guard) import           Data.Char import           Data.FileEmbed import           Data.List           (foldl', nub)@@ -108,24 +107,27 @@ -- [[点,出发],[点出,发]] -- 出发点 -- [[出发]]+-- 穿上外套+-- This can be broken up in two ways: 穿 上外 套 and 穿上 外套+-- We want the second, more greedy tokenization. lookupNonDet :: Text -> CCDict -> Maybe [[Entry]]-lookupNonDet key trie = do-  entries <- lookupMatches key trie-  let longest = maximum (map (T.length . entrySimplified) entries)-  if longest == 1-    then return [entries]-    else return $ do-      entry <- entries-      let len = T.length (entrySimplified entry)--      case lookupMatches (T.drop len key) trie of-        Just rest | len < longest -> do-          next <- rest-          guard (T.length (entrySimplified next) + len > longest)-          return [entry, next]-        _nothing -> return [entry]--+lookupNonDet key trie = toMaybe $ beGreedy $+    step (lookupMatches key trie) $ \entry1 -> do+    let len = T.length (entrySimplified entry1)+    entry2 <- toList $ lookupMatches (T.drop len key) trie+    return [entry1, entry2]+  where+    step Nothing fn = []+    step (Just [x]) fn = return [x]+    step (Just lst) fn = lst >>= fn+    beGreedy lst =+      let len = sum . map (T.length . entrySimplified)+          longest = maximum (map len lst)+      in filter (\x -> len x == longest) lst+    toList Nothing = []+    toList (Just lst) = lst+    toMaybe [] = Nothing+    toMaybe lst = Just lst -------------------------------------------------- -- Tokenizer @@ -180,18 +182,20 @@       	, ("后生活", ["后","生活"])       	, ("不愿意", ["不","愿意"])       	, ("点出发", ["点","出发"])-        , ("不会跳舞", ["不会","跳舞"]) ]+        , ("老婆婆", ["老","婆婆"])+        , ("不会跳舞", ["不会","跳舞"])+        , ("穿上外套", ["穿上","外套"]) ]  flat :: [Token] -> [Text] flat tokens = [ entrySimplified entry | KnownWord entry <- tokens ]  type NonDet = Tree [Token] --- ppNonDet :: [NonDet] -> String--- ppNonDet forest = drawForest (map (fmap (unwords . map ppToken)) forest)---   where---     ppToken (KnownWord entry) = T.unpack (entryChinese entry)---     ppToken (UnknownWord txt) = T.unpack txt+_ppNonDet :: [NonDet] -> String+_ppNonDet forest = drawForest (map (fmap (unwords . map ppToken)) forest)+  where+    ppToken (KnownWord entry) = T.unpack (entrySimplified entry)+    ppToken (UnknownWord txt) = T.unpack txt  -- compactNonDet :: NonDet -> NonDet -- compactNonDet (Node a [Node b rest]) =