cndict 0.4.7 → 0.4.8

raw patch · 2 files changed

+6/−2 lines, 2 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• cndict.cabal +1/−1
• src/Data/Chinese/CCDict.hs +5/−1

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                cndict-version:             0.4.7+version:             0.4.8 synopsis:            Chinese/Mandarin <-> English dictionary, Chinese lexer. -- description: license:             PublicDomain

@@ -291,7 +291,11 @@  -- Enhanced tokenizer, mixed non-determistic and greedy algorithm tokenizer' :: CCDict -> Text -> [Token]-tokenizer' trie inp = collapseNonDet (tokenizerNondet trie inp)+tokenizer' trie inp = compress $ collapseNonDet (tokenizerNondet trie inp)+  where+    compress [] = []+    compress (UnknownWord a:UnknownWord b:xs) = compress (UnknownWord (a `T.append` b):xs)+    compress (x:xs) = x:compress xs  tokenizerNondet :: CCDict -> Text -> [NonDet] tokenizerNondet trie inp = map _compactNonDet $ go inp