diff --git a/cndict.cabal b/cndict.cabal
--- a/cndict.cabal
+++ b/cndict.cabal
@@ -2,7 +2,7 @@
 -- documentation, see http://haskell.org/cabal/users-guide/
 
 name:                cndict
-version:             0.4.7
+version:             0.4.8
 synopsis:            Chinese/Mandarin <-> English dictionary, Chinese lexer.
 -- description:
 license:             PublicDomain
diff --git a/src/Data/Chinese/CCDict.hs b/src/Data/Chinese/CCDict.hs
--- a/src/Data/Chinese/CCDict.hs
+++ b/src/Data/Chinese/CCDict.hs
@@ -291,7 +291,11 @@
 
 -- Enhanced tokenizer, mixed non-determistic and greedy algorithm
 tokenizer' :: CCDict -> Text -> [Token]
-tokenizer' trie inp = collapseNonDet (tokenizerNondet trie inp)
+tokenizer' trie inp = compress $ collapseNonDet (tokenizerNondet trie inp)
+  where
+    compress [] = []
+    compress (UnknownWord a:UnknownWord b:xs) = compress (UnknownWord (a `T.append` b):xs)
+    compress (x:xs) = x:compress xs
 
 tokenizerNondet :: CCDict -> Text -> [NonDet]
 tokenizerNondet trie inp = map _compactNonDet $ go inp