Condor 0.1 → 0.2
raw patch · 13 files changed
+460/−267 lines, 13 filesdep +text
Dependencies added: text
Files
- CHANGES +10/−1
- Condor.cabal +18/−8
- src/Condor/DataTypes.hs +46/−0
- src/Condor/Index.hs +57/−43
- src/Condor/Language/English/Porter.hs +8/−5
- src/Condor/Language/English/StopWords.hs +134/−132
- src/Condor/Readers/Text.hs +29/−0
- src/Condor/Text.hs +11/−10
- src/IO.hs +39/−0
- src/Main.hs +43/−31
- test-src/Condor/IndexTest.hs +54/−29
- test-src/Condor/Language/English/PorterTest.hs +2/−1
- test-src/Condor/TextTest.hs +9/−7
CHANGES view
@@ -1,1 +1,10 @@-0.1 Initial version+Version 0.2 2014-01-08+* Changes in Index API+* Add recursive option to command line tool.+* Post list contains document id instead of document name. ++++Version 0.1 2014-01-04 +* Basic index implementation+* Basic command line tool
Condor.cabal view
@@ -1,5 +1,5 @@ name: Condor-version: 0.1+version: 0.2 cabal-version: >= 1.10 build-type: Simple author: Krzysztof Langner@@ -28,25 +28,32 @@ binary >=0.5.1 && <0.6, containers >=0.5.0 && <0.6, directory >=1.2.0 && <1.3,- filepath >=1.3.0 && <1.4+ filepath >=1.3.0 && <1.4,+ text >=1 && <1.1+ ghc-options: -Wall other-modules: + Condor.Index,+ Condor.Readers.Text, Condor.Text,- Condor.Index+ IO library hs-source-dirs: src default-language: Haskell2010 ghc-options: -Wall- exposed-modules: Condor.Index- other-modules: + exposed-modules: + Condor.DataTypes,+ Condor.Index, Condor.Language.English.Porter, Condor.Language.English.StopWords,+ Condor.Readers.Text, Condor.Text build-depends: base >= 4 && <4.7, containers >=0.5.0 && <0.6,- binary >=0.5.1 && <0.6+ binary >=0.5.1 && <0.6,+ text >=1 && <1.1 test-suite unit-tests type: detailed-0.9@@ -58,7 +65,9 @@ HUnit >=1.2.5 && <1.3, Cabal >=1.16.0 && <1.17, containers >=0.5.0 && <0.6,- binary >=0.5.1 && <0.6+ binary >=0.5.1 && <0.6,+ text >=1 && <1.1+ other-modules: Condor.Index, Condor.IndexTest,@@ -67,7 +76,8 @@ Condor.Language.English.StopWords, Condor.Text, Condor.TextTest,- Distribution.TestSuite.HUnit+ Distribution.TestSuite.HUnit,+ Condor.DataTypes hs-source-dirs: src, test-src
+ src/Condor/DataTypes.hs view
@@ -0,0 +1,46 @@+{- |+Module : Condor.DataTypes+Copyright : Copyright (C) 2014 Krzysztof Langner+License : The MIT License (MIT)++Maintainer : Krzysztof Langner <klangner@gmail.com>+Stability : alpha+Portability : portable++Common to all modules data types definitions.+-}+module Condor.DataTypes + ( DocName+ , Document(..)+ , docFromStrings+ , docName+ , docText+ ) where++import qualified Data.Text as T+++type DocName = T.Text++-- | Field consists of title and content+data Field = Field T.Text T.Text+ +-- | Document with name and contents +data Document = Document DocName [Field]+++-- | Create simple field from strings+fieldFromStrings :: String -> String -> Field+fieldFromStrings k v = Field (T.pack k) (T.pack v) ++-- | Create simple document with name and single field content.+docFromStrings :: String -> String -> Document+docFromStrings t c = Document (T.pack t) [fieldFromStrings "content" c]++-- | Get document name+docName :: Document -> T.Text+docName (Document a _) = a++-- | Get text from all fields+docText :: Document -> T.Text+docText (Document _ fs) = T.concat $ map (\(Field _ y) -> y) fs
src/Condor/Index.hs view
@@ -9,83 +9,98 @@ This module contains functions which create, update and search index. Default implementation uses algorithms for english language (stemming, stop words etc.)-But it should be possible to customize it for any language by modifying Index data type. +Functions in this module (for performance reasons) are based on unicode strings Data.Text.+ -} module Condor.Index ( DocName , Index- , DocContent , addDocument+ , addDocTerms , emptyIndex , search+ , searchTerms , termCount ) where import qualified Data.Map as Map import qualified Data.List as List+import qualified Data.Text as T+import qualified Data.Text.Encoding as E import Data.Binary import Condor.Text+import Condor.DataTypes (DocName, Document(..), docName, docText) import Condor.Language.English.StopWords (isStopWord) import Condor.Language.English.Porter (stem) -type DocName = String-type DocContent = String---- | Index parameters. Those parameters can be used to change how--- the text is processed before adding to the index-data IndexParams = IndexParams { ignore :: String -> Bool- , stemmer :: String -> String- }- --- An instance of Binary to encode and decode an IndexParams in binary-instance Binary IndexParams where- put _ = put (0 :: Word8)- get = do tag <- getWord8- case tag of- _ -> return $ IndexParams isStopWord stem +type Term = T.Text -- | Inverted index-data Index = Index { terms :: Map.Map String [String]- , params :: IndexParams+data Index = Index { terms :: Map.Map Term [Int]+ , docs :: [DocName] } -- An instance of Binary to encode and decode an IndexParams in binary instance Binary Index where- put i = do put (terms i)- put (params i)+ put i = do + put (terms i)+ put (docs i) get = do i <- get- p <- get- return $ Index i p + d <- get+ return $ Index i d +-- An instance of Binary to encode and decode T.Text+instance Binary T.Text where+ put i = do put (E.encodeUtf8 i)+ get = do i <- get+ return $ E.decodeUtf8 i + -- | Create empty index. -- This index will be configured for english language. emptyIndex :: Index-emptyIndex = Index Map.empty (IndexParams isStopWord stem)+emptyIndex = Index Map.empty [] --- | Add document to the index-addDocument :: DocName -> DocContent -> Index -> Index-addDocument d c ix = Index (foldl f (terms ix) ws) (params ix)- where ws = splitWords (params ix) c- f i t = case Map.lookup t i of - Just a -> Map.insert t (d:a) i- Nothing -> Map.insert t [d] i+-- | Add document to the index.+-- This function uses algorithms for english language to split document content+-- into index terms.+addDocument :: Document -> Index -> Index+addDocument d idx = addDocTerms (docName d) (splitTerms content) idx+ where content = docText d --- | Search term in the index-search :: Index -> DocContent -> [DocName]-search ix s = List.nub $ foldl (++) [] ys- where ys = map (searchTerm ix) ws- ws = splitWords (params ix) s+-- | Add document to the index.+-- This function should be used if document content should be splitted into terms+-- with custom algorithms.+addDocTerms :: DocName -> [Term] -> Index -> Index+addDocTerms d c ix = Index (foldl f (terms ix) c) (d:docs ix)+ where f ix' t = case Map.lookup t ix' of + Just a -> Map.insert t (index:a) ix'+ Nothing -> Map.insert t [index] ix'+ index = length (docs ix) +-- | Search terms given as single string in the index+-- This function uses algorithms for english language to split query into tokens.+search :: Index -> String -> [DocName]+search ix s = searchTerms ix (splitTerms (T.pack s))+++-- | Search terms given as array in the index.+-- This function should be used if query should be splitted into terms+-- with custom algorithms+searchTerms :: Index -> [Term] -> [DocName]+searchTerms ix s = List.nub $ foldl (++) [] ys+ where ys = map (findDocs ix) s++ -- | Search single term in the index-searchTerm :: Index -> String -> [DocName]-searchTerm ix s = case Map.lookup s (terms ix) of- Just a -> a+findDocs :: Index -> Term -> [DocName]+findDocs ix s = case Map.lookup s (terms ix) of+ Just a -> map ((reverse (docs ix))!!) a Nothing -> [] @@ -94,11 +109,10 @@ termCount ix = Map.size (terms ix) --- | Split text into tokens.+-- | Split text into terms. -- This function removes stop words and stems words-splitWords :: IndexParams -> String -> [String]-splitWords p s = map (stemmer p) (filter f t)+splitTerms :: T.Text -> [Term]+splitTerms s = map stem (filter (not . isStopWord) t) where t = tokenize s- f = \x -> not ((ignore p) x)
src/Condor/Language/English/Porter.hs view
@@ -5,7 +5,15 @@ import Control.Monad import Data.Maybe import Data.List+import qualified Data.Text as T ++-- | API stemming function+stem :: T.Text -> T.Text+stem s | T.length s < 3 = s+ | otherwise = T.pack $ allSteps (T.unpack s)++ isConsonant :: [Char] -> Int -> Bool isConsonant str i | c `elem` "aeiou" = False@@ -158,8 +166,3 @@ allSteps :: [Char] -> [Char] allSteps = step5 . step4 . step3 . step2 . step1--stem :: [Char] -> [Char]-stem s | length s < 3 = s- | otherwise = allSteps s-
src/Condor/Language/English/StopWords.hs view
@@ -12,139 +12,141 @@ -} module Condor.Language.English.StopWords (isStopWord) where -import Data.Set +import qualified Data.Set as S +import qualified Data.Text as T -- | Predicate to check if given words is a stop word-isStopWord :: String -> Bool-isStopWord w = member w stopWords+isStopWord :: T.Text -> Bool+isStopWord w = S.member w stopWords -stopWords :: Set String-stopWords = fromList [ "i"- , "me"- , "my"- , "myself"- , "we"- , "our"- , "ours"- , "ourselves"- , "you"- , "your"- , "yours"- , "yourself"- , "yourselves"- , "he"- , "him"- , "his"- , "himself"- , "she"- , "her"- , "hers"- , "herself"- , "it"- , "its"- , "itself"- , "they"- , "them"- , "their"- , "theirs"- , "themselves"- , "what"- , "which"- , "who"- , "whom"- , "this"- , "that"- , "these"- , "those"- , "am"- , "is"- , "are"- , "was"- , "were"- , "be"- , "been"- , "being"- , "have"- , "has"- , "had"- , "having"- , "do"- , "does"- , "did"- , "doing"- , "a"- , "an"- , "the"- , "and"- , "but"- , "if"- , "or"- , "because"- , "as"- , "until"- , "while"- , "of"- , "at"- , "by"- , "for"- , "with"- , "about"- , "against"- , "between"- , "into"- , "through"- , "during"- , "before"- , "after"- , "above"- , "below"- , "to"- , "from"- , "up"- , "down"- , "in"- , "out"- , "on"- , "off"- , "over"- , "under"- , "again"- , "further"- , "then"- , "once"- , "here"- , "there"- , "when"- , "where"- , "why"- , "how"- , "all"- , "any"- , "both"- , "each"- , "few"- , "more"- , "most"- , "other"- , "some"- , "such"- , "no"- , "nor"- , "not"- , "only"- , "own"- , "same"- , "so"- , "than"- , "too"- , "very"- , "s"- , "t"- , "can"- , "will"- , "just"- , "don"- , "should"- , "now"- ]+stopWords :: S.Set T.Text+stopWords = S.fromList $ map T.pack+ [ "i"+ , "me"+ , "my"+ , "myself"+ , "we"+ , "our"+ , "ours"+ , "ourselves"+ , "you"+ , "your"+ , "yours"+ , "yourself"+ , "yourselves"+ , "he"+ , "him"+ , "his"+ , "himself"+ , "she"+ , "her"+ , "hers"+ , "herself"+ , "it"+ , "its"+ , "itself"+ , "they"+ , "them"+ , "their"+ , "theirs"+ , "themselves"+ , "what"+ , "which"+ , "who"+ , "whom"+ , "this"+ , "that"+ , "these"+ , "those"+ , "am"+ , "is"+ , "are"+ , "was"+ , "were"+ , "be"+ , "been"+ , "being"+ , "have"+ , "has"+ , "had"+ , "having"+ , "do"+ , "does"+ , "did"+ , "doing"+ , "a"+ , "an"+ , "the"+ , "and"+ , "but"+ , "if"+ , "or"+ , "because"+ , "as"+ , "until"+ , "while"+ , "of"+ , "at"+ , "by"+ , "for"+ , "with"+ , "about"+ , "against"+ , "between"+ , "into"+ , "through"+ , "during"+ , "before"+ , "after"+ , "above"+ , "below"+ , "to"+ , "from"+ , "up"+ , "down"+ , "in"+ , "out"+ , "on"+ , "off"+ , "over"+ , "under"+ , "again"+ , "further"+ , "then"+ , "once"+ , "here"+ , "there"+ , "when"+ , "where"+ , "why"+ , "how"+ , "all"+ , "any"+ , "both"+ , "each"+ , "few"+ , "more"+ , "most"+ , "other"+ , "some"+ , "such"+ , "no"+ , "nor"+ , "not"+ , "only"+ , "own"+ , "same"+ , "so"+ , "than"+ , "too"+ , "very"+ , "s"+ , "t"+ , "can"+ , "will"+ , "just"+ , "don"+ , "should"+ , "now"+ ]
+ src/Condor/Readers/Text.hs view
@@ -0,0 +1,29 @@+{-# LANGUAGE BangPatterns #-}+{- |+Module : Condor.Readers.Text+Copyright : Copyright (C) 2014 Krzysztof Langner+License : The MIT License (MIT)++Maintainer : Krzysztof Langner <klangner@gmail.com>+Stability : alpha+Portability : portable++Reader for text files. Strict version.+Uses bang patters to force hGetContents to read file+-}+module Condor.Readers.Text(readDocument) where++import System.IO+import Condor.DataTypes (Document, docFromStrings)++ +-- | read text as UTF8 and return as document+readDocument :: FilePath -> IO Document+readDocument path = do+ withFile path ReadMode (\h -> do+ hSetEncoding h utf8+ !contents <- hGetContents h + hClose h+ return $ docFromStrings path contents + ) +
src/Condor/Text.hs view
@@ -17,20 +17,21 @@ ( tokenize ) where -import Data.Char (toLower)+import qualified Data.Text as T +-- | List of separators isSeparator :: Char -> Bool isSeparator s = elem s " .,!?" --- | Split string into tokens-tokenize :: String -> [String]-tokenize xs = case dropWhile isSeparator xs of- "" -> []- s' -> foldCase w : tokenize s''- where (w, s'') = break isSeparator s'+-- | Remove suffix separator +removeSeparators:: T.Text -> T.Text+removeSeparators = T.dropWhile isSeparator --- | Convert string to lower case -foldCase :: String -> String-foldCase xs = map toLower xs +-- | Split string into tokens+tokenize :: T.Text -> [T.Text]+tokenize xs = if T.length s == 0 then []+ else (T.toCaseFold t : tokenize r)+ where s = removeSeparators xs+ (t, r) = T.break isSeparator (removeSeparators xs)
+ src/IO.hs view
@@ -0,0 +1,39 @@+{- |+Module : IO+Copyright : Copyright (C) 2013-2014 Krzysztof Langner+License : The MIT License (MIT)++Helper module with functions operating on IO+-}+module IO + ( listFiles+ , listDirs+ )where++import System.Directory (canonicalizePath, getDirectoryContents, doesDirectoryExist)+import Data.List+import Control.Monad+++-- | list files+listFiles :: FilePath -> IO [FilePath] +listFiles p = do + contents <- list p+ filterM (fmap not . doesDirectoryExist) contents++ +-- | list subdirectories+listDirs :: FilePath -> IO [FilePath] +listDirs p = do+ contents <- list p+ filterM doesDirectoryExist contents+ +-- | list directory content+list :: FilePath -> IO [FilePath] +list p = do + ds <- getDirectoryContents p+ let filtered = filter f ds+ path <- canonicalizePath $ p + return $ map ((path++"/")++) filtered+ where f x = and [x /= ".", x /= "..", (not . isPrefixOf ".") x]+
src/Main.hs view
@@ -1,16 +1,17 @@ module Main where import System.Environment-import System.IO-import System.Directory (getDirectoryContents, doesDirectoryExist)+import System.Directory (removeFile, renameFile) import Control.Exception import Control.Monad import Data.Binary import Condor.Index+import qualified Condor.Readers.Text as TextReader+import IO -- | Commands dispatcher-dispatch :: [(String, [String] -> Index -> IO Index)] +dispatch :: [(String, [String] -> Index -> IO ())] dispatch = [ ("index", indexCmd) , ("search", searchCmd) ]@@ -19,61 +20,72 @@ -- This is dummy main function (not working yet) main :: IO ()-main = do +main = do idx <- (readIndex indexFile) `catch` readIndexError (command:args) <- getArgs let (Just action) = lookup command dispatch- idx2 <- action args idx- -- This line is neccessary since otherwise there is problem with locked file- -- Lazy evaluation first opens file for saving index - putStrLn $ "Entries count: " ++ show (termCount idx2)- writeIndex indexFile idx2 + action args idx -- | Read Index data from file readIndex :: FilePath -> IO Index-readIndex p = decodeFile p+readIndex p = {-# SCC "read" #-} decodeFile p -- | Create empty index if can't read from file readIndexError :: IOError -> IO Index readIndexError _ = return emptyIndex +-- | Silent IO error+nullError :: IOError -> IO ()+nullError _ = return ()+ -- | Write index data to the file writeIndex :: FilePath -> Index -> IO ()-writeIndex p i = do encodeFile p i+writeIndex p idx = do+ putStrLn $ "Entries count: " ++ show (termCount idx)+ let temp = p ++ ".temp" + encodeFile temp idx+ removeFile p `catch` nullError+ renameFile temp p -- | Command to index all files from given folder -indexCmd :: [String] -> Index -> IO Index -indexCmd (p:_) idx = do +indexCmd :: [String] -> Index -> IO () +indexCmd [p] idx = do + idx2 <- indexFolder False idx p+ writeIndex indexFile idx2 +indexCmd [p, "-r"] idx = do + idx2 <- indexFolder True idx p+ writeIndex indexFile idx2+indexCmd _ _ = error "add command requires path to the documents"++-- | Index given folder +indexFolder :: Bool -> Index -> FilePath -> IO Index +indexFolder False idx p = do putStrLn $ "Added folder: " ++ p- ds <- getDirectoryContents p - let ps = map ((p++"/")++) ds- fs <- filterM (fmap not . doesDirectoryExist) ps+ fs <- listFiles p idx2 <- foldM addFile idx fs return idx2-indexCmd _ _ = error "add command requires path to the documents"+indexFolder True idx p = do + idx2 <- indexFolder False idx p+ ds <- listDirs p+ idx3 <- foldM (indexFolder True) idx2 ds+ return idx3 -- | Add file to the index addFile :: Index -> FilePath -> IO Index addFile idx p = do- putStrLn $ "Load content from: " ++ p- withFile p ReadMode (\h -> do- hSetEncoding h utf8- contents <- hGetContents h - putStrLn $ "Content length: " ++ show (length contents)- let idx2 = addDocument p contents idx- putStrLn $ "Index entries: " ++ show (termCount idx2)- return idx2 - ) - + doc <- TextReader.readDocument p+ let idx2 = addDocument doc idx+ return idx2 + -- | Command tosearch index-searchCmd :: [String] -> Index -> IO Index+searchCmd :: [String] -> Index -> IO () searchCmd (t:_) idx = do - let result = search idx t+ let result = {-# SCC "search" #-} search idx t putStrLn $ "Search term: " ++ show t ++ " found in documents: "- _ <- mapM putStrLn result- return idx+ _ <- mapM (putStrLn . show) result+ return () searchCmd _ _ = error "search command requires search terms"
test-src/Condor/IndexTest.hs view
@@ -13,39 +13,64 @@ module Condor.IndexTest (testCases) where import Data.Binary+import qualified Data.Text as T+import Condor.DataTypes (DocName, Document(..), docFromStrings) import Condor.Index import Test.HUnit testCases :: [(String, Test)]-testCases = [("Index", t) | t <- tests]--tests :: [Test]-tests = [ TestCase $ prop_empty- , TestCase $ prop_termCount ("doc1", "one two three") 3- , TestCase $ prop_termCount ("doc1", "one and two or three") 3- , TestCase $ prop_search "two" "doc1" [("doc1", "one two three")]- , TestCase $ prop_search "one" "doc1" [ ("doc1", "one two three") - , ("doc2", "forty two")- ]- , TestCase $ prop_search "two" "doc2" [ ("doc1", "one two three") - , ("doc2", "forty two")- ]- , TestCase $ prop_search "two" "doc1" [("doc1", "One Two Three")]- , TestCase $ prop_search "Three" "doc1" [("doc1", "one two three")]- , TestCase $ prop_search "one Three" "doc1" [("doc1", "one two three")]- , TestCase $ prop_search_count "two one" 2 [ ("doc1", "one two three") - , ("doc2", "forty two")- ]- , TestCase $ prop_serialize [ ("doc1", "one two three") - , ("doc2", "forty two")- ] +testCases = [ ( "Empty index has length 0"+ , TestCase $ prop_empty)+ + , ( "Space separates terms"+ , TestCase $ prop_termCount (docFromStrings "doc1" "one two three") 3)+ + , ( "Check stop words"+ , TestCase $ prop_termCount (docFromStrings "doc1" "one and two or three") 3)+ + , ( "Search for term with the same case"+ , TestCase $ prop_search "two" (T.pack "doc1") [docFromStrings "doc1" "one two three"])+ + , ( "Search for first term"+ , TestCase $ prop_search "one" (T.pack "doc1") [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two"+ ])+ + , ( "Search for term in second document"+ , TestCase $ prop_search "two" (T.pack "doc2") [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two"+ ])+ + , ( "Search for lower case term. Document has upper case"+ , TestCase $ prop_search "two" (T.pack "doc1") [docFromStrings "doc1" "One Two Three"])+ + , ( "Search for upperr case term. Document has lower case"+ , TestCase $ prop_search "Three" (T.pack "doc1") [docFromStrings "doc1" "one two three"])+ + , ( "Search for 2 terms"+ , TestCase $ prop_search "one Three" (T.pack "doc1") [docFromStrings "doc1" "one two three"])+ + , ( "Single doc"+ , TestCase $ prop_search_count "one" 1 [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two"+ ])+ + , ( "Document should be returned only once"+ , TestCase $ prop_search_count "two one" 2 [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two"+ ])+ + , ( "encode . decode == id"+ , TestCase $ prop_serialize [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two"+ ]) ] -- | Helper function to populate index -indexFromDocs :: [(DocName, DocContent)] -> Index+indexFromDocs :: [Document] -> Index indexFromDocs ds = foldl f emptyIndex ds- where f i (d, c) = addDocument d c i+ where f i d = addDocument d i -- | Check empty index @@ -53,21 +78,21 @@ prop_empty = assertEqual "Empty index has 0 size" 0 (termCount emptyIndex) -- | Check if document is found -prop_search :: String -> DocName -> [(DocName, DocContent)] -> Assertion +prop_search :: String -> DocName -> [Document] -> Assertion prop_search s e ds = assertEqual s True $ elem e (search idx s) where idx = indexFromDocs ds -- | Count number of returned documents -prop_search_count :: String -> Int -> [(DocName, DocContent)] -> Assertion +prop_search_count :: String -> Int -> [Document] -> Assertion prop_search_count s n ds = assertEqual s n $ length (search idx s) where idx = indexFromDocs ds -- | Check number of terms -prop_termCount :: (DocName, DocContent) -> Int -> Assertion-prop_termCount (d, c) n = assertEqual ("Index size: " ++ c) n (termCount $ addDocument d c emptyIndex) +prop_termCount :: Document -> Int -> Assertion+prop_termCount d n = assertEqual "Count terms" n (termCount $ addDocument d emptyIndex) -- | Check empty index -prop_serialize :: [(DocName, DocContent)] -> Assertion +prop_serialize :: [Document] -> Assertion prop_serialize ds = assertEqual "Serialize" (termCount idx) (termCount idx') where idx = indexFromDocs ds idx' = decode (encode idx)
test-src/Condor/Language/English/PorterTest.hs view
@@ -12,6 +12,7 @@ module Condor.Language.English.PorterTest (testCases) where +import qualified Data.Text as T import Condor.Language.English.Porter import Test.HUnit @@ -32,4 +33,4 @@ ] prop_stem :: String -> String -> Assertion -prop_stem a b = assertEqual a b (stem a) +prop_stem a b = assertEqual a (T.pack b) (stem (T.pack a))
test-src/Condor/TextTest.hs view
@@ -12,20 +12,22 @@ module Condor.TextTest (testCases) where +import qualified Data.Text as T import Condor.Text import Test.HUnit + testCases :: [(String, Test)] testCases = [("Text", t) | t <- tests] tests :: [Test]-tests = [ TestCase $ prop_tokenize_count "one two three" 3- , TestCase $ prop_tokenize_count "one,two " 2- , TestCase $ prop_tokenize_token " one?two! " 1 "two"+tests = [ TestCase $ prop_tokenize_count (T.pack "one two three") 3+ , TestCase $ prop_tokenize_count (T.pack "one,two ") 2+ , TestCase $ prop_tokenize_token (T.pack " one?two! ") 1 (T.pack "two") ] -prop_tokenize_count :: String -> Int -> Assertion -prop_tokenize_count xs n = assertEqual xs n (length (tokenize xs)) +prop_tokenize_count :: T.Text -> Int -> Assertion +prop_tokenize_count xs n = assertEqual (show xs) n (length (tokenize xs)) -prop_tokenize_token :: String -> Int -> String -> Assertion -prop_tokenize_token xs i t = assertEqual xs t ((tokenize xs)!!i) +prop_tokenize_token :: T.Text -> Int -> T.Text -> Assertion +prop_tokenize_token xs i t = assertEqual (show xs) t ((tokenize xs)!!i)