diff --git a/CHANGES b/CHANGES --- a/CHANGES +++ b/CHANGES @@ -1,1 +1,10 @@ -0.1 Initial version +Version 0.2 2014-01-08 +* Changes in Index API +* Add recursive option to command line tool. +* Post list contains document id instead of document name. + + + +Version 0.1 2014-01-04 +* Basic index implementation +* Basic command line tool diff --git a/Condor.cabal b/Condor.cabal --- a/Condor.cabal +++ b/Condor.cabal @@ -1,5 +1,5 @@ name: Condor -version: 0.1 +version: 0.2 cabal-version: >= 1.10 build-type: Simple author: Krzysztof Langner @@ -28,25 +28,32 @@ binary >=0.5.1 && <0.6, containers >=0.5.0 && <0.6, directory >=1.2.0 && <1.3, - filepath >=1.3.0 && <1.4 + filepath >=1.3.0 && <1.4, + text >=1 && <1.1 + ghc-options: -Wall other-modules: + Condor.Index, + Condor.Readers.Text, Condor.Text, - Condor.Index + IO library hs-source-dirs: src default-language: Haskell2010 ghc-options: -Wall - exposed-modules: Condor.Index - other-modules: + exposed-modules: + Condor.DataTypes, + Condor.Index, Condor.Language.English.Porter, Condor.Language.English.StopWords, + Condor.Readers.Text, Condor.Text build-depends: base >= 4 && <4.7, containers >=0.5.0 && <0.6, - binary >=0.5.1 && <0.6 + binary >=0.5.1 && <0.6, + text >=1 && <1.1 test-suite unit-tests type: detailed-0.9 @@ -58,7 +65,9 @@ HUnit >=1.2.5 && <1.3, Cabal >=1.16.0 && <1.17, containers >=0.5.0 && <0.6, - binary >=0.5.1 && <0.6 + binary >=0.5.1 && <0.6, + text >=1 && <1.1 + other-modules: Condor.Index, Condor.IndexTest, @@ -67,7 +76,8 @@ Condor.Language.English.StopWords, Condor.Text, Condor.TextTest, - Distribution.TestSuite.HUnit + Distribution.TestSuite.HUnit, + Condor.DataTypes hs-source-dirs: src, test-src diff --git a/src/Condor/DataTypes.hs b/src/Condor/DataTypes.hs new file mode 100644 --- /dev/null +++ b/src/Condor/DataTypes.hs @@ -0,0 +1,46 @@ +{- | +Module : Condor.DataTypes +Copyright : Copyright (C) 2014 Krzysztof Langner +License : The MIT License (MIT) + +Maintainer : Krzysztof Langner +Stability : alpha +Portability : portable + +Common to all modules data types definitions. +-} +module Condor.DataTypes + ( DocName + , Document(..) + , docFromStrings + , docName + , docText + ) where + +import qualified Data.Text as T + + +type DocName = T.Text + +-- | Field consists of title and content +data Field = Field T.Text T.Text + +-- | Document with name and contents +data Document = Document DocName [Field] + + +-- | Create simple field from strings +fieldFromStrings :: String -> String -> Field +fieldFromStrings k v = Field (T.pack k) (T.pack v) + +-- | Create simple document with name and single field content. +docFromStrings :: String -> String -> Document +docFromStrings t c = Document (T.pack t) [fieldFromStrings "content" c] + +-- | Get document name +docName :: Document -> T.Text +docName (Document a _) = a + +-- | Get text from all fields +docText :: Document -> T.Text +docText (Document _ fs) = T.concat $ map (\(Field _ y) -> y) fs diff --git a/src/Condor/Index.hs b/src/Condor/Index.hs --- a/src/Condor/Index.hs +++ b/src/Condor/Index.hs @@ -9,83 +9,98 @@ This module contains functions which create, update and search index. Default implementation uses algorithms for english language (stemming, stop words etc.) -But it should be possible to customize it for any language by modifying Index data type. +Functions in this module (for performance reasons) are based on unicode strings Data.Text. + -} module Condor.Index ( DocName , Index - , DocContent , addDocument + , addDocTerms , emptyIndex , search + , searchTerms , termCount ) where import qualified Data.Map as Map import qualified Data.List as List +import qualified Data.Text as T +import qualified Data.Text.Encoding as E import Data.Binary import Condor.Text +import Condor.DataTypes (DocName, Document(..), docName, docText) import Condor.Language.English.StopWords (isStopWord) import Condor.Language.English.Porter (stem) -type DocName = String -type DocContent = String - --- | Index parameters. Those parameters can be used to change how --- the text is processed before adding to the index -data IndexParams = IndexParams { ignore :: String -> Bool - , stemmer :: String -> String - } - --- An instance of Binary to encode and decode an IndexParams in binary -instance Binary IndexParams where - put _ = put (0 :: Word8) - get = do tag <- getWord8 - case tag of - _ -> return $ IndexParams isStopWord stem +type Term = T.Text -- | Inverted index -data Index = Index { terms :: Map.Map String [String] - , params :: IndexParams +data Index = Index { terms :: Map.Map Term [Int] + , docs :: [DocName] } -- An instance of Binary to encode and decode an IndexParams in binary instance Binary Index where - put i = do put (terms i) - put (params i) + put i = do + put (terms i) + put (docs i) get = do i <- get - p <- get - return $ Index i p + d <- get + return $ Index i d +-- An instance of Binary to encode and decode T.Text +instance Binary T.Text where + put i = do put (E.encodeUtf8 i) + get = do i <- get + return $ E.decodeUtf8 i + -- | Create empty index. -- This index will be configured for english language. emptyIndex :: Index -emptyIndex = Index Map.empty (IndexParams isStopWord stem) +emptyIndex = Index Map.empty [] --- | Add document to the index -addDocument :: DocName -> DocContent -> Index -> Index -addDocument d c ix = Index (foldl f (terms ix) ws) (params ix) - where ws = splitWords (params ix) c - f i t = case Map.lookup t i of - Just a -> Map.insert t (d:a) i - Nothing -> Map.insert t [d] i +-- | Add document to the index. +-- This function uses algorithms for english language to split document content +-- into index terms. +addDocument :: Document -> Index -> Index +addDocument d idx = addDocTerms (docName d) (splitTerms content) idx + where content = docText d --- | Search term in the index -search :: Index -> DocContent -> [DocName] -search ix s = List.nub $ foldl (++) [] ys - where ys = map (searchTerm ix) ws - ws = splitWords (params ix) s +-- | Add document to the index. +-- This function should be used if document content should be splitted into terms +-- with custom algorithms. +addDocTerms :: DocName -> [Term] -> Index -> Index +addDocTerms d c ix = Index (foldl f (terms ix) c) (d:docs ix) + where f ix' t = case Map.lookup t ix' of + Just a -> Map.insert t (index:a) ix' + Nothing -> Map.insert t [index] ix' + index = length (docs ix) +-- | Search terms given as single string in the index +-- This function uses algorithms for english language to split query into tokens. +search :: Index -> String -> [DocName] +search ix s = searchTerms ix (splitTerms (T.pack s)) + + +-- | Search terms given as array in the index. +-- This function should be used if query should be splitted into terms +-- with custom algorithms +searchTerms :: Index -> [Term] -> [DocName] +searchTerms ix s = List.nub $ foldl (++) [] ys + where ys = map (findDocs ix) s + + -- | Search single term in the index -searchTerm :: Index -> String -> [DocName] -searchTerm ix s = case Map.lookup s (terms ix) of - Just a -> a +findDocs :: Index -> Term -> [DocName] +findDocs ix s = case Map.lookup s (terms ix) of + Just a -> map ((reverse (docs ix))!!) a Nothing -> [] @@ -94,11 +109,10 @@ termCount ix = Map.size (terms ix) --- | Split text into tokens. +-- | Split text into terms. -- This function removes stop words and stems words -splitWords :: IndexParams -> String -> [String] -splitWords p s = map (stemmer p) (filter f t) +splitTerms :: T.Text -> [Term] +splitTerms s = map stem (filter (not . isStopWord) t) where t = tokenize s - f = \x -> not ((ignore p) x) diff --git a/src/Condor/Language/English/Porter.hs b/src/Condor/Language/English/Porter.hs --- a/src/Condor/Language/English/Porter.hs +++ b/src/Condor/Language/English/Porter.hs @@ -5,7 +5,15 @@ import Control.Monad import Data.Maybe import Data.List +import qualified Data.Text as T + +-- | API stemming function +stem :: T.Text -> T.Text +stem s | T.length s < 3 = s + | otherwise = T.pack $ allSteps (T.unpack s) + + isConsonant :: [Char] -> Int -> Bool isConsonant str i | c `elem` "aeiou" = False @@ -158,8 +166,3 @@ allSteps :: [Char] -> [Char] allSteps = step5 . step4 . step3 . step2 . step1 - -stem :: [Char] -> [Char] -stem s | length s < 3 = s - | otherwise = allSteps s - diff --git a/src/Condor/Language/English/StopWords.hs b/src/Condor/Language/English/StopWords.hs --- a/src/Condor/Language/English/StopWords.hs +++ b/src/Condor/Language/English/StopWords.hs @@ -12,139 +12,141 @@ -} module Condor.Language.English.StopWords (isStopWord) where -import Data.Set +import qualified Data.Set as S +import qualified Data.Text as T -- | Predicate to check if given words is a stop word -isStopWord :: String -> Bool -isStopWord w = member w stopWords +isStopWord :: T.Text -> Bool +isStopWord w = S.member w stopWords -stopWords :: Set String -stopWords = fromList [ "i" - , "me" - , "my" - , "myself" - , "we" - , "our" - , "ours" - , "ourselves" - , "you" - , "your" - , "yours" - , "yourself" - , "yourselves" - , "he" - , "him" - , "his" - , "himself" - , "she" - , "her" - , "hers" - , "herself" - , "it" - , "its" - , "itself" - , "they" - , "them" - , "their" - , "theirs" - , "themselves" - , "what" - , "which" - , "who" - , "whom" - , "this" - , "that" - , "these" - , "those" - , "am" - , "is" - , "are" - , "was" - , "were" - , "be" - , "been" - , "being" - , "have" - , "has" - , "had" - , "having" - , "do" - , "does" - , "did" - , "doing" - , "a" - , "an" - , "the" - , "and" - , "but" - , "if" - , "or" - , "because" - , "as" - , "until" - , "while" - , "of" - , "at" - , "by" - , "for" - , "with" - , "about" - , "against" - , "between" - , "into" - , "through" - , "during" - , "before" - , "after" - , "above" - , "below" - , "to" - , "from" - , "up" - , "down" - , "in" - , "out" - , "on" - , "off" - , "over" - , "under" - , "again" - , "further" - , "then" - , "once" - , "here" - , "there" - , "when" - , "where" - , "why" - , "how" - , "all" - , "any" - , "both" - , "each" - , "few" - , "more" - , "most" - , "other" - , "some" - , "such" - , "no" - , "nor" - , "not" - , "only" - , "own" - , "same" - , "so" - , "than" - , "too" - , "very" - , "s" - , "t" - , "can" - , "will" - , "just" - , "don" - , "should" - , "now" - ] +stopWords :: S.Set T.Text +stopWords = S.fromList $ map T.pack + [ "i" + , "me" + , "my" + , "myself" + , "we" + , "our" + , "ours" + , "ourselves" + , "you" + , "your" + , "yours" + , "yourself" + , "yourselves" + , "he" + , "him" + , "his" + , "himself" + , "she" + , "her" + , "hers" + , "herself" + , "it" + , "its" + , "itself" + , "they" + , "them" + , "their" + , "theirs" + , "themselves" + , "what" + , "which" + , "who" + , "whom" + , "this" + , "that" + , "these" + , "those" + , "am" + , "is" + , "are" + , "was" + , "were" + , "be" + , "been" + , "being" + , "have" + , "has" + , "had" + , "having" + , "do" + , "does" + , "did" + , "doing" + , "a" + , "an" + , "the" + , "and" + , "but" + , "if" + , "or" + , "because" + , "as" + , "until" + , "while" + , "of" + , "at" + , "by" + , "for" + , "with" + , "about" + , "against" + , "between" + , "into" + , "through" + , "during" + , "before" + , "after" + , "above" + , "below" + , "to" + , "from" + , "up" + , "down" + , "in" + , "out" + , "on" + , "off" + , "over" + , "under" + , "again" + , "further" + , "then" + , "once" + , "here" + , "there" + , "when" + , "where" + , "why" + , "how" + , "all" + , "any" + , "both" + , "each" + , "few" + , "more" + , "most" + , "other" + , "some" + , "such" + , "no" + , "nor" + , "not" + , "only" + , "own" + , "same" + , "so" + , "than" + , "too" + , "very" + , "s" + , "t" + , "can" + , "will" + , "just" + , "don" + , "should" + , "now" + ] diff --git a/src/Condor/Readers/Text.hs b/src/Condor/Readers/Text.hs new file mode 100644 --- /dev/null +++ b/src/Condor/Readers/Text.hs @@ -0,0 +1,29 @@ +{-# LANGUAGE BangPatterns #-} +{- | +Module : Condor.Readers.Text +Copyright : Copyright (C) 2014 Krzysztof Langner +License : The MIT License (MIT) + +Maintainer : Krzysztof Langner +Stability : alpha +Portability : portable + +Reader for text files. Strict version. +Uses bang patters to force hGetContents to read file +-} +module Condor.Readers.Text(readDocument) where + +import System.IO +import Condor.DataTypes (Document, docFromStrings) + + +-- | read text as UTF8 and return as document +readDocument :: FilePath -> IO Document +readDocument path = do + withFile path ReadMode (\h -> do + hSetEncoding h utf8 + !contents <- hGetContents h + hClose h + return $ docFromStrings path contents + ) + diff --git a/src/Condor/Text.hs b/src/Condor/Text.hs --- a/src/Condor/Text.hs +++ b/src/Condor/Text.hs @@ -17,20 +17,21 @@ ( tokenize ) where -import Data.Char (toLower) +import qualified Data.Text as T +-- | List of separators isSeparator :: Char -> Bool isSeparator s = elem s " .,!?" --- | Split string into tokens -tokenize :: String -> [String] -tokenize xs = case dropWhile isSeparator xs of - "" -> [] - s' -> foldCase w : tokenize s'' - where (w, s'') = break isSeparator s' +-- | Remove suffix separator +removeSeparators:: T.Text -> T.Text +removeSeparators = T.dropWhile isSeparator --- | Convert string to lower case -foldCase :: String -> String -foldCase xs = map toLower xs +-- | Split string into tokens +tokenize :: T.Text -> [T.Text] +tokenize xs = if T.length s == 0 then [] + else (T.toCaseFold t : tokenize r) + where s = removeSeparators xs + (t, r) = T.break isSeparator (removeSeparators xs) diff --git a/src/IO.hs b/src/IO.hs new file mode 100644 --- /dev/null +++ b/src/IO.hs @@ -0,0 +1,39 @@ +{- | +Module : IO +Copyright : Copyright (C) 2013-2014 Krzysztof Langner +License : The MIT License (MIT) + +Helper module with functions operating on IO +-} +module IO + ( listFiles + , listDirs + )where + +import System.Directory (canonicalizePath, getDirectoryContents, doesDirectoryExist) +import Data.List +import Control.Monad + + +-- | list files +listFiles :: FilePath -> IO [FilePath] +listFiles p = do + contents <- list p + filterM (fmap not . doesDirectoryExist) contents + + +-- | list subdirectories +listDirs :: FilePath -> IO [FilePath] +listDirs p = do + contents <- list p + filterM doesDirectoryExist contents + +-- | list directory content +list :: FilePath -> IO [FilePath] +list p = do + ds <- getDirectoryContents p + let filtered = filter f ds + path <- canonicalizePath $ p + return $ map ((path++"/")++) filtered + where f x = and [x /= ".", x /= "..", (not . isPrefixOf ".") x] + diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -1,16 +1,17 @@ module Main where import System.Environment -import System.IO -import System.Directory (getDirectoryContents, doesDirectoryExist) +import System.Directory (removeFile, renameFile) import Control.Exception import Control.Monad import Data.Binary import Condor.Index +import qualified Condor.Readers.Text as TextReader +import IO -- | Commands dispatcher -dispatch :: [(String, [String] -> Index -> IO Index)] +dispatch :: [(String, [String] -> Index -> IO ())] dispatch = [ ("index", indexCmd) , ("search", searchCmd) ] @@ -19,61 +20,72 @@ -- This is dummy main function (not working yet) main :: IO () -main = do +main = do idx <- (readIndex indexFile) `catch` readIndexError (command:args) <- getArgs let (Just action) = lookup command dispatch - idx2 <- action args idx - -- This line is neccessary since otherwise there is problem with locked file - -- Lazy evaluation first opens file for saving index - putStrLn $ "Entries count: " ++ show (termCount idx2) - writeIndex indexFile idx2 + action args idx -- | Read Index data from file readIndex :: FilePath -> IO Index -readIndex p = decodeFile p +readIndex p = {-# SCC "read" #-} decodeFile p -- | Create empty index if can't read from file readIndexError :: IOError -> IO Index readIndexError _ = return emptyIndex +-- | Silent IO error +nullError :: IOError -> IO () +nullError _ = return () + -- | Write index data to the file writeIndex :: FilePath -> Index -> IO () -writeIndex p i = do encodeFile p i +writeIndex p idx = do + putStrLn $ "Entries count: " ++ show (termCount idx) + let temp = p ++ ".temp" + encodeFile temp idx + removeFile p `catch` nullError + renameFile temp p -- | Command to index all files from given folder -indexCmd :: [String] -> Index -> IO Index -indexCmd (p:_) idx = do +indexCmd :: [String] -> Index -> IO () +indexCmd [p] idx = do + idx2 <- indexFolder False idx p + writeIndex indexFile idx2 +indexCmd [p, "-r"] idx = do + idx2 <- indexFolder True idx p + writeIndex indexFile idx2 +indexCmd _ _ = error "add command requires path to the documents" + +-- | Index given folder +indexFolder :: Bool -> Index -> FilePath -> IO Index +indexFolder False idx p = do putStrLn $ "Added folder: " ++ p - ds <- getDirectoryContents p - let ps = map ((p++"/")++) ds - fs <- filterM (fmap not . doesDirectoryExist) ps + fs <- listFiles p idx2 <- foldM addFile idx fs return idx2 -indexCmd _ _ = error "add command requires path to the documents" +indexFolder True idx p = do + idx2 <- indexFolder False idx p + ds <- listDirs p + idx3 <- foldM (indexFolder True) idx2 ds + return idx3 -- | Add file to the index addFile :: Index -> FilePath -> IO Index addFile idx p = do - putStrLn $ "Load content from: " ++ p - withFile p ReadMode (\h -> do - hSetEncoding h utf8 - contents <- hGetContents h - putStrLn $ "Content length: " ++ show (length contents) - let idx2 = addDocument p contents idx - putStrLn $ "Index entries: " ++ show (termCount idx2) - return idx2 - ) - + doc <- TextReader.readDocument p + let idx2 = addDocument doc idx + return idx2 + -- | Command tosearch index -searchCmd :: [String] -> Index -> IO Index +searchCmd :: [String] -> Index -> IO () searchCmd (t:_) idx = do - let result = search idx t + let result = {-# SCC "search" #-} search idx t putStrLn $ "Search term: " ++ show t ++ " found in documents: " - _ <- mapM putStrLn result - return idx + _ <- mapM (putStrLn . show) result + return () searchCmd _ _ = error "search command requires search terms" diff --git a/test-src/Condor/IndexTest.hs b/test-src/Condor/IndexTest.hs --- a/test-src/Condor/IndexTest.hs +++ b/test-src/Condor/IndexTest.hs @@ -13,39 +13,64 @@ module Condor.IndexTest (testCases) where import Data.Binary +import qualified Data.Text as T +import Condor.DataTypes (DocName, Document(..), docFromStrings) import Condor.Index import Test.HUnit testCases :: [(String, Test)] -testCases = [("Index", t) | t <- tests] - -tests :: [Test] -tests = [ TestCase $ prop_empty - , TestCase $ prop_termCount ("doc1", "one two three") 3 - , TestCase $ prop_termCount ("doc1", "one and two or three") 3 - , TestCase $ prop_search "two" "doc1" [("doc1", "one two three")] - , TestCase $ prop_search "one" "doc1" [ ("doc1", "one two three") - , ("doc2", "forty two") - ] - , TestCase $ prop_search "two" "doc2" [ ("doc1", "one two three") - , ("doc2", "forty two") - ] - , TestCase $ prop_search "two" "doc1" [("doc1", "One Two Three")] - , TestCase $ prop_search "Three" "doc1" [("doc1", "one two three")] - , TestCase $ prop_search "one Three" "doc1" [("doc1", "one two three")] - , TestCase $ prop_search_count "two one" 2 [ ("doc1", "one two three") - , ("doc2", "forty two") - ] - , TestCase $ prop_serialize [ ("doc1", "one two three") - , ("doc2", "forty two") - ] +testCases = [ ( "Empty index has length 0" + , TestCase $ prop_empty) + + , ( "Space separates terms" + , TestCase $ prop_termCount (docFromStrings "doc1" "one two three") 3) + + , ( "Check stop words" + , TestCase $ prop_termCount (docFromStrings "doc1" "one and two or three") 3) + + , ( "Search for term with the same case" + , TestCase $ prop_search "two" (T.pack "doc1") [docFromStrings "doc1" "one two three"]) + + , ( "Search for first term" + , TestCase $ prop_search "one" (T.pack "doc1") [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two" + ]) + + , ( "Search for term in second document" + , TestCase $ prop_search "two" (T.pack "doc2") [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two" + ]) + + , ( "Search for lower case term. Document has upper case" + , TestCase $ prop_search "two" (T.pack "doc1") [docFromStrings "doc1" "One Two Three"]) + + , ( "Search for upperr case term. Document has lower case" + , TestCase $ prop_search "Three" (T.pack "doc1") [docFromStrings "doc1" "one two three"]) + + , ( "Search for 2 terms" + , TestCase $ prop_search "one Three" (T.pack "doc1") [docFromStrings "doc1" "one two three"]) + + , ( "Single doc" + , TestCase $ prop_search_count "one" 1 [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two" + ]) + + , ( "Document should be returned only once" + , TestCase $ prop_search_count "two one" 2 [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two" + ]) + + , ( "encode . decode == id" + , TestCase $ prop_serialize [ docFromStrings "doc1" "one two three" + , docFromStrings "doc2" "forty two" + ]) ] -- | Helper function to populate index -indexFromDocs :: [(DocName, DocContent)] -> Index +indexFromDocs :: [Document] -> Index indexFromDocs ds = foldl f emptyIndex ds - where f i (d, c) = addDocument d c i + where f i d = addDocument d i -- | Check empty index @@ -53,21 +78,21 @@ prop_empty = assertEqual "Empty index has 0 size" 0 (termCount emptyIndex) -- | Check if document is found -prop_search :: String -> DocName -> [(DocName, DocContent)] -> Assertion +prop_search :: String -> DocName -> [Document] -> Assertion prop_search s e ds = assertEqual s True $ elem e (search idx s) where idx = indexFromDocs ds -- | Count number of returned documents -prop_search_count :: String -> Int -> [(DocName, DocContent)] -> Assertion +prop_search_count :: String -> Int -> [Document] -> Assertion prop_search_count s n ds = assertEqual s n $ length (search idx s) where idx = indexFromDocs ds -- | Check number of terms -prop_termCount :: (DocName, DocContent) -> Int -> Assertion -prop_termCount (d, c) n = assertEqual ("Index size: " ++ c) n (termCount $ addDocument d c emptyIndex) +prop_termCount :: Document -> Int -> Assertion +prop_termCount d n = assertEqual "Count terms" n (termCount $ addDocument d emptyIndex) -- | Check empty index -prop_serialize :: [(DocName, DocContent)] -> Assertion +prop_serialize :: [Document] -> Assertion prop_serialize ds = assertEqual "Serialize" (termCount idx) (termCount idx') where idx = indexFromDocs ds idx' = decode (encode idx) diff --git a/test-src/Condor/Language/English/PorterTest.hs b/test-src/Condor/Language/English/PorterTest.hs --- a/test-src/Condor/Language/English/PorterTest.hs +++ b/test-src/Condor/Language/English/PorterTest.hs @@ -12,6 +12,7 @@ module Condor.Language.English.PorterTest (testCases) where +import qualified Data.Text as T import Condor.Language.English.Porter import Test.HUnit @@ -32,4 +33,4 @@ ] prop_stem :: String -> String -> Assertion -prop_stem a b = assertEqual a b (stem a) +prop_stem a b = assertEqual a (T.pack b) (stem (T.pack a)) diff --git a/test-src/Condor/TextTest.hs b/test-src/Condor/TextTest.hs --- a/test-src/Condor/TextTest.hs +++ b/test-src/Condor/TextTest.hs @@ -12,20 +12,22 @@ module Condor.TextTest (testCases) where +import qualified Data.Text as T import Condor.Text import Test.HUnit + testCases :: [(String, Test)] testCases = [("Text", t) | t <- tests] tests :: [Test] -tests = [ TestCase $ prop_tokenize_count "one two three" 3 - , TestCase $ prop_tokenize_count "one,two " 2 - , TestCase $ prop_tokenize_token " one?two! " 1 "two" +tests = [ TestCase $ prop_tokenize_count (T.pack "one two three") 3 + , TestCase $ prop_tokenize_count (T.pack "one,two ") 2 + , TestCase $ prop_tokenize_token (T.pack " one?two! ") 1 (T.pack "two") ] -prop_tokenize_count :: String -> Int -> Assertion -prop_tokenize_count xs n = assertEqual xs n (length (tokenize xs)) +prop_tokenize_count :: T.Text -> Int -> Assertion +prop_tokenize_count xs n = assertEqual (show xs) n (length (tokenize xs)) -prop_tokenize_token :: String -> Int -> String -> Assertion -prop_tokenize_token xs i t = assertEqual xs t ((tokenize xs)!!i) +prop_tokenize_token :: T.Text -> Int -> T.Text -> Assertion +prop_tokenize_token xs i t = assertEqual (show xs) t ((tokenize xs)!!i)