diff --git a/glider-nlp.cabal b/glider-nlp.cabal --- a/glider-nlp.cabal +++ b/glider-nlp.cabal @@ -1,5 +1,5 @@ name: glider-nlp -version: 0.1 +version: 0.4 cabal-version: >=1.10 build-type: Simple author: Krzysztof Langner @@ -28,9 +28,9 @@ default-language: Haskell2010 ghc-options: -Wall build-depends: - base >= 4 && <4.7, - text >=1 && <1.1, - containers >=0.5.0 && <0.6 + base >= 4 && <5, + text >=1 && <2, + containers >=0.5 && <1 exposed-modules: Glider.NLP.Language.English.Porter, Glider.NLP.Language.English.StopWords, @@ -40,28 +40,28 @@ other-modules: Glider.NLP.Language.Default test-suite unit-tests - type: detailed-0.9 - test-module: AllTests + type: exitcode-stdio-1.0 + main-is: Spec.hs default-language: Haskell2010 ghc-options: -Wall -rtsopts build-depends: - base >= 4 && <4.7, - HUnit >=1.2.5 && <1.3, - Cabal >=1.16.0 && <1.17, - text >=1 && <1.1, - containers >=0.5.0 && <0.6 + base >= 4 && <5, + hspec >=2 && <3, + Cabal >=1 && <2, + text >=1 && <2, + containers >=0.5 && <1 hs-source-dirs: src, test-src other-modules: - Distribution.TestSuite.HUnit, Glider.NLP.Language.Default, - Glider.NLP.Language.DefaultTest, Glider.NLP.Language.English.Porter, - Glider.NLP.Language.English.PorterTest, Glider.NLP.Language.English.StopWords, Glider.NLP.Language.Polish.StopWords, Glider.NLP.Statistics, - Glider.NLP.StatisticsTest, Glider.NLP.Tokenizer, - Glider.NLP.TokenizerTest + + Glider.NLP.Language.DefaultSpec, + Glider.NLP.Language.English.PorterSpec, + Glider.NLP.StatisticsSpec, + Glider.NLP.TokenizerSpec diff --git a/src/Glider/NLP/Statistics.hs b/src/Glider/NLP/Statistics.hs --- a/src/Glider/NLP/Statistics.hs +++ b/src/Glider/NLP/Statistics.hs @@ -1,6 +1,7 @@ +{-# LANGUAGE OverloadedStrings #-} {- | Module : Glider.NLP.Statistics -Copyright : Copyright (C) 2013-2014 Krzysztof Langner +Copyright : Copyright (C) 2013-2016 Krzysztof Langner License : BSD3 Maintainer : Krzysztof Langner @@ -23,13 +24,13 @@ -- | Count number of words in the text. -- --- > countWords (T.pack "one two three") == 3 +-- > countWords "one two three" == 3 countWords :: Text -> Int countWords = List.length . getWords . tokenize -- | Count word frequency -- --- > wordFreq (T.pack "one two, three one") == [("one", 2), ("two", 1), ("three", 1)] +-- > wordFreq "one two, three one" == [("one", 2), ("two", 1), ("three", 1)] wordFreq :: Text -> [(Text, Int)] wordFreq a = [(List.head xs, List.length xs) | xs <- List.group tokens] where tokens = List.sort $ (foldCase . getWords . tokenize) a diff --git a/src/Glider/NLP/Tokenizer.hs b/src/Glider/NLP/Tokenizer.hs --- a/src/Glider/NLP/Tokenizer.hs +++ b/src/Glider/NLP/Tokenizer.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE OverloadedStrings #-} {- | Module : Glider.NLP.Tokenizer Copyright : Copyright (C) 2013-2014 Krzysztof Langner @@ -8,8 +9,8 @@ Portability : portable This module contains functions which parses text into tokens. -tokens are not normalized. If you need all tokens from the document then -check function "tokenize". If you need only words (na dots, numbers etc.) +Tokens are not normalized. If you need all tokens from the document then +use function "tokenize". If you need only words (no dots, numbers etc.) then check function "getWords". -} @@ -46,16 +47,16 @@ -- > tokenize "one two." == [Word "one", Whitespace, Word "two", "Separator "."] tokenize :: Text -> [Token] tokenize xs = case allParser xs of - [(v, out)] -> (v:tokenize out) - _ -> [] + [(v, out)] -> v : tokenize out + _ -> [] --- | Exctract all words from tokens +-- | Extract all words from tokens -- -- > getWords "one two." == ["one", "two"] getWords :: [Token] -> [Text] getWords [] = [] getWords (x:xs) = case x of - Word a -> (a: getWords xs) + Word a -> a: getWords xs _ -> getWords xs -- | Convert all words to the same case @@ -81,13 +82,13 @@ -- | Parse punctuation punctuationParser :: Parser punctuationParser xs | null xs = [] - | isPunctuation (head xs) = [(Punctuation (head xs), (tail xs))] + | isPunctuation (head xs) = [(Punctuation (head xs), tail xs)] | otherwise = [] -- | Parse symbol symbolParser :: Parser symbolParser xs | null xs = [] - | isSymbol (head xs) = [(Symbol (head xs), (tail xs))] + | isSymbol (head xs) = [(Symbol (head xs), tail xs)] | otherwise = [] -- | Parse whitespaces diff --git a/test-src/AllTests.hs b/test-src/AllTests.hs deleted file mode 100644 --- a/test-src/AllTests.hs +++ /dev/null @@ -1,17 +0,0 @@ -module AllTests (tests) where - -import qualified Distribution.TestSuite as C -import qualified Distribution.TestSuite.HUnit as H -import Glider.NLP.TokenizerTest (testCases) -import Glider.NLP.StatisticsTest (testCases) -import Glider.NLP.Language.English.PorterTest (testCases) -import Glider.NLP.Language.DefaultTest (testCases) - - -tests :: IO [C.Test] -tests = return $ map (uncurry H.test) $ Glider.NLP.TokenizerTest.testCases - ++ Glider.NLP.Language.English.PorterTest.testCases - ++ Glider.NLP.Language.DefaultTest.testCases - ++ Glider.NLP.StatisticsTest.testCases - - diff --git a/test-src/Distribution/TestSuite/HUnit.hs b/test-src/Distribution/TestSuite/HUnit.hs deleted file mode 100644 --- a/test-src/Distribution/TestSuite/HUnit.hs +++ /dev/null @@ -1,66 +0,0 @@ -------------------------------------------------------------------------------- --- | --- Module : Distribution.TestSuite.HUnit --- Copyright : Patrick Brisbin 2013 --- --- Maintainer : pbrisbin@gmail.com --- Portability : portable --- --- Test interface for running HUnit tests via cabal. --- --- Usage: --- --- > import Test.HUnit --- > --- > import qualified Distribution.TestSuite as C --- > import qualified Distribution.TestSuite.HUnit as H --- > --- > tests :: IO [C.Test] --- > tests = return $ map (uncurry H.test) testCases --- > --- > testCases :: [(String, Test)] --- > testCases = [("Truth test", truthTest)] --- > --- > truthTest :: Test --- > truthTest = assertEqual True True --- --- An early version of this module, written by Thomas Tuegel --- , can be found at: --- --- --- --- This file shares no code with that one, as the changes in Cabal 1.16 --- required it be entirely rewritten. I've retained Thomas' LICENSE but --- removed his copyright; I hope that's appropriate. --- -------------------------------------------------------------------------------- -module Distribution.TestSuite.HUnit (test) where - -import Distribution.TestSuite -import qualified Test.HUnit as H - -test :: String -> H.Test -> Test -test n = Test . testInstance n - -testInstance :: String -> H.Test -> TestInstance -testInstance n t = TestInstance - { run = runTest t - , name = n - , tags = [] - , options = [] - , setOption = \_ _ -> Right $ testInstance n t - } - -runTest :: H.Test -> IO Progress -runTest = fmap snd . H.performTest onStart onError onFailure (Finished Pass) - - where - - onStart :: H.State -> Progress -> IO Progress - onStart _ = return - - onError :: String -> H.State -> Progress -> IO Progress - onError msg _ _ = return $ Finished (Error msg) - - onFailure :: String -> H.State -> Progress -> IO Progress - onFailure msg _ _ = return $ Finished (Fail msg) diff --git a/test-src/Glider/NLP/Language/DefaultSpec.hs b/test-src/Glider/NLP/Language/DefaultSpec.hs new file mode 100644 --- /dev/null +++ b/test-src/Glider/NLP/Language/DefaultSpec.hs @@ -0,0 +1,21 @@ +{-# LANGUAGE OverloadedStrings #-} +{- | +Module : Glider.NLP.Language.DefaultSpec +Copyright : Copyright (C) 2013-2014 Krzysztof Langner +License : BSD3 + +Maintainer : Krzysztof Langner +Stability : alpha +Portability : portable +-} + +module Glider.NLP.Language.DefaultSpec (spec) where + +import Test.Hspec +import Glider.NLP.Language.Default + + +spec :: Spec +spec = describe "Stop word" $ do + it "less then 4 chars." $ isStopWord "ala" `shouldBe` True + it "4 chars." $ isStopWord "abcd" `shouldBe` False diff --git a/test-src/Glider/NLP/Language/DefaultTest.hs b/test-src/Glider/NLP/Language/DefaultTest.hs deleted file mode 100644 --- a/test-src/Glider/NLP/Language/DefaultTest.hs +++ /dev/null @@ -1,28 +0,0 @@ -{- | -Module : Glider.NLP.Language.DefaultTest -Copyright : Copyright (C) 2013-2014 Krzysztof Langner -License : BSD3 - -Maintainer : Krzysztof Langner -Stability : alpha -Portability : portable --} - -module Glider.NLP.Language.DefaultTest (testCases) where - -import Data.Text -import Glider.NLP.Language.Default -import Test.HUnit - - -testCases :: [(String, Test)] -testCases = [ ( "Less then 4 chars" - , TestCase $ prop_isStopWord "abc" True) - , ( "4 chars not stop word" - , TestCase $ prop_isStopWord "abcd" False) - ] - --- | Check empty index -prop_isStopWord :: String -> Bool -> Assertion -prop_isStopWord a e = assertEqual a e (isStopWord (pack a)) - diff --git a/test-src/Glider/NLP/Language/English/PorterSpec.hs b/test-src/Glider/NLP/Language/English/PorterSpec.hs new file mode 100644 --- /dev/null +++ b/test-src/Glider/NLP/Language/English/PorterSpec.hs @@ -0,0 +1,31 @@ +{-# LANGUAGE OverloadedStrings #-} +{- | +Module : Glider.NLP.Language.English.Porter2Test +Copyright : Copyright (C) 2013-2014 Krzysztof Langner +License : BSD3 + +Maintainer : Krzysztof Langner +Stability : alpha +Portability : portable +-} + +module Glider.NLP.Language.English.PorterSpec (spec) where + +import qualified Data.Text as T +import Test.Hspec +import Glider.NLP.Language.English.Porter + + +spec :: Spec +spec = do + + describe "Porter" $ do + it "consign" $ stem "consign" `shouldBe` "consign" + it "class's" $ stem "class's" `shouldBe` "class'" + it "classes" $ stem "classes" `shouldBe` "class" + it "cried" $ stem "cried" `shouldBe` "cri" + it "ties" $ stem "ties" `shouldBe` "ti" + it "gas" $ stem "gas" `shouldBe` "ga" + it "gaps" $ stem "gaps" `shouldBe` "gap" + it "bleed" $ stem "bleed" `shouldBe` "bleed" + it "guaranteed" $ stem "guaranteed" `shouldBe` "guarante" diff --git a/test-src/Glider/NLP/Language/English/PorterTest.hs b/test-src/Glider/NLP/Language/English/PorterTest.hs deleted file mode 100644 --- a/test-src/Glider/NLP/Language/English/PorterTest.hs +++ /dev/null @@ -1,34 +0,0 @@ -{- | -Module : Glider.NLP.Language.English.Porter2Test -Copyright : Copyright (C) 2013-2014 Krzysztof Langner -License : BSD3 - -Maintainer : Krzysztof Langner -Stability : alpha -Portability : portable --} - -module Glider.NLP.Language.English.PorterTest (testCases) where - -import qualified Data.Text as T -import Glider.NLP.Language.English.Porter -import Test.HUnit - - -testCases :: [(String, Test)] -testCases = [("Porter", t) | t <- tests] - -tests :: [Test] -tests = [ TestCase $ prop_stem "consign" "consign" - , TestCase $ prop_stem "class's" "class'" - , TestCase $ prop_stem "classes" "class" - , TestCase $ prop_stem "cried" "cri" - , TestCase $ prop_stem "ties" "ti" - , TestCase $ prop_stem "gas" "ga" - , TestCase $ prop_stem "gaps" "gap" - , TestCase $ prop_stem "bleed" "bleed" - , TestCase $ prop_stem "guaranteed" "guarante" - ] - -prop_stem :: String -> String -> Assertion -prop_stem a b = assertEqual a (T.pack b) (stem (T.pack a)) diff --git a/test-src/Glider/NLP/StatisticsSpec.hs b/test-src/Glider/NLP/StatisticsSpec.hs new file mode 100644 --- /dev/null +++ b/test-src/Glider/NLP/StatisticsSpec.hs @@ -0,0 +1,28 @@ +{-# LANGUAGE OverloadedStrings #-} +{- | +Module : Glider.NLP.IndexTest +Copyright : Copyright (C) 2013-2016 Krzysztof Langner +License : BSD3 + +Maintainer : Krzysztof Langner +Stability : alpha +Portability : portable +-} + +module Glider.NLP.StatisticsSpec (spec) where + +import qualified Data.Text as T +import Test.Hspec +import Glider.NLP.Statistics + + +spec :: Spec +spec = do + + describe "countWords" $ do + it "text tokens" $ countWords "one two three" `shouldBe` 3 + + describe "wordFreq" $ do + it "single word" $ wordFreq "one" `shouldBe` [("one", 1)] + it "2 words frequency" $ wordFreq "one two" `shouldBe` [("one", 1), ("two", 1)] + it "Multi words frequency" $ wordFreq "one two one" `shouldBe` [("one", 2), ("two", 1)] diff --git a/test-src/Glider/NLP/StatisticsTest.hs b/test-src/Glider/NLP/StatisticsTest.hs deleted file mode 100644 --- a/test-src/Glider/NLP/StatisticsTest.hs +++ /dev/null @@ -1,39 +0,0 @@ -{- | -Module : Glider.NLP.IndexTest -Copyright : Copyright (C) 2013-2014 Krzysztof Langner -License : BSD3 - -Maintainer : Krzysztof Langner -Stability : alpha -Portability : portable --} - -module Glider.NLP.StatisticsTest (testCases) where - -import qualified Data.Text as T -import Glider.NLP.Statistics -import Test.HUnit - - -testCases :: [(String, Test)] -testCases = [ ( "Count words" - , TestCase $ prop_countWords "one two three" 3) - - , ( "Single word frequency" - , TestCase $ prop_wordFreq "one" [("one", 1)]) - - , ( "2 words frequency" - , TestCase $ prop_wordFreq "one two" [("one", 1), ("two", 1)]) - - , ( "Multi words frequency" - , TestCase $ prop_wordFreq "one two one" [("one", 2), ("two", 1)]) - ] - --- | Count number of words -prop_countWords :: String -> Int -> Assertion -prop_countWords xs n = assertEqual xs n $ countWords (T.pack xs) - --- | Check word frequency -prop_wordFreq :: String -> [(String, Int)] -> Assertion -prop_wordFreq xs es = assertEqual xs expected $ wordFreq (T.pack xs) - where expected = [(T.pack a, b) | (a, b) <- es] diff --git a/test-src/Glider/NLP/TokenizerSpec.hs b/test-src/Glider/NLP/TokenizerSpec.hs new file mode 100644 --- /dev/null +++ b/test-src/Glider/NLP/TokenizerSpec.hs @@ -0,0 +1,28 @@ +{-# LANGUAGE OverloadedStrings #-} +{- | +Module : Glider.NLP.TokenizerSpec +Copyright : Copyright (C) 2013-2014 Krzysztof Langner +License : BSD3 + +Maintainer : Krzysztof Langner +Stability : alpha +Portability : portable +-} + +module Glider.NLP.TokenizerSpec (spec) where + +import Glider.NLP.Tokenizer +import Test.Hspec + + +spec :: Spec +spec = describe "Tokenize" $ do + + it "empty string" $ tokenize "" `shouldBe` [] + + it "text strings" $ tokenize "one two three" `shouldBe` [ Word "one" + , Whitespace + , Word "two" + , Whitespace + , Word "three" + ] diff --git a/test-src/Glider/NLP/TokenizerTest.hs b/test-src/Glider/NLP/TokenizerTest.hs deleted file mode 100644 --- a/test-src/Glider/NLP/TokenizerTest.hs +++ /dev/null @@ -1,35 +0,0 @@ -{- | -Module : Glider.NLP.TokenizerTest -Copyright : Copyright (C) 2013-2014 Krzysztof Langner -License : BSD3 - -Maintainer : Krzysztof Langner -Stability : alpha -Portability : portable --} - -module Glider.NLP.TokenizerTest (testCases) where - -import qualified Data.Text as T -import Glider.NLP.Tokenizer -import Test.HUnit - - -testCases :: [(String, Test)] -testCases = [ ( "Empty string" - , TestCase $ prop_tokenize [] "" - ) - - , ( "Empty string" - , TestCase $ prop_tokenize [ Word $ T.pack "one" - , Whitespace - , Word $ T.pack "two" - , Whitespace - , Word $ T.pack "three" - ] - "one two three" - ) - ] - -prop_tokenize :: [Token] -> String -> Assertion -prop_tokenize e a = assertEqual a e (tokenize (T.pack a)) diff --git a/test-src/Spec.hs b/test-src/Spec.hs new file mode 100644 --- /dev/null +++ b/test-src/Spec.hs @@ -0,0 +1,1 @@ +{-# OPTIONS_GHC -F -pgmF hspec-discover #-}