glider-nlp 0.1 → 0.4
raw patch · 14 files changed
+138/−246 lines, 14 filesdep +hspecdep −HUnitdep ~Cabaldep ~basedep ~containers
Dependencies added: hspec
Dependencies removed: HUnit
Dependency ranges changed: Cabal, base, containers, text
Files
- glider-nlp.cabal +16/−16
- src/Glider/NLP/Statistics.hs +4/−3
- src/Glider/NLP/Tokenizer.hs +9/−8
- test-src/AllTests.hs +0/−17
- test-src/Distribution/TestSuite/HUnit.hs +0/−66
- test-src/Glider/NLP/Language/DefaultSpec.hs +21/−0
- test-src/Glider/NLP/Language/DefaultTest.hs +0/−28
- test-src/Glider/NLP/Language/English/PorterSpec.hs +31/−0
- test-src/Glider/NLP/Language/English/PorterTest.hs +0/−34
- test-src/Glider/NLP/StatisticsSpec.hs +28/−0
- test-src/Glider/NLP/StatisticsTest.hs +0/−39
- test-src/Glider/NLP/TokenizerSpec.hs +28/−0
- test-src/Glider/NLP/TokenizerTest.hs +0/−35
- test-src/Spec.hs +1/−0
glider-nlp.cabal view
@@ -1,5 +1,5 @@ name: glider-nlp-version: 0.1+version: 0.4 cabal-version: >=1.10 build-type: Simple author: Krzysztof Langner@@ -28,9 +28,9 @@ default-language: Haskell2010 ghc-options: -Wall build-depends: - base >= 4 && <4.7,- text >=1 && <1.1,- containers >=0.5.0 && <0.6+ base >= 4 && <5,+ text >=1 && <2,+ containers >=0.5 && <1 exposed-modules: Glider.NLP.Language.English.Porter, Glider.NLP.Language.English.StopWords,@@ -40,28 +40,28 @@ other-modules: Glider.NLP.Language.Default test-suite unit-tests- type: detailed-0.9- test-module: AllTests+ type: exitcode-stdio-1.0+ main-is: Spec.hs default-language: Haskell2010 ghc-options: -Wall -rtsopts build-depends: - base >= 4 && <4.7,- HUnit >=1.2.5 && <1.3,- Cabal >=1.16.0 && <1.17,- text >=1 && <1.1,- containers >=0.5.0 && <0.6+ base >= 4 && <5,+ hspec >=2 && <3,+ Cabal >=1 && <2,+ text >=1 && <2,+ containers >=0.5 && <1 hs-source-dirs: src, test-src other-modules: - Distribution.TestSuite.HUnit, Glider.NLP.Language.Default,- Glider.NLP.Language.DefaultTest, Glider.NLP.Language.English.Porter,- Glider.NLP.Language.English.PorterTest, Glider.NLP.Language.English.StopWords, Glider.NLP.Language.Polish.StopWords, Glider.NLP.Statistics,- Glider.NLP.StatisticsTest, Glider.NLP.Tokenizer,- Glider.NLP.TokenizerTest++ Glider.NLP.Language.DefaultSpec,+ Glider.NLP.Language.English.PorterSpec,+ Glider.NLP.StatisticsSpec,+ Glider.NLP.TokenizerSpec
src/Glider/NLP/Statistics.hs view
@@ -1,6 +1,7 @@+{-# LANGUAGE OverloadedStrings #-} {- | Module : Glider.NLP.Statistics-Copyright : Copyright (C) 2013-2014 Krzysztof Langner+Copyright : Copyright (C) 2013-2016 Krzysztof Langner License : BSD3 Maintainer : Krzysztof Langner <klangner@gmail.com>@@ -23,13 +24,13 @@ -- | Count number of words in the text. ----- > countWords (T.pack "one two three") == 3+-- > countWords "one two three" == 3 countWords :: Text -> Int countWords = List.length . getWords . tokenize -- | Count word frequency ----- > wordFreq (T.pack "one two, three one") == [("one", 2), ("two", 1), ("three", 1)]+-- > wordFreq "one two, three one" == [("one", 2), ("two", 1), ("three", 1)] wordFreq :: Text -> [(Text, Int)] wordFreq a = [(List.head xs, List.length xs) | xs <- List.group tokens] where tokens = List.sort $ (foldCase . getWords . tokenize) a
src/Glider/NLP/Tokenizer.hs view
@@ -1,3 +1,4 @@+{-# LANGUAGE OverloadedStrings #-} {- | Module : Glider.NLP.Tokenizer Copyright : Copyright (C) 2013-2014 Krzysztof Langner@@ -8,8 +9,8 @@ Portability : portable This module contains functions which parses text into tokens. -tokens are not normalized. If you need all tokens from the document then -check function "tokenize". If you need only words (na dots, numbers etc.) +Tokens are not normalized. If you need all tokens from the document then+use function "tokenize". If you need only words (no dots, numbers etc.) then check function "getWords". -}@@ -46,16 +47,16 @@ -- > tokenize "one two." == [Word "one", Whitespace, Word "two", "Separator "."] tokenize :: Text -> [Token] tokenize xs = case allParser xs of- [(v, out)] -> (v:tokenize out)- _ -> []+ [(v, out)] -> v : tokenize out+ _ -> [] --- | Exctract all words from tokens+-- | Extract all words from tokens -- -- > getWords "one two." == ["one", "two"] getWords :: [Token] -> [Text] getWords [] = [] getWords (x:xs) = case x of- Word a -> (a: getWords xs)+ Word a -> a: getWords xs _ -> getWords xs -- | Convert all words to the same case@@ -81,13 +82,13 @@ -- | Parse punctuation punctuationParser :: Parser punctuationParser xs | null xs = []- | isPunctuation (head xs) = [(Punctuation (head xs), (tail xs))]+ | isPunctuation (head xs) = [(Punctuation (head xs), tail xs)] | otherwise = [] -- | Parse symbol symbolParser :: Parser symbolParser xs | null xs = []- | isSymbol (head xs) = [(Symbol (head xs), (tail xs))]+ | isSymbol (head xs) = [(Symbol (head xs), tail xs)] | otherwise = [] -- | Parse whitespaces
− test-src/AllTests.hs
@@ -1,17 +0,0 @@-module AllTests (tests) where--import qualified Distribution.TestSuite as C-import qualified Distribution.TestSuite.HUnit as H-import Glider.NLP.TokenizerTest (testCases)-import Glider.NLP.StatisticsTest (testCases)-import Glider.NLP.Language.English.PorterTest (testCases)-import Glider.NLP.Language.DefaultTest (testCases)---tests :: IO [C.Test]-tests = return $ map (uncurry H.test) $ Glider.NLP.TokenizerTest.testCases- ++ Glider.NLP.Language.English.PorterTest.testCases- ++ Glider.NLP.Language.DefaultTest.testCases- ++ Glider.NLP.StatisticsTest.testCases- -
− test-src/Distribution/TestSuite/HUnit.hs
@@ -1,66 +0,0 @@----------------------------------------------------------------------------------- |--- Module : Distribution.TestSuite.HUnit--- Copyright : Patrick Brisbin 2013------ Maintainer : pbrisbin@gmail.com--- Portability : portable------ Test interface for running HUnit tests via cabal.------ Usage:------ > import Test.HUnit--- >--- > import qualified Distribution.TestSuite as C--- > import qualified Distribution.TestSuite.HUnit as H--- >--- > tests :: IO [C.Test]--- > tests = return $ map (uncurry H.test) testCases--- >--- > testCases :: [(String, Test)]--- > testCases = [("Truth test", truthTest)]--- >--- > truthTest :: Test--- > truthTest = assertEqual True True------ An early version of this module, written by Thomas Tuegel--- <ttuegel@gmail.com>, can be found at:------ <http://community.haskell.org/~ttuegel/cabal-test-hunit>------ This file shares no code with that one, as the changes in Cabal 1.16--- required it be entirely rewritten. I've retained Thomas' LICENSE but--- removed his copyright; I hope that's appropriate.------------------------------------------------------------------------------------module Distribution.TestSuite.HUnit (test) where--import Distribution.TestSuite-import qualified Test.HUnit as H--test :: String -> H.Test -> Test-test n = Test . testInstance n--testInstance :: String -> H.Test -> TestInstance-testInstance n t = TestInstance- { run = runTest t- , name = n- , tags = []- , options = []- , setOption = \_ _ -> Right $ testInstance n t- }--runTest :: H.Test -> IO Progress-runTest = fmap snd . H.performTest onStart onError onFailure (Finished Pass)-- where-- onStart :: H.State -> Progress -> IO Progress- onStart _ = return-- onError :: String -> H.State -> Progress -> IO Progress- onError msg _ _ = return $ Finished (Error msg)-- onFailure :: String -> H.State -> Progress -> IO Progress- onFailure msg _ _ = return $ Finished (Fail msg)
+ test-src/Glider/NLP/Language/DefaultSpec.hs view
@@ -0,0 +1,21 @@+{-# LANGUAGE OverloadedStrings #-}+{- |+Module : Glider.NLP.Language.DefaultSpec+Copyright : Copyright (C) 2013-2014 Krzysztof Langner+License : BSD3++Maintainer : Krzysztof Langner <klangner@gmail.com>+Stability : alpha+Portability : portable+-}++module Glider.NLP.Language.DefaultSpec (spec) where++import Test.Hspec+import Glider.NLP.Language.Default+++spec :: Spec+spec = describe "Stop word" $ do+ it "less then 4 chars." $ isStopWord "ala" `shouldBe` True+ it "4 chars." $ isStopWord "abcd" `shouldBe` False
− test-src/Glider/NLP/Language/DefaultTest.hs
@@ -1,28 +0,0 @@-{- |-Module : Glider.NLP.Language.DefaultTest-Copyright : Copyright (C) 2013-2014 Krzysztof Langner-License : BSD3--Maintainer : Krzysztof Langner <klangner@gmail.com>-Stability : alpha-Portability : portable--}--module Glider.NLP.Language.DefaultTest (testCases) where--import Data.Text -import Glider.NLP.Language.Default-import Test.HUnit---testCases :: [(String, Test)]-testCases = [ ( "Less then 4 chars" - , TestCase $ prop_isStopWord "abc" True)- , ( "4 chars not stop word" - , TestCase $ prop_isStopWord "abcd" False)- ]---- | Check empty index -prop_isStopWord :: String -> Bool -> Assertion -prop_isStopWord a e = assertEqual a e (isStopWord (pack a)) -
+ test-src/Glider/NLP/Language/English/PorterSpec.hs view
@@ -0,0 +1,31 @@+{-# LANGUAGE OverloadedStrings #-}+{- |+Module : Glider.NLP.Language.English.Porter2Test+Copyright : Copyright (C) 2013-2014 Krzysztof Langner+License : BSD3++Maintainer : Krzysztof Langner <klangner@gmail.com>+Stability : alpha+Portability : portable+-}++module Glider.NLP.Language.English.PorterSpec (spec) where++import qualified Data.Text as T+import Test.Hspec+import Glider.NLP.Language.English.Porter+++spec :: Spec+spec = do++ describe "Porter" $ do+ it "consign" $ stem "consign" `shouldBe` "consign"+ it "class's" $ stem "class's" `shouldBe` "class'"+ it "classes" $ stem "classes" `shouldBe` "class"+ it "cried" $ stem "cried" `shouldBe` "cri"+ it "ties" $ stem "ties" `shouldBe` "ti"+ it "gas" $ stem "gas" `shouldBe` "ga"+ it "gaps" $ stem "gaps" `shouldBe` "gap"+ it "bleed" $ stem "bleed" `shouldBe` "bleed"+ it "guaranteed" $ stem "guaranteed" `shouldBe` "guarante"
− test-src/Glider/NLP/Language/English/PorterTest.hs
@@ -1,34 +0,0 @@-{- |-Module : Glider.NLP.Language.English.Porter2Test-Copyright : Copyright (C) 2013-2014 Krzysztof Langner-License : BSD3--Maintainer : Krzysztof Langner <klangner@gmail.com>-Stability : alpha-Portability : portable--}--module Glider.NLP.Language.English.PorterTest (testCases) where--import qualified Data.Text as T-import Glider.NLP.Language.English.Porter-import Test.HUnit---testCases :: [(String, Test)]-testCases = [("Porter", t) | t <- tests]--tests :: [Test]-tests = [ TestCase $ prop_stem "consign" "consign"- , TestCase $ prop_stem "class's" "class'"- , TestCase $ prop_stem "classes" "class"- , TestCase $ prop_stem "cried" "cri"- , TestCase $ prop_stem "ties" "ti"- , TestCase $ prop_stem "gas" "ga"- , TestCase $ prop_stem "gaps" "gap"- , TestCase $ prop_stem "bleed" "bleed"- , TestCase $ prop_stem "guaranteed" "guarante"- ]- -prop_stem :: String -> String -> Assertion -prop_stem a b = assertEqual a (T.pack b) (stem (T.pack a))
+ test-src/Glider/NLP/StatisticsSpec.hs view
@@ -0,0 +1,28 @@+{-# LANGUAGE OverloadedStrings #-}+{- |+Module : Glider.NLP.IndexTest+Copyright : Copyright (C) 2013-2016 Krzysztof Langner+License : BSD3++Maintainer : Krzysztof Langner <klangner@gmail.com>+Stability : alpha+Portability : portable+-}++module Glider.NLP.StatisticsSpec (spec) where++import qualified Data.Text as T+import Test.Hspec+import Glider.NLP.Statistics+++spec :: Spec+spec = do++ describe "countWords" $ do+ it "text tokens" $ countWords "one two three" `shouldBe` 3++ describe "wordFreq" $ do+ it "single word" $ wordFreq "one" `shouldBe` [("one", 1)]+ it "2 words frequency" $ wordFreq "one two" `shouldBe` [("one", 1), ("two", 1)]+ it "Multi words frequency" $ wordFreq "one two one" `shouldBe` [("one", 2), ("two", 1)]
− test-src/Glider/NLP/StatisticsTest.hs
@@ -1,39 +0,0 @@-{- |-Module : Glider.NLP.IndexTest-Copyright : Copyright (C) 2013-2014 Krzysztof Langner-License : BSD3--Maintainer : Krzysztof Langner <klangner@gmail.com>-Stability : alpha-Portability : portable--}--module Glider.NLP.StatisticsTest (testCases) where--import qualified Data.Text as T-import Glider.NLP.Statistics-import Test.HUnit---testCases :: [(String, Test)]-testCases = [ ( "Count words"- , TestCase $ prop_countWords "one two three" 3)- - , ( "Single word frequency"- , TestCase $ prop_wordFreq "one" [("one", 1)])- - , ( "2 words frequency"- , TestCase $ prop_wordFreq "one two" [("one", 1), ("two", 1)])- - , ( "Multi words frequency"- , TestCase $ prop_wordFreq "one two one" [("one", 2), ("two", 1)])- ]- --- | Count number of words -prop_countWords :: String -> Int -> Assertion -prop_countWords xs n = assertEqual xs n $ countWords (T.pack xs)---- | Check word frequency -prop_wordFreq :: String -> [(String, Int)] -> Assertion -prop_wordFreq xs es = assertEqual xs expected $ wordFreq (T.pack xs)- where expected = [(T.pack a, b) | (a, b) <- es]
+ test-src/Glider/NLP/TokenizerSpec.hs view
@@ -0,0 +1,28 @@+{-# LANGUAGE OverloadedStrings #-}+{- |+Module : Glider.NLP.TokenizerSpec+Copyright : Copyright (C) 2013-2014 Krzysztof Langner+License : BSD3++Maintainer : Krzysztof Langner <klangner@gmail.com>+Stability : alpha+Portability : portable+-}++module Glider.NLP.TokenizerSpec (spec) where++import Glider.NLP.Tokenizer+import Test.Hspec+++spec :: Spec+spec = describe "Tokenize" $ do++ it "empty string" $ tokenize "" `shouldBe` []++ it "text strings" $ tokenize "one two three" `shouldBe` [ Word "one"+ , Whitespace+ , Word "two"+ , Whitespace+ , Word "three"+ ]
− test-src/Glider/NLP/TokenizerTest.hs
@@ -1,35 +0,0 @@-{- |-Module : Glider.NLP.TokenizerTest-Copyright : Copyright (C) 2013-2014 Krzysztof Langner-License : BSD3--Maintainer : Krzysztof Langner <klangner@gmail.com>-Stability : alpha-Portability : portable--}--module Glider.NLP.TokenizerTest (testCases) where--import qualified Data.Text as T-import Glider.NLP.Tokenizer-import Test.HUnit---testCases :: [(String, Test)]-testCases = [ ( "Empty string"- , TestCase $ prop_tokenize [] ""- )- - , ( "Empty string"- , TestCase $ prop_tokenize [ Word $ T.pack "one"- , Whitespace- , Word $ T.pack "two"- , Whitespace- , Word $ T.pack "three"- ] - "one two three"- ) - ]- -prop_tokenize :: [Token] -> String -> Assertion -prop_tokenize e a = assertEqual a e (tokenize (T.pack a))
+ test-src/Spec.hs view
@@ -0,0 +1,1 @@+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}