fullstop 0.1.3.2 → 0.1.4
raw patch · 4 files changed
+81/−24 lines, 4 filesdep ~QuickChecksetup-changed
Dependency ranges changed: QuickCheck
Files
- NLP/FullStop.hs +3/−7
- NLP/FullStop/Ignore.hs +72/−0
- Setup.hs +1/−13
- fullstop.cabal +5/−4
NLP/FullStop.hs view
@@ -26,6 +26,8 @@ import Data.List import Data.List.Split +import NLP.FullStop.Ignore ( titles, abbreviations, initials )+ -- ------------------------------------------------------------ -- -- ------------------------------------------------------------@@ -66,7 +68,7 @@ squish = squishBy (\x _ -> any (`isWordSuffixOf` x) abbreviations) . squishBy (\_ y -> not (startsWithSpace y)) . squishBy (\x _ -> looksLikeAnInitial (dropWhile isSpace x))- . squishBy (\x _ -> any (`isWordSuffixOf` x) titles)+ . squishBy (\x _ -> any (`isWordSuffixOf` x) (titles ++ initials)) . squishBy (\x y -> endsWithDigit x && startsWithDigit y) where looksLikeAnInitial x =@@ -99,12 +101,6 @@ then helper (x0:acc) x1 xs else assemble acc x0 : helper [] x1 xs assemble acc x0 = reverse (x0 : acc)--titles :: [String]-titles = [ "Mr.", "Mrs.", "Dr.", "St." ]--abbreviations :: [String]-abbreviations = [ "cf.", "eg.", "ie.", "i.e.", "e.g." ] x `isWordSuffixOf` y | x `isSuffixOf` y = case drop (length x) (reverse y) of
+ NLP/FullStop/Ignore.hs view
@@ -0,0 +1,72 @@+-- | Strings whose internal "." should not be treated as sentence boundaries++module NLP.FullStop.Ignore where++titles :: [String]+titles =+ [ "Adj."+ , "Adm."+ , "Adv."+ , "Asst."+ , "Bart."+ , "Bldg."+ , "Brig."+ , "Bros."+ , "Capt."+ , "Cmdr."+ , "Col."+ , "Comdr."+ , "Con."+ , "Cpl."+ , "DR."+ , "Dr."+ , "Ens."+ , "Gen."+ , "Gov."+ , "Hon."+ , "Hosp."+ , "Insp."+ , "Lt."+ , "MM."+ , "MR."+ , "MRS."+ , "MS."+ , "Maj."+ , "Messrs."+ , "Mlle."+ , "Mme."+ , "Mr."+ , "Mrs."+ , "Ms."+ , "Msgr."+ , "Op."+ , "Ord."+ , "Pfc."+ , "Ph."+ , "Prof."+ , "Pvt."+ , "Rep."+ , "Reps."+ , "Rev."+ , "Sen."+ , "Sens."+ , "Sfc."+ , "Sgt."+ , "Sr."+ , "St."+ , "Supt."+ ]++abbreviations :: [String]+abbreviations =+ [ "cf."+ , "eg."+ , "ie."+ , "i.e."+ , "e.g."+ , "v."+ , "vs."+ ]++initials :: [String]+initials = map (: ".") ['A'..'Z']
Setup.hs view
@@ -1,14 +1,2 @@-import Distribution.PackageDescription import Distribution.Simple-import Distribution.Simple.LocalBuildInfo-import Distribution.Simple.Setup--import System.Cmd-import System.FilePath--main = defaultMainWithHooks hooks- where hooks = simpleUserHooks { runTests = runTests' }--runTests' :: Args -> Bool -> PackageDescription -> LocalBuildInfo -> IO ()-runTests' _ _ _ lbi = system testprog >> return ()- where testprog = (buildDir lbi) </> "hstest-fullstop" </> "hstest-fullstop"+main = defaultMain
fullstop.cabal view
@@ -1,5 +1,5 @@ name: fullstop-version: 0.1.3.2+version: 0.1.4 synopsis: Simple sentence segmenter description: FullStop splits texts into sentences, using some orthographical conventions (used in English and hopefully other languages).@@ -17,10 +17,10 @@ license: BSD3 license-file: LICENSE author: Eric Kow-maintainer: Eric Kow <kowey@comp-ling.com>+maintainer: Eric Kow <eric@erickow.com> homepage: http://hub.darcs.net/kowey/fullstop cabal-version: >= 1.10-build-type: Custom+build-type: Simple source-repository head type: darcs@@ -28,6 +28,7 @@ library exposed-modules: NLP.FullStop+ , NLP.FullStop.Ignore -- NB: there actualy isn't any particular need for GHC 6.10 build-Depends: base >= 3 && < 5@@ -50,7 +51,7 @@ build-depends: base , fullstop , HUnit == 1.2.*- , QuickCheck == 2.4.*+ , QuickCheck == 2.* , tasty , tasty-hunit , tasty-quickcheck