fullstop 0.1 → 0.1.1

raw patch · 3 files changed

+52/−16 lines, 3 filesdep ~basenew-component:exe:fullstopPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: base

API changes (from Hackage documentation)

Files

• NLP/FullStop.hs +45/−14
• fullstop.cabal +4/−2
• fullstop.hs +3/−0

@@ -46,10 +46,14 @@      , testCaseSegments "condense"  ["What?!", "Yeah"]   "What?! Yeah"      , testCaseSegments "URLs"    ["Check out http://www.example.com.", "OK?"]                                    "Check out http://www.example.com. OK?"-     , testCaseNoSplit "titles"    "Mr. Doe, Mrs. Durand and Dr. Singh"+     , testCaseNoSplit "titles"    "Mr. Doe, Mrs. Durand, St. Orolo and Dr. Singh"+     , testCaseNoSplit "abbreviations"   "e.g., or eg., i.e. or ie. should not be split"+     , testCaseSegments "abbreviations 2"   ["No lie.", "Honestly"] "No lie. Honestly"      , testCaseNoSplit "initials"  "E. Y. Kow"+     , testCaseNoSplit "initials 2"  "Hello, E. Y. Kow"+     , testCaseSegments "initials counter" [ "E. Y. Kow.", "Hello" ] "E. Y. Kow. Hello"      , testCaseNoSplit "numbers"   "version 2.3.99.2"-     -- TODO: what's a good way of dealing with ellipsis?+     -- TODO: what's a good way of dealing with ellipsis (...)?      -- TODO: He said "Foo." Bar (tricky because Foo. "Bar" is legit)      -- TODO: Very likely to be cases where it's just plain ambiguous      ]@@ -87,31 +91,34 @@ --   segmentation right, maybe implement something which returns --   the N most probable segmentations instead. segment :: String -> [String]-segment = map (dropWhile isSpace) . squish-        . ( split+segment = map (dropWhile isSpace) . squish . breakup++-- | Helper function to segment+breakup :: String -> [String]+breakup = split           . condense       -- "huh?!"           . dropFinalBlank -- strings that end with terminator           . keepDelimsR    -- we want to preserve terminators           $ oneOf stopPunctuation-          )  stopPunctuation :: [Char] stopPunctuation = [ '.', '?', '!' ] -titles :: [String]-titles = [ "Mr.", "Mrs.", "Dr." ]- -- ------------------------------------------------------------ -- putting some pieces back together -- ------------------------------------------------------------ -squish = squishBy (\_ y -> not (startsWithSpace y))-       . squishBy (\x _ -> looksLikeAnInitial x)-       . squishBy (\x _ -> any (`isSuffixOf` x) titles)+squish = squishBy (\x _ -> any (`isWordSuffixOf` x) abbreviations)+       . squishBy (\_ y -> not (startsWithSpace y))+       . squishBy (\x _ -> looksLikeAnInitial (dropWhile isSpace x))+       . squishBy (\x _ -> any (`isWordSuffixOf` x) titles)        . squishBy (\x y -> endsWithDigit x  && startsWithDigit y)  where-  looksLikeAnInitial [_,'.'] = True-  looksLikeAnInitial _ = False+  looksLikeAnInitial x =+   case reverse x of+    ('.':i:[])  -> isUpper i+    ('.':i:s:_) -> isSpace s && isUpper i+    _ -> False   --   startsW f [] = False   startsW f (x:_) = f x@@ -124,4 +131,28 @@      ('.':x:_) -> isDigit x      _ -> False -squishBy f = map concat . groupBy f+-- | This is *not* (map concat . groupBy f) because the latter+--   just checks equality on the first element of each group.+--   We, on the other hand, want to check by the nearest neighbour+squishBy :: (String -> String-> Bool) -> [String] -> [String]+squishBy _ []     = []+squishBy eq (x:xs) = map concat (helper [] x xs)+ where+  helper acc x0 [] = [assemble acc x0]+  helper acc x0 (x1:xs) =+   if x0 `eq` x1+      then helper (x0:acc) x1 xs+      else assemble acc x0 : helper [] x1 xs+  assemble acc x0 = reverse (x0 : acc)++titles :: [String]+titles = [ "Mr.", "Mrs.", "Dr.", "St." ]++abbreviations :: [String]+abbreviations = [ "cf.", "eg.", "ie.", "i.e.", "e.g." ]++x  `isWordSuffixOf` y | x `isSuffixOf` y =+ case drop (length x) (reverse y) of+   []    -> True -- x == y+   (z:_) -> not (isAlpha z)+x  `isWordSuffixOf` y = False

@@ -1,5 +1,5 @@ name:                fullstop-version:             0.1+version:             0.1.1 synopsis:            Simple sentence segmenter description:         FullStop splits texts into sentences, using some orthographical                      conventions (used in English and hopefully other languages).@@ -26,7 +26,7 @@     exposed-modules:  NLP.FullStop      -- NB: there actualy isn't any particular need for GHC 6.10-    build-Depends: base >= 3 && < 4.1+    build-Depends: base >= 3 && < 4.3                  , HUnit == 1.2.*                  , QuickCheck == 2.1.*                  , test-framework       == 0.2.*@@ -34,6 +34,8 @@                  , test-framework-quickcheck2 == 0.2.*                  , split == 0.1.* +executable fullstop+    main-is:  fullstop.hs  executable hstest-fullstop     main-is:  Tests.hs

@@ -0,0 +1,3 @@+import NLP.FullStop++main = interact (unlines . segment)