punkt 0.1.0 → 0.1.1

raw patch · 3 files changed

+39/−13 lines, 3 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• lib/NLP/Punkt.hs +37/−11
• lib/NLP/Punkt/Match.hs +1/−1
• punkt.cabal +1/−1

@@ -12,20 +12,30 @@ import Control.Applicative ((<$>), (<*>), (<|>)) import qualified Control.Monad.Reader as Reader -import NLP.Punkt.Match (re_split, re_split_pos, intrasep, word_seps)+import NLP.Punkt.Match (re_split_pos, word_seps) +-- | Carries various orthographic statistics for a particular textual type. data OrthoFreq = OrthoFreq {     freq_lower :: Int,+    -- ^ number of lowercase occurrences     freq_upper :: Int,+    -- ^ uppercase occurrences     freq_first_lower :: Int,+    -- ^ number of lowercase occurrences in the first position of a sentence     freq_internal_upper :: Int,+    -- ^ number of uppercase occurrences strictly internal to a sentence     freq_after_ender :: Int+    -- ^ number of occurences in the first position     }     deriving Show +-- | Represents training data obtained from a corpus required by Punkt. data PunktData = PunktData {-    type_count :: HashMap Text Int,  -- abbreviation counter+    type_count :: HashMap Text Int,+    -- ^ Occurrences of each textual type, case-insensitive. Used during Punkt's+    -- type-based stage. Also contains occurrences of trailing periods.     ortho_count :: HashMap Text OrthoFreq,+    -- ^ Dictionary of orthographic data for each textual type.     collocations :: HashMap (Text, Text) Int,     total_enders :: Int,     total_toks :: Int@@ -57,7 +67,7 @@ is_word :: Token -> Bool is_word tok = case entity tok of { Word _ _ -> True; _ -> False; } --- dunning log likelihood modified by kiss/strunk+-- | Dunning log likelihood modified by Kiss/Strunk strunk_log :: Double -> Double -> Double -> Double -> Double strunk_log a b ab n = -2 * (null - alt)     where@@ -65,7 +75,7 @@     alt = ab * log p2 + (a - ab) * log (1 - p2)     (p1, p2) = (b / n, 0.99) --- vanilla dunning log likelihood+-- | Dunning's original log likelihood dunning_log :: Double -> Double -> Double -> Double -> Double dunning_log a b ab n | b == 0 || ab == 0 = 0                      | otherwise = -2 * (s1 + s2 - s3 - s4)@@ -77,8 +87,13 @@     s4 = if b == ab then 0 else         (b - ab) * log p2 + (n - a - b + ab) * log (1 - p2) +ask_type_count :: Punkt (HashMap Text Int) ask_type_count = Reader.liftM type_count Reader.ask++ask_total_toks :: Num a => Punkt a ask_total_toks = Reader.liftM (fromIntegral . total_toks) Reader.ask++ask_total_enders :: Num a => Punkt a ask_total_enders = Reader.liftM (fromIntegral . total_enders) Reader.ask  ask_ortho :: Text -> Punkt OrthoFreq@@ -90,24 +105,27 @@     return . fromIntegral . Map.lookupDefault 0 (norm w0_, norm w1_)     =<< collocations <$> Reader.ask --- c(w, ~.)+-- | Occurrences of a textual type, strictly ignoring trailing period.+-- @c(w, ~.)@. Case-insensitive. freq :: Text -> Punkt Double freq w_ = ask_type_count >>= return . fromIntegral . Map.lookupDefault 0 w     where w = norm w_ --- c(w, .)+-- | Occurrences of a textual type with trailing period. @c(w, .)@.+-- Case-insensitive. freq_snoc_dot :: Text -> Punkt Double freq_snoc_dot w_ = freq wdot where wdot = w_ `Text.snoc` '.' -- potential slowdown if ghc doesn't know that norm "." == "." --- c(w) == c(w, .) + c(w, ~.)+-- | @c(w) == c(w, .) + c(w, ~.)@. Case-insensitive. freq_type :: Text -> Punkt Double freq_type w_ = (+) <$> freq w_ <*> freq_snoc_dot w_  dlen :: Text -> Double dlen = fromIntegral . Text.length --- probability that (w_ `snoc` '.') is an abbreviation.+-- | Returns the log likelihood that (w_ `snoc` '.') is an abbreviation.+-- Case-insensitive. prob_abbr :: Text -> Punkt Double prob_abbr w_ = compensate =<< strunk_log <$> freq_type w_ <*> freq "."                                          <*> freq_snoc_dot w_ <*> ask_total_toks@@ -120,7 +138,8 @@     f_len = 1 / exp (dlen $ Text.filter (/= '.') w_)     f_periods = 1 + dlen (Text.filter (== '.') w_) --- decides if w is a sentence ender based on its capitalization+-- | Decides if @w@ is a sentence ender based on its capitalization.+-- Case-insensitive. decide_ortho :: Text -> Punkt (Maybe Bool) decide_ortho w_ = ask_ortho w_ >>= return . decide' w_     where@@ -135,23 +154,26 @@         ever_title = freq_upper wortho > 0         never_lower_start = freq_first_lower wortho == 0 --- special orthographic heuristic for post-possible-initial tokens.+-- | Special orthographic heuristic for post-possible-initial tokens.+-- Case-insensitive. decide_initial_ortho :: Text -> Punkt (Maybe Bool) decide_initial_ortho w_ = do     neverlower <- (== 0) . freq_lower <$> ask_ortho w_     orthosays <- decide_ortho w_     return $ orthosays <|> if neverlower then Just False else Nothing --- probability that w_ is a frequent sentence starter+-- | Log likelihood that @w@ is a frequent sentence starter. Case-insensitive. prob_starter :: Text -> Punkt Double prob_starter w_ = dunning_log <$> ask_total_enders <*> freq_type w_                               <*> fafterend <*> ask_total_toks     where fafterend = fromIntegral . freq_after_ender <$> ask_ortho w_ +-- | Computes the collocational likelihood of @w@ and @x@. Case-insensitive. prob_colloc :: Text -> Text -> Punkt Double prob_colloc w_ x_ = dunning_log <$> freq_type w_ <*> freq_type x_                                 <*> ask_colloc w_ x_ <*> ask_total_toks +-- | Builds a dictionary of textual type frequencies from a stream of tokens. build_type_count :: [Token] -> HashMap Text Int build_type_count = List.foldl' update initcount     where@@ -286,9 +308,13 @@         (spaces, _) -> Just (p, Text.length spaces + p)     where notSpace = not . isSpace +-- | Main export of the entire package. Splits a corpus into its constituent+-- sentences. split_sentences :: Text -> [Text] split_sentences corpus = map (uncurry $ substring corpus) slices     where slices = find_breaks corpus +-- | @runPunkt data computation@ runs @computation@ using @data@ collected from+-- a corpus using 'build_punkt_data'. runPunkt :: PunktData -> Punkt a -> a runPunkt = flip Reader.runReader

@@ -15,7 +15,6 @@ import "regex-tdfa-text" Text.Regex.TDFA.Text (compile) import "regex-tdfa" Text.Regex.TDFA (Regex, matchOnceText, blankCompOpt,                                      ExecOption(..))-import Data.Maybe (maybe) import Data.Either (lefts)  re_split_impl :: Regex -> Text -> [Either Text Text]@@ -43,5 +42,6 @@ re_compile :: Text -> Regex re_compile re = rv where Right rv = compile blankCompOpt (ExecOption False) re +word_seps, intrasep :: Regex word_seps = re_compile "([ \t\n]+|-{2,}|—|\\.{2,}|\\.( \\.)+|…|[!\\?;:]{1,})" intrasep = re_compile "[-'’]"

@@ -1,5 +1,5 @@ name: punkt-version: 0.1.0+version: 0.1.1 synopsis: Multilingual unsupervised sentence tokenization with Punkt. description: Multilingual unsupervised sentence tokenization with Punkt. license: MIT