alpino-tools 0.0.2 → 0.0.4
raw patch · 4 files changed
+21/−49 lines, 4 filesdep ~enumeratorPVP: major bump suggested
API removals or changes: PVP suggests a major version bump
Dependency ranges changed: enumerator
API changes (from Hackage documentation)
- Data.Alpino.Model.Enumerator: filter :: Monad m => (a -> Bool) -> Enumeratee a a m b
Files
- alpino-tools.cabal +4/−5
- src/Data/Alpino/Model/Enumerator.hs +5/−28
- src/model_oracle.hs +5/−7
- src/model_statistics_data.hs +7/−9
alpino-tools.cabal view
@@ -1,5 +1,5 @@ Name: alpino-tools-Version: 0.0.2+Version: 0.0.4 License: OtherLicense License-file: LICENSE Copyright: Copyright 2010 Daniël de Kok@@ -7,9 +7,8 @@ Maintainer: Daniël de Kok <me@danieldk.eu> Homepage: http://github.com/danieldk/alpino-tools Category: Natural Language Processing, Data-Synopsis: Alpino data processing tools-Description: Tools for processing data of the Alpino parser/generator- for Dutch.+Synopsis: Alpino data manipulation tools+Description: Tools for manipulating data for the Alpino parser for Dutch. Cabal-Version: >= 1.2 Build-Type: Simple @@ -17,7 +16,7 @@ Exposed-Modules: Data.Alpino.Model, Data.Alpino.Model.Enumerator Build-Depends: base >= 4 && < 5, bytestring >= 0.9.1.7, utf8-string >= 0.3.6, bytestring-lexing >= 0.2.1,- enumerator >= 0.4.1, transformers >= 0.2.2.0,+ enumerator >= 0.4.5 && < 0.5, transformers >= 0.2.2.0, containers >= 0.3.0.0, random >= 1.0.0.3, random-shuffle >= 0.0.2 HS-Source-Dirs: src
src/Data/Alpino/Model/Enumerator.hs view
@@ -13,7 +13,6 @@ concat, groupBy, groupByKey,- filter, filterFeatures, filterFeaturesFunctor, instanceGenerator,@@ -26,17 +25,16 @@ scoreToNorm ) where -import Prelude hiding (concat, filter, head, mapM)+import Prelude hiding (concat, filter, mapM) import Control.Exception.Base (Exception)-import qualified Control.Monad as CM import Control.Monad.IO.Class (MonadIO(..), liftIO) import Control.Monad.Trans.Class (lift) import qualified Data.Alpino.Model as AM import qualified Data.ByteString as B import qualified Data.ByteString.UTF8 as BU import qualified Data.Enumerator as E-import Data.Enumerator hiding (isEOF, length, map)-import qualified Data.List as L+import Data.Enumerator hiding (isEOF, head, length, map)+import qualified Data.Enumerator.List as EL import qualified Data.Set as Set import Data.Typeable import System.IO (isEOF)@@ -79,7 +77,7 @@ case h of Nothing -> return $ Continue k Just e -> do- xs <- E.span $ f e+ xs <- EL.takeWhile $ f e newStep <- lift $ runIteratee $ k $ Chunks [xs] loop newStep loop step = return step@@ -114,39 +112,18 @@ runIteratee (k (Chunks [line])) >>= loop loop step = return step --- | Enumeratee that filters with a predicate.-filter :: (Monad m) => (a -> Bool) -> Enumeratee a a m b-filter f = loop- where loop = checkDone $ continue . step- step k EOF = yield (Continue k) EOF- step k (Chunks []) = continue $ step k- step k (Chunks xs) = do- newStep <- lift $ runIteratee $ k $ Chunks $ L.filter f xs- loop newStep- -- | Enumeratee concatenating lists. concat :: (Monad m) => Enumeratee [a] a m b concat = loop where loop (Continue k) = do- h <- E.head+ h <- EL.head case h of Nothing -> return $ Continue k Just e -> do newStep <- lift $ runIteratee $ k $ Chunks e loop newStep loop step = return step---mapM :: Monad m => (ao -> m ai) -> Enumeratee ao ai m b-mapM f = loop where- loop = checkDone $ continue . step- step k EOF = yield (Continue k) EOF- step k (Chunks []) = continue $ step k- step k (Chunks xs) = ( do- ys <- lift $ CM.mapM f xs- k $ Chunks ys- ) >>== loop mapMaybeEnum :: (Exception e, Monad m) => e -> (ao -> Maybe ai) -> Enumeratee ao ai m b
src/model_oracle.hs view
@@ -10,13 +10,11 @@ import Text.Printf (printf) sumCount :: (Monad m, Fractional a, Integral b) => Iteratee a m (a, b)-sumCount = liftI $ step (0.0, 0)- where step acc@(!sumAcc, !lenAcc) chunk =- case chunk of- Chunks [] -> Continue $ returnI . step acc- Chunks xs -> Continue $ returnI . (step $- (sumAcc + sum xs, lenAcc + genericLength xs))- EOF -> Yield acc EOF+sumCount = continue $ step (0.0, 0)+ where step acc (Chunks []) = continue $ step acc+ step (!sumAcc, !lenAcc) (Chunks xs) =+ continue $ step (sumAcc + sum xs, lenAcc + genericLength xs)+ step acc EOF = yield acc EOF main :: IO () main = do
src/model_statistics_data.hs view
@@ -11,15 +11,13 @@ import Text.Printf (printf) statistics :: Monad m => Iteratee [AM.TrainingInstance] m (Int, Int, Int)-statistics = liftI $ step (0, 0, 0)- where step acc@(!lenSumAcc, !lenAcc, !maxLenAcc) chunk =- case chunk of- Chunks [] -> Continue $ returnI . step acc- Chunks xs -> Continue $ returnI . (step $- (lenSumAcc + (sum $ map length xs),- lenAcc + genericLength xs,- max maxLenAcc $ maximum $ map length xs))- EOF -> Yield acc EOF+statistics = continue $ step (0, 0, 0)+ where step acc (Chunks []) = continue $ step acc+ step (!lenSumAcc, !lenAcc, !maxLenAcc) (Chunks xs) =+ continue $ (step $ (lenSumAcc + (sum $ map length xs),+ lenAcc + genericLength xs,+ max maxLenAcc $ maximum $ map length xs))+ step acc EOF = yield acc EOF main :: IO () main = do