alpino-tools 0.1.0 → 0.2.0

raw patch · 14 files changed

+390/−427 lines, 14 filesdep +MonadRandomdep +alpino-toolsdep +conduitdep −enumeratordep −randomdep −transformersdep ~bytestringdep ~bytestring-lexingdep ~containersPVP ok

version bump matches the API change (PVP)

Dependencies added: MonadRandom, alpino-tools, conduit, resourcet

Dependencies removed: enumerator, random, transformers

Dependency ranges changed: bytestring, bytestring-lexing, containers, hexpat-pickle, mtl, random-shuffle, rosezipper, utf8-string

API changes (from Hackage documentation)

Files

• alpino-tools.cabal +40/−31
• src/Data/Alpino/Model.hs +12/−12
• src/Data/Alpino/Model/Conduit.hs +115/−0
• src/Data/Alpino/Model/Enumerator.hs +0/−180
• src/model_filter_data.hs +0/−61
• src/model_oracle.hs +0/−26
• src/model_random_sample.hs +0/−41
• src/model_rescore_data.hs +0/−47
• src/model_statistics_data.hs +0/−29
• utils/model_filter_data.hs +62/−0
• utils/model_oracle.hs +28/−0
• utils/model_random_sample.hs +51/−0
• utils/model_rescore_data.hs +48/−0
• utils/model_statistics_data.hs +34/−0

@@ -1,51 +1,60 @@-Name:		alpino-tools-Version:	0.1.0-License:	OtherLicense-License-file:	LICENSE-Copyright:	Copyright 2010 Daniël de Kok-Author:		Daniël de Kok <me@danieldk.eu>-Maintainer:	Daniël de Kok <me@danieldk.eu>-Homepage:	http://github.com/danieldk/alpino-tools-Category:	Natural Language Processing, Data-Synopsis:	Alpino data manipulation tools-Description:	Tools for manipulating data for the Alpino parser for Dutch.-Cabal-Version:	>= 1.2-Build-Type:	Simple+Name:          alpino-tools+Version:       0.2.0+License:       OtherLicense+License-file:  LICENSE+Copyright:     Copyright 2010-2012 Daniël de Kok+Author:        Daniël de Kok <me@danieldk.eu>+Maintainer:    Daniël de Kok <me@danieldk.eu>+Homepage:      http://github.com/danieldk/alpino-tools+Category:      Natural Language Processing, Data+Synopsis:      Alpino data manipulation tools+Description:   Tools for manipulating data for the Alpino parser for Dutch.+Cabal-Version: >= 1.8+Build-Type:    Simple  Library   Exposed-Modules:      Data.Alpino.DepStruct, Data.Alpino.DepStruct.Pickle,                         Data.Alpino.DepStruct.Triples, Data.Alpino.Model,-                        Data.Alpino.Model.Enumerator-  Build-Depends:	base >= 4 && < 5, bytestring >= 0.9.1.7,-                        utf8-string >= 0.3.6, bytestring-lexing >= 0.2.1,-                        enumerator >= 0.4.8 && < 0.5, transformers >= 0.2.2.0,-                        containers >= 0.3.0.0, random >= 1.0.0.3,-                        random-shuffle >= 0.0.2, hexpat-pickle >= 0.4,-                        rosezipper >= 0.2, mtl >= 2.0.1.0+                        Data.Alpino.Model.Conduit+  Build-Depends:        base >= 4 && < 5, bytestring == 0.9.2.*,+                        containers == 0.4.*, utf8-string == 0.3.*,+                        bytestring-lexing == 0.4.*,+                        conduit == 0.4.*, MonadRandom == 0.1.*,+                        random-shuffle == 0.0.3, hexpat-pickle == 0.5,+                        resourcet == 0.3.*, rosezipper == 0.2,+                        mtl == 2.0.1.*   HS-Source-Dirs:       src-  Ghc-Options:		-O2 -Wall+  Ghc-Options:          -O2 -Wall  Executable at_model_rescore_data-  HS-Source-Dirs:       src+  HS-Source-Dirs:       utils   main-is:              model_rescore_data.hs-  Ghc-Options:		-O2+  Build-Depends:        base >= 4 && < 5, alpino-tools, conduit == 0.4.*+  Ghc-Options:          -O2 -Wall  Executable at_model_filter_data-  HS-Source-Dirs:       src+  HS-Source-Dirs:       utils   main-is:              model_filter_data.hs-  Ghc-Options:		-O2+  Build-Depends:        base >= 4 && < 5, alpino-tools, conduit == 0.4.*,+                        containers == 0.4.*, utf8-string == 0.3.*+  Ghc-Options:          -O2 -Wall  Executable at_model_oracle-  HS-Source-Dirs:       src+  HS-Source-Dirs:       utils   main-is:              model_oracle.hs-  Ghc-Options:		-O2+  Build-Depends:        base >= 4 && < 5, alpino-tools, conduit == 0.4.*+  Ghc-Options:          -O2 -Wall  Executable at_model_random_sample-  HS-Source-Dirs:       src+  HS-Source-Dirs:       utils   main-is:              model_random_sample.hs-  Ghc-Options:		-O2+  Build-Depends:        base >= 4 && < 5, alpino-tools, conduit == 0.4.*,+                        resourcet == 0.3.*, mtl == 2.0.1.*,+                        MonadRandom == 0.1.*+  Ghc-Options:          -O2 -Wall -fno-warn-orphans  Executable at_model_statistics_data-  HS-Source-Dirs:       src+  HS-Source-Dirs:       utils   main-is:              model_statistics_data.hs-  Ghc-Options:		-O2+  Build-Depends:        base >= 4 && < 5, alpino-tools, conduit == 0.4.*+  Ghc-Options:          -O2 -Wall

@@ -29,6 +29,8 @@                            trainingInstanceToBs                          ) where +import Control.Monad (liftM)+import Control.Monad.Random.Class (MonadRandom) import qualified Data.ByteString as B import Data.ByteString.Internal (c2w) import Data.ByteString.Lex.Double (readDouble)@@ -37,8 +39,7 @@ import Data.Maybe (fromJust) import qualified Data.Set as Set import GHC.Word (Word8)-import System.Random (RandomGen)-import System.Random.Shuffle (shuffle')+import System.Random.Shuffle (shuffleM) import Text.Printf (printf)  -- | A training instance.@@ -68,11 +69,11 @@  -- | Find the highest score of a context. bestScore :: [TrainingInstance] -> Double-bestScore = foldl (\acc e -> max acc $ instanceScore e) 0.0+bestScore = foldl (\acc -> max acc . instanceScore) 0.0  -- | Find the highest score of a context (strict). bestScore' :: [TrainingInstance] -> Double-bestScore' = foldl' (\acc e -> max acc $ instanceScore e) 0.0+bestScore' = foldl' (\acc -> max acc . instanceScore) 0.0  -- | -- Read a training instance from a `BU.ByteString`.@@ -157,7 +158,7 @@ filterFeatures f keepFeatures i =     i { instanceFeatures = FeaturesList $ filter keep $                    parsedFeatures $ instanceFeatures i}-    where keep fv = f $ Set.member (feature fv) keepFeatures+    where keep = f . flip Set.member keepFeatures . feature  -- | -- Filter features by their functor. A modifier function can be applied,@@ -167,17 +168,16 @@ filterFeaturesFunctor f keepFeatures i =     i { instanceFeatures = FeaturesList $ filter keep $ parsedFeatures $                    instanceFeatures i}-    where keep fv = f $ Set.member (functor $ feature fv) keepFeatures+    where keep = f . flip Set.member keepFeatures . functor . feature           functor func = B.split argOpen func !! 0           argOpen = c2w '('  -- | Extract a random sample from a list of instances.-randomSample :: RandomGen g => g -> Int -> [TrainingInstance] ->-                [TrainingInstance]-randomSample g n i-    | instLen <= n = i-    | otherwise = take n $ shuffle' i instLen g-    where instLen = length i+randomSample :: MonadRandom m => Int -> [TrainingInstance] ->+  m [TrainingInstance]+randomSample n i+  | length i <= n = return $ i+  | otherwise     = take n `liftM` shuffleM i  -- | -- Convert the quality scores to binary scores. The instances

@@ -0,0 +1,115 @@+{-# LANGUAGE DeriveDataTypeable #-}+{-# LANGUAGE DoAndIfThenElse #-}++module Data.Alpino.Model.Conduit (+  addNewLine,+  bestScore,+  concat,+  filterFeatures,+  filterFeaturesFunctor,+  groupByKey,+  randomSample,+  scoreToBinary,+  scoreToBinaryNorm,+  scoreToNorm,+  bsToTrainingInstance,+  trainingInstanceToBS+) where++import           Prelude hiding (concat)+import           Control.Exception.Base (Exception)+import           Control.Monad.Random.Class (MonadRandom(..))+import qualified Data.Alpino.Model as AM+import qualified Data.ByteString as B+import           Data.Conduit (Conduit, ConduitStateResult(StateProducing),+                   conduitState)+import qualified Data.Conduit.List as CL+import qualified Data.Set as Set+import Data.Typeable+import Control.Monad.Trans.Resource (MonadThrow (..))++data InvalidDataException = InvalidDataException String+  deriving Typeable++instance Exception InvalidDataException++instance Show InvalidDataException where+  show (InvalidDataException e) = show e++addNewLine :: Monad m => Conduit B.ByteString m B.ByteString+addNewLine =+  CL.map (\bs -> B.snoc bs 10)++-- | Retrieve the best score from a list of training instances.+bestScore :: Monad m => Conduit [AM.TrainingInstance] m Double+bestScore = CL.map AM.bestScore'++concat :: Monad m => Conduit [a] m a+concat =+  conduitState+    ()+    (\_ v -> return $ StateProducing () v)+    (\_   -> return [])+++-- |+-- Filter features by exact names. A modifier function can be applied,+-- for instance, the 'not' function would exclude the specified features.+filterFeatures :: Monad m => (Bool -> Bool) -> Set.Set B.ByteString ->+  Conduit AM.TrainingInstance m AM.TrainingInstance+filterFeatures f keepFeatures = CL.map (AM.filterFeatures f keepFeatures)++-- |+-- Filter features by their functor. A modifier function can be applied,+-- for instance, the 'not' function would exclude the specified features.+filterFeaturesFunctor :: Monad m => (Bool -> Bool) -> Set.Set B.ByteString ->+  Conduit AM.TrainingInstance m AM.TrainingInstance+filterFeaturesFunctor f keepFeatures =+    CL.map (AM.filterFeaturesFunctor f keepFeatures)++-- | Group training instances by key.+groupByKey :: Monad m =>+           Conduit AM.TrainingInstance m [AM.TrainingInstance]+groupByKey = CL.groupBy keyEq+    where keyEq i1 i2 = AM.instanceType i1 == AM.instanceType i2 &&+                        AM.instanceKey i1 == AM.instanceKey i2++randomSample :: MonadRandom m => Int ->+  Conduit [AM.TrainingInstance] m [AM.TrainingInstance]+randomSample size = CL.mapM $ AM.randomSample size++-- |+-- Conduit recaculating scores to binary scores (/1.0/ for best,+-- /0.0/ for the rest).+scoreToBinary :: Monad m =>+  Conduit [AM.TrainingInstance] m [AM.TrainingInstance]+scoreToBinary = CL.map AM.scoreToBinary++-- |+-- Conduit recalculating scores, dividing a score of /1.0/ uniformly+-- over instances with the highest quality score.+scoreToBinaryNorm :: Monad m =>+  Conduit [AM.TrainingInstance] m [AM.TrainingInstance]+scoreToBinaryNorm = CL.map AM.scoreToBinaryNorm++-- |+-- Conduit that normalized instance scores over all instances+-- in the list.+scoreToNorm :: Monad m =>+  Conduit [AM.TrainingInstance] m [AM.TrainingInstance]+scoreToNorm = CL.map AM.scoreToNorm++-- XXX: Use proper serialization?+bsToTrainingInstance :: MonadThrow m => Conduit B.ByteString m AM.TrainingInstance+bsToTrainingInstance = CL.mapM cvtBS+  where+    cvtBS b =+      case AM.bsToTrainingInstance b of+        Just i -> return $ i+        Nothing -> monadThrow $+          InvalidDataException "Could not parse instance."++-- XXX: Use some proper serialization here?+-- | Convert `TrainingInstance`s to `B.ByteString`s.+trainingInstanceToBS :: Monad m => Conduit AM.TrainingInstance m B.ByteString+trainingInstanceToBS = CL.map AM.trainingInstanceToBs

@@ -1,180 +0,0 @@-{-# OPTIONS_GHC -XDeriveDataTypeable #-}--- |--- Module      : Data.Alpino.Model.Enumerator--- Copyright   : (c) 2010 Daniël de Kok--- License     : Apache 2------ Maintainer  : Daniël de Kok <me@danieldk.eu>--- Stability   : experimental------ Enumerators derived from Data.Alpino.Model--module Data.Alpino.Model.Enumerator ( bestScore,-                                      concat,-                                      groupBy,-                                      groupByKey,-                                      filterFeatures,-                                      filterFeaturesFunctor,-                                      instanceGenerator,-                                      instanceParser,-                                      lineEnum,-                                      printByteString,-                                      randomSample,-                                      scoreToBinary,-                                      scoreToBinaryNorm,-                                      scoreToNorm-                                    ) where--import Prelude hiding (concat, filter, mapM)-import Control.Exception.Base (Exception)-import Control.Monad.IO.Class (MonadIO(..), liftIO)-import Control.Monad.Trans.Class (lift)-import qualified Data.Alpino.Model as AM-import qualified Data.ByteString as B-import qualified Data.ByteString.UTF8 as BU-import qualified Data.Enumerator.List as EL-import Data.Enumerator hiding (isEOF, head, length, map)-import qualified Data.Set as Set-import Data.Typeable-import System.IO (isEOF)-import System.Random (getStdRandom, split)--data InvalidDataException = InvalidDataException String-                            deriving Typeable--instance Exception InvalidDataException--instance Show InvalidDataException where-    show (InvalidDataException e) = show e---- | Retrieve the best score from a list of training instances.-bestScore :: (Monad m) =>-               Enumeratee [AM.TrainingInstance] Double m b-bestScore = EL.map AM.bestScore'---- |--- Filter features by exact names. A modifier function can be applied,--- for instance, the 'not' function would exclude the specified features.-filterFeatures :: (Monad m) =>  (Bool -> Bool) -> Set.Set B.ByteString ->-                  Enumeratee AM.TrainingInstance AM.TrainingInstance m b-filterFeatures f keepFeatures = EL.map (AM.filterFeatures f keepFeatures)---- |--- Filter features by their functor. A modifier function can be applied,--- for instance, the 'not' function would exclude the specified features.-filterFeaturesFunctor :: (Monad m) =>  (Bool -> Bool) -> Set.Set B.ByteString ->-                         Enumeratee AM.TrainingInstance AM.TrainingInstance m b-filterFeaturesFunctor f keepFeatures =-    EL.map (AM.filterFeaturesFunctor f keepFeatures)---- | Enumeratee grouping chunks according to an equality function.-groupBy :: (Monad m, Eq a) => (a -> a -> Bool) ->-           Enumeratee a [a] m b-groupBy f = loop-    where loop (Continue k) = do-            h <- peek-            case h of-              Nothing -> return $ Continue k-              Just e -> do-                     xs <- EL.takeWhile $ f e-                     newStep <- lift $ runIteratee $ k $ Chunks [xs]-                     loop newStep-          loop step = return step---- | Group training instances by key.-groupByKey :: (Monad m) =>-           Enumeratee AM.TrainingInstance [AM.TrainingInstance] m b-groupByKey = groupBy keyEq-    where keyEq i1 i2 = AM.instanceType i1 == AM.instanceType i2 &&-                        AM.instanceKey i1 == AM.instanceKey i2---- | Enumeratee that converts `BU.ByteString` to `AM.TrainingInstance`.-instanceParser :: (Monad m) =>-                  Enumeratee BU.ByteString AM.TrainingInstance m b-instanceParser = mapMaybeEnum (InvalidDataException "Could not parse instance.")-                 AM.bsToTrainingInstance---- | Enumeratee that converts `AM.TrainingInstance` to `B.ByteString`.-instanceGenerator :: (Monad m) =>-                     Enumeratee AM.TrainingInstance B.ByteString m b-instanceGenerator = EL.map AM.trainingInstanceToBs---- | Enumerator of lines read from the standard input.-lineEnum :: MonadIO m => Enumerator B.ByteString m b-lineEnum = Iteratee . loop-    where loop (Continue k) = do-            eof <- liftIO isEOF-            case eof of-              True -> return $ Continue k-              False -> do-                       line <- liftIO B.getLine-                       runIteratee (k (Chunks [line])) >>= loop-          loop step = return step---- | Enumeratee concatenating lists.-concat :: (Monad m) =>-              Enumeratee [a] a m b-concat = loop-    where loop (Continue k) = do-            h <- EL.head-            case h of-              Nothing -> return $ Continue k-              Just e -> do-                     newStep <- lift $ runIteratee $ k $ Chunks e-                     loop newStep-          loop step = return step--mapMaybeEnum :: (Exception e, Monad m) => e -> (ao -> Maybe ai) ->-                Enumeratee ao ai m b-mapMaybeEnum exception f = loop where-    loop = checkDone $ continue . step-    step k EOF = yield (Continue k) EOF-    step k (Chunks []) = continue $ step k-    step k (Chunks xs) = case mapMaybeMaybe f xs of-                     Just ys -> k (Chunks ys) >>== loop-                     Nothing -> throwError exception---- If one function application fails return Nothing, otherwise Just xs-mapMaybeMaybe :: (a -> Maybe b) -> [a] -> Maybe [b]-mapMaybeMaybe _ []     = Just []-mapMaybeMaybe f (x:xs) = do-  r <- f x-  rs <- mapMaybeMaybe f xs-  return $ r:rs---- | Iterator printing `B.ByteString` to the standard output.-printByteString :: MonadIO m => Iteratee B.ByteString m ()-printByteString = continue step-    where step (Chunks []) = continue step-          step (Chunks xs) = liftIO (mapM_ B.putStrLn xs) >> continue step-          step EOF = yield () EOF---- | Extract a random sample of @n@ instances from a context.-randomSample :: (MonadIO m) => Int ->-                Enumeratee [AM.TrainingInstance] [AM.TrainingInstance] m b-randomSample n = EL.mapM (liftIO . sampleFun)-    where sampleFun :: [AM.TrainingInstance] -> IO [AM.TrainingInstance]-          sampleFun i = do-            gen <- getStdRandom split-            return $ AM.randomSample gen n i---- |--- Enumerator recaculating scores to binary scores (/1.0/ for best,--- /0.0/ for the rest).-scoreToBinary :: (Monad m) =>-                 Enumeratee [AM.TrainingInstance] [AM.TrainingInstance] m b-scoreToBinary = EL.map AM.scoreToBinary---- |--- Enumerator recalculating scores, dividing a score of /1.0/ uniformly--- over instances with the highest quality score.-scoreToBinaryNorm :: (Monad m) =>-                     Enumeratee [AM.TrainingInstance] [AM.TrainingInstance] m b-scoreToBinaryNorm = EL.map AM.scoreToBinaryNorm---- |--- Enumerator that normalized instance scores over all instances--- in the list.-scoreToNorm :: (Monad m) =>-               Enumeratee [AM.TrainingInstance] [AM.TrainingInstance] m b-scoreToNorm = EL.map AM.scoreToNorm

@@ -1,61 +0,0 @@-module Main where--import Prelude hiding (concat)--import Control.Monad (unless)-import Data.Alpino.Model.Enumerator-import Data.ByteString.UTF8 (fromString)-import Data.Enumerator (($$), joinI, run_)-import Data.List as L-import qualified Data.Set as Set-import System.Console.GetOpt-import System.Environment-import System.Exit-import System.IO--main :: IO ()-main = do-  (options, args) <- getOptions--  let filter0 = if elem FilterFeatures options-                then filterFeatures-                else filterFeaturesFunctor-  let filter = if elem InverseFilter options-               then filter0 not-               else filter0 id--  unless (not $ null args) $ do-         name <- getProgName-         hPutStrLn stderr $ usageInfo (usage name) optionInfo-         exitFailure--  let keepFeatures = Set.fromList $ map fromString args--  run_ $ lineEnum $$ joinI $ instanceParser $$-       joinI $ filter keepFeatures $$-       joinI $ instanceGenerator $$ printByteString--data Option = FilterFeatures | FilterFunctors | InverseFilter-            deriving Eq--optionInfo :: [OptDescr Option]-optionInfo =-    [ Option ['f'] ["functor"] (NoArg FilterFunctors) "filter feature functors",-      Option ['i'] ["inverse"] (NoArg InverseFilter) "exclude specified features"]--usage :: String -> String-usage name = "Usage: " ++ name ++ " <OPTION> [FEATURES]\n"--getOptions :: IO ([Option], [String])-getOptions = do-  args <- getArgs-  let (options, keep, errors) = getOpt Permute optionInfo args-  unless (null errors) $ do-               name <- getProgName-               hPutStrLn stderr $ L.concat errors-               hPutStrLn stderr $ usageInfo (usage name) optionInfo-               exitFailure--  case options of-    []        -> return ([FilterFeatures], keep)-    otherwise -> return (options, keep)

@@ -1,26 +0,0 @@-{-# OPTIONS -XBangPatterns #-}--module Main where--import Prelude hiding (concat, filter)--import Data.Alpino.Model.Enumerator-import Data.Enumerator hiding (isEOF, length)-import Data.List (genericLength)-import Text.Printf (printf)--sumCount :: (Monad m, Fractional a, Integral b) => Iteratee a m (a, b)-sumCount = continue $ step (0.0, 0)-    where step acc (Chunks []) = continue $ step acc-          step (!sumAcc, !lenAcc) (Chunks xs) =-              continue $ step (sumAcc + sum xs, lenAcc + genericLength xs)-          step acc EOF = yield acc EOF--main :: IO ()-main = do-  (scoreSum, scoreLen) <- run_ $ lineEnum $$ joinI $ instanceParser $$-                          joinI $ groupByKey $$-                          joinI $ bestScore $$ sumCount-  putStrLn $ "Contexts: " ++ (show scoreLen)-  putStrLn $ "Oracle: " ++ (printf("%.4f") $ scoreSum / fromIntegral scoreLen)-

@@ -1,41 +0,0 @@-module Main where--import Prelude hiding (concat)--import Control.Monad (unless)-import Data.Alpino.Model.Enumerator-import Data.Enumerator (($$), joinI, run_)-import qualified Data.List as L-import System.Console.GetOpt-import System.Environment-import System.Exit-import System.IO--main :: IO ()-main = do-  options <- getOptions-  let (SampleSize n) = head options    -  run_ $ lineEnum $$ joinI $ instanceParser $$ joinI $ groupByKey $$-       joinI $ randomSample n $$ joinI $ concat $$-       joinI $ instanceGenerator $$ printByteString--data Option = SampleSize Int--optionInfo :: [OptDescr Option]-optionInfo =-    [ Option "n" ["sample_size"] (ReqArg (\n -> SampleSize . read $ n) "NUMBER")-                 "random sample size"]--usage :: String -> String-usage name = "Usage: " ++ name ++ " <OPTION>\n"--getOptions :: IO ([Option])-getOptions = do-  args <- getArgs-  let (options, _, errors) = getOpt Permute optionInfo args-  unless (null errors && length options == 1) $ do-               name <- getProgName-               hPutStrLn stderr $ L.concat errors-               hPutStrLn stderr $ usageInfo (usage name) optionInfo-               exitFailure-  return options

@@ -1,47 +0,0 @@-module Main where--import Prelude hiding (concat)--import Control.Monad (unless)-import Data.Alpino.Model.Enumerator-import Data.Enumerator (($$), joinI, run_)-import qualified Data.List as L-import System.Console.GetOpt-import System.Environment-import System.Exit-import System.IO--main :: IO ()-main = do-  option <- getOptions-  let score = case option of-                Binary           -> scoreToBinary-                BinaryNormalized -> scoreToBinaryNorm-                Normalized       -> scoreToNorm-    -  run_ $ lineEnum $$ joinI $ instanceParser $$ joinI $ groupByKey $$-       joinI $ score $$ joinI $ concat $$-       joinI $ instanceGenerator $$ printByteString--data Option = Binary | Normalized | BinaryNormalized--optionInfo :: [OptDescr Option]-optionInfo =-    [ Option ['b'] ["binary"] (NoArg Binary) "convert to binary scores",-      Option ['i'] ["binary_normalize"] (NoArg BinaryNormalized)-                 "binary normalize over context",-      Option ['n'] ["normalize"] (NoArg Normalized) "normalize over context" ]--usage :: String -> String-usage name = "Usage: " ++ name ++ " <OPTION>\n"--getOptions :: IO (Option)-getOptions = do-  args <- getArgs-  let (options, _, errors) = getOpt Permute optionInfo args-  unless (null errors && length options == 1) $ do-               name <- getProgName-               hPutStrLn stderr $ L.concat errors-               hPutStrLn stderr $ usageInfo (usage name) optionInfo-               exitFailure-  return $ head options

@@ -1,29 +0,0 @@-{-# OPTIONS -XBangPatterns #-}--module Main where--import Prelude hiding (concat, filter)--import qualified Data.Alpino.Model as AM-import Data.Alpino.Model.Enumerator-import Data.Enumerator hiding (isEOF, length, map)-import Data.List (genericLength)-import Text.Printf (printf)--statistics :: Monad m => Iteratee [AM.TrainingInstance] m (Int, Int, Int)-statistics = continue $ step (0, 0, 0)-    where step acc (Chunks []) = continue $ step acc-          step (!lenSumAcc, !lenAcc, !maxLenAcc) (Chunks xs) =-              continue $ (step $ (lenSumAcc + (sum $ map length xs),-                                  lenAcc + genericLength xs,-                                  max maxLenAcc $ maximum $ map length xs))-          step acc EOF = yield acc EOF--main :: IO ()-main = do-  (lenSum, len, maxLen) <- run_ $ lineEnum $$ joinI $ instanceParser $$-                           joinI $ groupByKey $$ statistics-  putStrLn $ "Contexts: " ++ (show len)-  putStrLn $ "Max. events: " ++ (show maxLen)-  putStrLn $ "Avg. events: " ++ (printf "%.2f" $ (fromIntegral lenSum / fromIntegral len :: Double))-

@@ -0,0 +1,62 @@+module Main where++import Prelude hiding (concat)++import Control.Monad (unless)+import Data.Alpino.Model.Conduit+import Data.ByteString.UTF8 (fromString)+import Data.Conduit (runResourceT, ($=), ($$))+import qualified Data.Conduit.Binary as CB+import Data.List as L+import qualified Data.Set as Set+import System.Console.GetOpt+import System.Environment+import System.Exit+import System.IO++main :: IO ()+main = do+  (options, args) <- getOptions++  let filter0 = if elem FilterFeatures options+                then filterFeatures+                else filterFeaturesFunctor+  let filter1 = if elem InverseFilter options+               then filter0 not+               else filter0 id++  unless (not $ null args) $ do+         name <- getProgName+         hPutStrLn stderr $ usageInfo (usage name) optionInfo+         exitFailure++  let keepFeatures = Set.fromList $ map fromString args++  runResourceT (CB.sourceHandle stdin $= CB.lines $= bsToTrainingInstance $=+    filter1 keepFeatures $= trainingInstanceToBS $= addNewLine $$+    CB.sinkHandle stdout)++data Option = FilterFeatures | FilterFunctors | InverseFilter+            deriving Eq++optionInfo :: [OptDescr Option]+optionInfo =+    [ Option ['f'] ["functor"] (NoArg FilterFunctors) "filter feature functors",+      Option ['i'] ["inverse"] (NoArg InverseFilter) "exclude specified features"]++usage :: String -> String+usage name = "Usage: " ++ name ++ " <OPTION> [FEATURES]\n"++getOptions :: IO ([Option], [String])+getOptions = do+  args <- getArgs+  let (options, keep, errors) = getOpt Permute optionInfo args+  unless (null errors) $ do+               name <- getProgName+               hPutStrLn stderr $ L.concat errors+               hPutStrLn stderr $ usageInfo (usage name) optionInfo+               exitFailure++  case options of+    []        -> return ([FilterFeatures], keep)+    _         -> return (options, keep)

@@ -0,0 +1,28 @@+{-# OPTIONS -XBangPatterns #-}++module Main where++import Prelude hiding (concat, filter)++import Data.Alpino.Model.Conduit+import Data.Conduit (Sink, runResourceT, ($=), ($$))+import qualified Data.Conduit.Binary as CB+import qualified Data.Conduit.List as CL+import System.IO (stdin)+import Text.Printf (printf)++sumCount :: (Monad m, Fractional a, Integral b) => Sink a m (a, b)+sumCount =+	CL.fold handleCtx (0, 0)+	where+		handleCtx (!scoreSum, !ctxs) score =+			(scoreSum + score, ctxs + 1)++main :: IO ()+main = do+  (scoreSum, scoreLen) <- runResourceT (CB.sourceHandle stdin $= CB.lines $=+  	bsToTrainingInstance $= groupByKey $= bestScore $$ sumCount) :: IO (Double, Int)++  putStrLn $ printf "Contexts: %d" scoreLen+  putStrLn $ printf "Oracle: %.4f" (scoreSum / fromIntegral scoreLen)+

@@ -0,0 +1,51 @@+module Main where++import Prelude hiding (concat)++import Control.Monad (unless)+import Control.Monad.Random (MonadRandom(..))+import Control.Monad.Trans (lift)+import Control.Monad.Trans.Resource (ResourceT)+import Data.Alpino.Model.Conduit+import Data.Conduit (($=), ($$), runResourceT)+import qualified Data.Conduit.Binary as CB+import qualified Data.List as L+import System.Console.GetOpt+import System.Environment+import System.Exit+import System.IO++instance MonadRandom m => MonadRandom (ResourceT m) where+  getRandom   = lift getRandom+  getRandoms  = lift getRandoms+  getRandomR  = lift . getRandomR+  getRandomRs = lift . getRandomRs++main :: IO ()+main = do+  options <- getOptions+  let (SampleSize n) = head options    +  runResourceT (CB.sourceHandle stdin $= CB.lines $= bsToTrainingInstance $=+    groupByKey $= randomSample n $= concat $=+    trainingInstanceToBS $= addNewLine $$ CB.sinkHandle stdout)++data Option = SampleSize Int++optionInfo :: [OptDescr Option]+optionInfo =+    [ Option "n" ["sample_size"] (ReqArg (\n -> SampleSize . read $ n) "NUMBER")+                 "random sample size"]++usage :: String -> String+usage name = "Usage: " ++ name ++ " <OPTION>\n"++getOptions :: IO ([Option])+getOptions = do+  args <- getArgs+  let (options, _, errors) = getOpt Permute optionInfo args+  unless (null errors && length options == 1) $ do+               name <- getProgName+               hPutStrLn stderr $ L.concat errors+               hPutStrLn stderr $ usageInfo (usage name) optionInfo+               exitFailure+  return options

@@ -0,0 +1,48 @@+module Main where++import Prelude hiding (concat)++import Control.Monad (unless)+import Data.Alpino.Model.Conduit+import Data.Conduit (runResourceT, ($=), ($$))+import qualified Data.Conduit.Binary as CB+import qualified Data.List as L+import System.Console.GetOpt+import System.Environment+import System.Exit+import System.IO++main :: IO ()+main = do+  option <- getOptions+  let score = case option of+                Binary           -> scoreToBinary+                BinaryNormalized -> scoreToBinaryNorm+                Normalized       -> scoreToNorm++  runResourceT (CB.sourceHandle stdin $= CB.lines $= bsToTrainingInstance $=+    groupByKey $= score $= concat $= trainingInstanceToBS $= addNewLine $$+    CB.sinkHandle stdout)++data Option = Binary | Normalized | BinaryNormalized++optionInfo :: [OptDescr Option]+optionInfo =+    [ Option ['b'] ["binary"] (NoArg Binary) "convert to binary scores",+      Option ['i'] ["binary_normalize"] (NoArg BinaryNormalized)+                 "binary normalize over context",+      Option ['n'] ["normalize"] (NoArg Normalized) "normalize over context" ]++usage :: String -> String+usage name = "Usage: " ++ name ++ " <OPTION>\n"++getOptions :: IO (Option)+getOptions = do+  args <- getArgs+  let (options, _, errors) = getOpt Permute optionInfo args+  unless (null errors && length options == 1) $ do+               name <- getProgName+               hPutStrLn stderr $ L.concat errors+               hPutStrLn stderr $ usageInfo (usage name) optionInfo+               exitFailure+  return $ head options

@@ -0,0 +1,34 @@+{-# OPTIONS -XBangPatterns #-}++module Main where++import Prelude hiding (concat, filter)++import qualified Data.Alpino.Model as AM+import Data.Alpino.Model.Conduit+import Data.Conduit (Sink, runResourceT, ($=), ($$))+import qualified Data.Conduit.Binary as CB+import qualified Data.Conduit.List as CL+import Data.List (genericLength)+import System.IO (stdin)+import Text.Printf (printf)++statistics :: Monad m => Sink [AM.TrainingInstance] m (Int, Int, Int)+statistics =+  CL.fold handleCtx (0, 0, 0)+  where+    handleCtx (!evts, !ctxs, !maxCtx) ctx =+      (evts + ctxLen, succ ctxs, max maxCtx ctxLen)+      where+        ctxLen =+          genericLength ctx++main :: IO ()+main = do+  (evts, ctxs, maxCtx) <- runResourceT (CB.sourceHandle stdin $=+    CB.lines $= bsToTrainingInstance $= groupByKey $$ statistics)++  putStrLn $ "Contexts: " ++ (show ctxs)+  putStrLn $ "Max. events: " ++ (show maxCtx)+  putStrLn $ "Avg. events: " ++ (printf "%.2f" $ (fromIntegral evts / fromIntegral ctxs :: Double))+