packages feed

nerf (empty) → 0.1.0

raw patch · 11 files changed

+755/−0 lines, 11 filesdep +adictdep +basedep +binarysetup-changed

Dependencies added: adict, base, binary, cmdargs, containers, crf-chain1, data-named, monad-ox, polimorf, polysoup, sgd, text, text-binary, vector

Files

+ LICENSE view
@@ -0,0 +1,26 @@+Copyright (c) 2012, IPI PAN+All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions+are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ NLP/Nerf.hs view
@@ -0,0 +1,81 @@+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE OverloadedStrings #-}++-- | Main module of the Nerf tool.++module NLP.Nerf+( Nerf (..)+, train+, ner+, tryOx+, module NLP.Nerf.Types+) where++import Control.Applicative ((<$>), (<*>))+import Data.Binary (Binary, put, get)+import qualified Data.Text.Lazy.IO as L++import Text.Named.Enamex (parseEnamex)+import qualified Data.Named.Tree as Tr+import qualified Data.Named.IOB as IOB++import Numeric.SGD (SgdArgs)+import qualified Data.CRF.Chain1 as CRF++import NLP.Nerf.Types+import NLP.Nerf.Schema (SchemaCfg, Schema, fromCfg, schematize)++-- | A Nerf consists of the observation schema configuration and the CRF model.+data Nerf = Nerf+    { schemaCfg :: SchemaCfg+    , crf       :: CRF.CRF Ob Lb }++instance Binary Nerf where+    put Nerf{..} = put schemaCfg >> put crf+    get = Nerf <$> get <*> get++flatten :: Schema a -> Tr.NeForest NE Word -> CRF.SentL Ob Lb+flatten schema forest =+    [ CRF.annotate x y+    | (x, y) <- zip xs ys ]+  where+    iob = IOB.encodeForest forest+    xs = schematize schema (map IOB.word iob)+    ys = map IOB.label iob++readFlat :: Schema a -> FilePath -> IO [CRF.SentL Ob Lb]+readFlat schema path = map (flatten schema) . parseEnamex <$> L.readFile path++drawSent :: CRF.SentL Ob Lb -> IO ()+drawSent sent = do+    let unDist (x, y) = (x, CRF.unDist y)+    mapM_ (print . unDist) sent+    putStrLn "" ++-- | Show results of observation extraction on the input ENAMEX file.+tryOx :: SchemaCfg -> FilePath -> IO ()+tryOx cfg path = do+    input <- readFlat (fromCfg cfg) path+    mapM_ drawSent input++-- | Train Nerf on the input data using the SGD method.+train+    :: SgdArgs              -- ^ Args for SGD+    -> SchemaCfg            -- ^ Observation schema configuration+    -> FilePath             -- ^ Train data (ENAMEX)+    -> Maybe FilePath       -- ^ Maybe eval data (ENAMEX)+    -> IO Nerf              -- ^ Nerf with resulting codec and model+train sgdArgs cfg trainPath evalPathM = do+    let schema = fromCfg cfg+        readTrain = readFlat schema trainPath+        readEvalM = evalPathM >>= \evalPath ->+            Just ([], readFlat schema evalPath)+    _crf <- CRF.train sgdArgs readTrain readEvalM CRF.presentFeats+    return $ Nerf cfg _crf++-- | Perform named entity recognition (NER) using the Nerf.+ner :: Nerf -> [Word] -> Tr.NeForest NE Word+ner nerf ws =+    let schema = fromCfg (schemaCfg nerf)+        xs = CRF.tag (crf nerf) (schematize schema ws)+    in  IOB.decodeForest [IOB.IOB w x | (w, x) <- zip ws xs]
+ NLP/Nerf/Dict.hs view
@@ -0,0 +1,47 @@+module NLP.Nerf.Dict+( preparePNEG+, prepareNELexicon+, module NLP.Nerf.Dict.Base+) where++import Control.Applicative ((<$>))+import Control.Arrow (first)+import Data.Binary (encodeFile)+import qualified Data.PoliMorf as Poli+import qualified Data.Map as M+import qualified Data.Text as T++import NLP.Nerf.Dict.Base+import NLP.Nerf.Dict.PNEG (readPNEG)+import NLP.Nerf.Dict.NELexicon (readNELexicon)+import qualified NLP.Adict.Trie as Trie++-- | Make dictionary consisting only from one word NEs.+mkDictW1 :: [Entry] -> NeDict+mkDictW1 =+    let oneWord x _ = not (isMultiWord x)+    in  siftDict oneWord . mkDict++-- | Parse the PNEG dictionary and save it in a binary form into+-- the output file.+preparePNEG+    :: FilePath     -- ^ Path to PNEG in the LMF format+    -> FilePath     -- ^ Output file+    -> IO ()+preparePNEG lmfPath outPath = do+    neDict <- mkDictW1 <$> readPNEG lmfPath+    saveDict outPath neDict++-- | Parse the NELexicon, merge it with the PoliMorf and serialize+-- into a binary, DAWG form.+prepareNELexicon+    :: FilePath     -- ^ Path to NELexicon+    -> FilePath     -- ^ Path to PoliMorf+    -> FilePath     -- ^ Output file+    -> IO ()+prepareNELexicon nePath poliPath outPath = do+    neDict  <- mkDictW1 <$> readNELexicon nePath+    baseMap <- Poli.mkBaseMap <$> Poli.readPoliMorf poliPath+    let neDict' = Poli.merge baseMap neDict+        trie    = Trie.fromList $ map (first T.unpack) (M.assocs neDict')+    encodeFile outPath (Trie.serialize trie)
+ NLP/Nerf/Dict/Base.hs view
@@ -0,0 +1,85 @@+-- | Basic types for dictionary handling. ++module NLP.Nerf.Dict.Base+(+-- * Lexicon entry+  Form+, isMultiWord+, NeType+, Entry (..)++-- * Dictionary+, NeDict+, mkDict+, siftDict+, saveDict+, loadDict++-- * Merging dictionaries+, merge+, diff+) where++import Control.Applicative ((<$>))+import Data.Binary (encodeFile, decodeFile)+import Data.Text.Binary ()+import qualified Data.Text as T+import qualified Data.Set as S+import qualified Data.Map as M++-- | A orthographic form.+type Form = T.Text++-- | Is the form a multiword one?+isMultiWord :: Form -> Bool+isMultiWord = (>1) . length . T.words++-- | A type of named entity.+type NeType = T.Text++-- | A Named Entity entry from the LMF dictionary.+data Entry = Entry+    { neOrth :: !Form	-- ^ Orthographic form of the NE+    , neType :: !NeType -- ^ Type of the NE+    } deriving (Show, Read, Eq, Ord)++-- | A NeDict is a map from forms to NE types.  Each NE may be annotated+-- with multiple types.+type NeDict = M.Map Form (S.Set NeType)++-- | Construct the dictionary from the list of entries.+mkDict :: [Entry] -> NeDict+mkDict xs = M.fromListWith S.union+    [(neOrth x, S.singleton $ neType x) | x <- xs]++-- | Remove dictionary entries which do not satisfy the predicate.+siftDict :: (Form -> S.Set NeType -> Bool) -> NeDict -> NeDict+siftDict f dict = M.fromList [(k, v) | (k, v) <- M.assocs dict, f k v]++-- | Save the dictionary in the file.+saveDict :: FilePath -> NeDict -> IO ()+saveDict = encodeFile++-- | Load the dictionary from the file.+loadDict :: FilePath -> IO NeDict+loadDict = decodeFile++-- | Merge dictionary resources.+merge :: [NeDict] -> NeDict+merge = M.unionsWith S.union++-- | Differentiate labels from separate dictionaries using+-- dictionary-unique prefixes.+diff :: [NeDict] -> [NeDict]+diff ds =+    [ mapS (addPrefix i) <$> dict+    | (i, dict) <- zip [0..] ds ]++-- | Map function over the set.+mapS :: Ord a => (a -> a) -> S.Set a -> S.Set a+mapS f s = S.fromList [f x | x <- S.toList s]+{-# INLINE mapS #-}++-- | Add integer prefix.+addPrefix :: Int -> T.Text -> T.Text+addPrefix = T.append . T.pack . show
+ NLP/Nerf/Dict/NELexicon.hs view
@@ -0,0 +1,24 @@+-- | Handling the NELexicon dictionary.++module NLP.Nerf.Dict.NELexicon+( parseNELexicon+, readNELexicon+) where++import qualified Data.Text.Lazy as L+import qualified Data.Text.Lazy.IO as L++import NLP.Nerf.Dict.Base++-- | Parse the NELexicon into a list of entries.+parseNELexicon :: L.Text -> [Entry]+parseNELexicon = map parseLine . L.lines++parseLine :: L.Text -> Entry+parseLine line = case L.break (==';') line of+    (_type, _form) -> Entry (L.toStrict $ L.tail _form) (L.toStrict _type)+    _   -> error $ "parseLine: invalid line \"" ++ L.unpack line ++ "\""++-- | Read the dictionary from the file.+readNELexicon :: FilePath -> IO [Entry]+readNELexicon = fmap parseNELexicon . L.readFile
+ NLP/Nerf/Dict/PNEG.hs view
@@ -0,0 +1,44 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE OverloadedStrings #-}++-- | Parsing the Gazetteer for Polish Named Entities (used formerly within+-- the SProUT platform) in the LMF format.++module NLP.Nerf.Dict.PNEG+( parsePNEG+, readPNEG+) where++import Text.XML.PolySoup+import qualified Data.Text as T+import qualified Data.Text.Lazy as L+import qualified Data.Text.Lazy.IO as L++import NLP.Nerf.Dict.Base++lmfP :: XmlParser L.Text [Entry]+lmfP = true ##> lexEntryP++lexEntryP :: XmlParser L.Text [Entry]+lexEntryP = tag "LexicalEntry" `joinR` do+    many_ $ cut $ tag "feat"+    _words <- many wordP+    sense  <- senseP+    return [Entry x sense | x <- _words]++wordP :: XmlParser L.Text Form+wordP = head <$> (tag "Lemma" <|> tag "WordForm" /> featP "writtenForm")++senseP :: XmlParser L.Text NeType+senseP = head <$> (tag "Sense" //> featP "externalReference" <|> featP "label")++featP :: L.Text -> XmlParser L.Text T.Text+featP x = L.toStrict <$> cut (tag "feat" *> hasAttr "att" x *> getAttr "val")++-- | Parse the dictionary to the list of entries.+parsePNEG :: L.Text -> [Entry]+parsePNEG = parseXml lmfP++-- | Read the dictionary from the file.+readPNEG :: FilePath -> IO [Entry]+readPNEG = fmap parsePNEG . L.readFile
+ NLP/Nerf/Schema.hs view
@@ -0,0 +1,232 @@+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE OverloadedStrings #-}++-- | Observation schema blocks for Nerf.++module NLP.Nerf.Schema+( +-- * Schema+  Ox+, Schema+, void+, sequenceS_++-- * Using the schema+, schematize++-- * Building schema++-- ** From config+, SchemaCfg (..)+, defaultCfg+, fromCfg++-- ** Schema blocks+, Block+, fromBlock+, orthS+, lemmaS+, shapeS+, shapePairS+, suffixS+, searchS+) where++import Control.Applicative ((<$>), (<*>))+import Control.Monad (forM_, join)+import Data.Maybe (maybeToList)+import Data.Binary (Binary, put, get, decodeFile)+import qualified Data.Char as C+import qualified Data.Set as S+import qualified Data.Map as M+import qualified Data.Vector as V+import qualified Data.Text as T++import qualified Data.CRF.Chain1 as CRF+import qualified Control.Monad.Ox as Ox+import qualified Control.Monad.Ox.Text as Ox++import NLP.Nerf.Types+import qualified NLP.Nerf.Dict as Dict++-- | The Ox monad specialized to word token type and text observations.+type Ox a = Ox.Ox Word T.Text a++-- | A schema is a block of the Ox computation performed within the+-- context of the sentence and the absolute sentence position.+type Schema a = V.Vector Word -> Int -> Ox a++-- | A dummy schema block.+void :: a -> Schema a+void x _ _ = return x++-- | Sequence the list of schemas and discard individual values.+sequenceS_ :: [Schema a] -> Schema ()+sequenceS_ xs sent =+    let ys = map ($sent) xs+    in  \k -> sequence_ (map ($k) ys)++-- | Record structure of the basic observation types.+data BaseOb = BaseOb+    { orth          :: Int -> Maybe T.Text+    , lowOrth       :: Int -> Maybe T.Text }++-- | Construct the 'BaseOb' structure given the sentence.+mkBaseOb :: V.Vector Word -> BaseOb+mkBaseOb sent = BaseOb+    { orth      = _orth+    , lowOrth   = _lowOrth }+  where+    at          = Ox.atWith sent+    _orth       = (id `at`)+    _lowOrth i  = T.toLower <$> _orth i++-- | A block is a chunk of the Ox computation performed within the+-- context of the sentence and the list of absolute sentence positions.+type Block a = V.Vector Word -> [Int] -> Ox a++-- | Transform the block to the schema dependent on the list of+-- relative sentence positions.+fromBlock :: Block a -> [Int] -> Schema a+fromBlock blk xs sent =+    let blkSent = blk sent+    in  \k -> blkSent [x + k | x <- xs]++-- | Orthographic observations determined with respect to the+-- list of relative positions.+orthS :: Block ()+orthS sent = \ks -> do+    mapM_ (Ox.save . lowOrth)    ks+    mapM_ (Ox.save . upOnlyOrth) ks+  where+    BaseOb{..}      = mkBaseOb sent+    upOnlyOrth i    = orth i >>= \x -> case T.any C.isUpper x of+        True    -> Just x+        False   -> Nothing++-- | Lemma substitute determined with respect to the list of+-- relative positions.+lemmaS :: Block ()+lemmaS sent = \ks -> do+    mapM_ lowLemma ks+  where+    BaseOb{..}      = mkBaseOb sent+    lowPrefix i j   = Ox.prefix j =<< lowOrth i+    lowSuffix i j   = Ox.suffix j =<< lowOrth i+    lowLemma i = Ox.group $ do+        mapM_ (Ox.save . lowPrefix i) [0, -1, -2, -3]+        mapM_ (Ox.save . lowSuffix i) [0, -1, -2, -3]++-- | Shape and packed shape determined with respect to the list of+-- relative positions.+shapeS :: Block ()+shapeS sent = \ks -> do+    mapM_ (Ox.save . shape)  ks+    mapM_ (Ox.save . shapeP) ks+  where+    BaseOb{..}      = mkBaseOb sent+    shape i         = Ox.shape <$> orth i+    shapeP i        = Ox.pack <$> shape i++-- | Shape pairs determined with respect to the list of relative positions.+shapePairS :: Block ()+shapePairS sent = \ks ->+    forM_ ks $ \i -> do+        Ox.save $ link <$> shape  i <*> shape  (i - 1)+        Ox.save $ link <$> shapeP i <*> shapeP (i - 1)+  where+    BaseOb{..}      = mkBaseOb sent+    shape i         = Ox.shape <$> orth i+    shapeP i        = Ox.pack <$> shape i+    link x y        = T.concat [x, "-", y]++-- | Several suffixes determined with respect to the list of+-- relative positions.+suffixS :: Block ()+suffixS sent = \ks ->+    forM_ ks $ \i ->+        mapM_ (Ox.save . lowSuffix i) [2, 3, 4]+  where+    BaseOb{..}      = mkBaseOb sent+    lowSuffix i j   = Ox.suffix j =<< lowOrth i++-- | Plain dictionary search determined with respect to the list of+-- relative positions.+searchS :: Dict.NeDict -> Block ()+searchS dict sent = \ks -> do+    mapM_ (Ox.saves . searchDict) ks+  where+    BaseOb{..}      = mkBaseOb sent+    searchDict i    = join . maybeToList $+        S.toList <$> (orth i >>= flip M.lookup dict)++-- | Configuration of the schema.  All configuration elements specify the+-- range over which a particular observation type should be taken on account.+-- For example, the @[-1, 0, 2]@ range means that observations of particular+-- type will be extracted with respect to previous (@k - 1@), current (@k@)+-- and after the next (@k + 2@) positions when identifying the observation+-- set for position @k@ in the input sentence.+data SchemaCfg = SchemaCfg+    { orthC         :: [Int]    -- ^ The 'orthS' schema block+    , lemmaC        :: [Int]    -- ^ The 'lemmaS' schema block+    , shapeC        :: [Int]    -- ^ The 'shapeS' schema block+    , shapePairC    :: [Int]    -- ^ The 'shapePairS' schema block+    , suffixC       :: [Int]    -- ^ The 'suffixS' schema block+    , dictC     :: Maybe (Dict.NeDict, [Int]) -- ^ The 'searchS' schema block+    }++instance Binary SchemaCfg where+    put SchemaCfg{..} = do+        put orthC+        put lemmaC+        put shapeC+        put shapePairC+        put suffixC+        put dictC+    get = SchemaCfg+        <$> get+        <*> get+        <*> get+        <*> get+        <*> get+        <*> get++-- | Default configuration for Nerf observation schema.+defaultCfg+    :: FilePath     -- ^ Path to 'Dict.NeDict' in a binary form+    -> IO SchemaCfg+defaultCfg nePath = do+    neDict <- decodeFile nePath+    return $ SchemaCfg+        { orthC         = [-1, 0]+        , lemmaC        = [-1, 0]+        , shapeC        = [-1, 0]+        , shapePairC    = [0]+        , suffixC       = [0]+        , dictC         = Just (neDict, [-1, 0]) }++mkBasicS :: Block () -> [Int] -> Schema ()+mkBasicS _   [] = void ()+mkBasicS blk xs = fromBlock blk xs++mkDictS :: Maybe (Dict.NeDict, [Int]) -> Schema ()+mkDictS (Just (d, xs))  = fromBlock (searchS d) xs+mkDictS Nothing         = void ()++-- | Build the schema based on the configuration.+fromCfg :: SchemaCfg -> Schema ()+fromCfg SchemaCfg{..} = sequenceS_+    [ mkBasicS orthS orthC+    , mkBasicS lemmaS lemmaC+    , mkBasicS shapeS shapeC+    , mkBasicS shapePairS shapePairC+    , mkBasicS suffixS suffixC+    , mkDictS dictC ]++-- | Use the schema to extract observations from the sentence.+schematize :: Schema a -> [Word] -> CRF.Sent Ob+schematize schema xs =+    map (S.fromList . Ox.execOx . schema v) [0 .. n - 1]+  where+    v = V.fromList xs+    n = V.length v
+ NLP/Nerf/Types.hs view
@@ -0,0 +1,25 @@+-- | Basic types.++module NLP.Nerf.Types+( Word+, NE+, Ob+, Lb+) where++import qualified Data.Text as T+import qualified Data.Named.IOB as IOB++-- | A word.+type Word = T.Text++-- | A named entity.+type NE = T.Text++-- | An observation consist of an index (of list type) and an actual+-- observation value.+type Ob = ([Int], T.Text)++-- | A label is created by encoding the named entity forest using the+-- IOB method.+type Lb = IOB.Label NE
+ Setup.lhs view
@@ -0,0 +1,4 @@+#! /usr/bin/env runhaskell++> import Distribution.Simple+> main = defaultMain
+ nerf.cabal view
@@ -0,0 +1,62 @@+name:               nerf+version:            0.1.0+synopsis:           Nerf, the named entity recognition tool based on linear-chain CRFs+description:+    The package provides the named entity recognition (NER) tool divided into a+    back-end library (see the "NLP.Nerf" module) and the front-end tool nerf.+    Using the library you can model and recognize named entities (NEs) which,+    for a particular sentence, take the form of forest with NE category values+    kept in internal nodes and sentence words kept in forest leaves.+    .+    To model NE forests we combine two different techniques. The IOB codec+    is used to translate to and fro between the original, forest representation+    of NEs and the sequence of atomic labels. In other words, it provides two+    isomorphic functions for encoding and decoding between both+    representations. Linear-chain conditional random fields, on the other hand,+    provide the framework for label modelling and tagging. +license:            BSD3+license-file:       LICENSE+cabal-version:      >= 1.6+copyright:          Copyright (c) 2012 IPI PAN+author:             Jakub Waszczuk+maintainer:         waszczuk.kuba@gmail.com+stability:          experimental+category:           Natural Language Processing+homepage:           https://github.com/kawu/nerf+build-type:         Simple++library+    build-depends:+        base >= 4 && < 5+      , containers+      , vector+      , text+      , binary+      , text-binary+      , polysoup >= 0.1 && < 0.2+      , crf-chain1 >= 0.2 && < 0.3+      , data-named >= 0.5 && < 0.6+      , monad-ox >= 0.2 && < 0.3+      , sgd >= 0.2.1 && < 0.3+      , polimorf >= 0.3.1 && < 0.4+      , adict >= 0.2 && < 0.3+      , cmdargs++    exposed-modules:+        NLP.Nerf+      , NLP.Nerf.Types+      , NLP.Nerf.Schema+      , NLP.Nerf.Dict+      , NLP.Nerf.Dict.Base+      , NLP.Nerf.Dict.PNEG+      , NLP.Nerf.Dict.NELexicon++    ghc-options: -Wall -O2++source-repository head+    type: git+    location: git://github.com/kawu/nerf.git++Executable nerf+  Hs-Source-Dirs: ., tools+  Main-is: nerf.hs
+ tools/nerf.hs view
@@ -0,0 +1,125 @@+{-# LANGUAGE DeriveDataTypeable #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE RecordWildCards #-}++import Control.Monad (forM_, when)+import System.Console.CmdArgs+import Data.Binary (encodeFile, decodeFile)+import Data.Text.Binary ()+import Text.Named.Enamex (showForest)+import qualified Numeric.SGD as SGD+import qualified Data.Text as T+import qualified Data.Text.Lazy as L+import qualified Data.Text.Lazy.IO as L++import NLP.Nerf (train, ner, tryOx)+import NLP.Nerf.Schema (defaultCfg)+import NLP.Nerf.Dict (preparePNEG, prepareNELexicon)++data Args+  = TrainMode+    { trainPath	    :: FilePath+    , neDictPath    :: FilePath+    , evalPath      :: Maybe FilePath+    , iterNum       :: Double+    , batchSize     :: Int+    , regVar        :: Double+    , gain0         :: Double+    , tau           :: Double+    , outNerf       :: FilePath }+  | NerMode+    { dataPath      :: FilePath+    , inNerf        :: FilePath }+  | OxMode+    { dataPath      :: FilePath+    , neDictPath    :: FilePath }+  | PnegMode+    { lmfPath       :: FilePath+    , outPath       :: FilePath }+  | NeLexMode+    { nePath        :: FilePath+    , poliPath      :: FilePath+    , outPath       :: FilePath }+  deriving (Data, Typeable, Show)++trainMode :: Args+trainMode = TrainMode+    { trainPath = def &= argPos 0 &= typ "TRAIN-FILE"+    , neDictPath = def &= argPos 1 &= typ "NE-DICT-FILE"+    , evalPath = def &= typFile &= help "Evaluation file"+    , iterNum = 10 &= help "Number of SGD iterations"+    , batchSize = 30 &= help "Batch size"+    , regVar = 10.0 &= help "Regularization variance"+    , gain0 = 1.0 &= help "Initial gain parameter"+    , tau = 5.0 &= help "Initial tau parameter"+    , outNerf = def &= typFile &= help "Output Nerf file" }++nerMode :: Args+nerMode = NerMode+    { inNerf = def &= argPos 0 &= typ "NERF-FILE"+    , dataPath = def &= argPos 2 &= typ "INPUT" }+--     , dataPath = def &= typFile &= help "Input" }+--         &= help "Input file; if not specified, read from stdin" }++oxMode :: Args+oxMode = OxMode+    { dataPath = def &= argPos 0 &= typ "DATA-FILE"+    , neDictPath = def &= argPos 1 &= typ "NE-DICT-FILE" }++pnegMode :: Args+pnegMode = PnegMode+    { lmfPath = def &= typ "PNEG" &= argPos 0+    , outPath = def &= typ "Output" &= argPos 1 }++neLexMode :: Args+neLexMode = NeLexMode+    { nePath = def &= typ "NELexicon" &= argPos 0+    , poliPath = def &= typ "PoliMorf" &= argPos 1+    , outPath = def &= typ "Output" &= argPos 2 }++argModes :: Mode (CmdArgs Args)+argModes = cmdArgsMode $ modes [trainMode, nerMode, oxMode, pnegMode, neLexMode]++main :: IO ()+main = do+    args <- cmdArgsRun argModes+    exec args++exec :: Args -> IO ()++exec TrainMode{..} = do+    cfg  <- defaultCfg neDictPath+    nerf <- train sgdArgs cfg trainPath evalPath+    when (not . null $ outNerf) $ do+        putStrLn $ "\nSaving model in " ++ outNerf ++ "..."+        encodeFile outNerf nerf+  where+    sgdArgs = SGD.SgdArgs+        { SGD.batchSize = batchSize+        , SGD.regVar = regVar+        , SGD.iterNum = iterNum+        , SGD.gain0 = gain0+        , SGD.tau = tau }++exec NerMode{..} = do+    nerf  <- decodeFile inNerf+    input <- readRaw dataPath+    forM_ input $ \sent -> do+        let forest = ner nerf sent+        L.putStrLn (showForest forest)++exec OxMode{..} = do+    cfg  <- defaultCfg neDictPath+    tryOx cfg dataPath++exec PnegMode{..} = preparePNEG lmfPath outPath+exec NeLexMode{..} = prepareNELexicon nePath poliPath outPath++parseRaw :: L.Text -> [[T.Text]]+parseRaw =+    let toStrict = map L.toStrict+    in  map (toStrict . L.words) . L.lines++readRaw :: FilePath -> IO [[T.Text]]+readRaw = fmap parseRaw . L.readFile