diff --git a/README.md b/README.md
new file mode 100644
--- /dev/null
+++ b/README.md
@@ -0,0 +1,157 @@
+Nerf
+====
+
+Nerf is a statistical named entity recognition (NER) tool based on linear-chain
+conditional random fields (CRFs).
+It has been adapted to recognize tree-like structures of NEs (i.e., with
+recursively embedded NEs) by using the joined label tagging method which
+-- for a particular sentence -- works as follows:
+
+ * CRF model is used to determine the most probable sequence of labels,
+ * Extended IOB method is used to decode the sequence into a forerst of NEs.
+
+The extended IOB method also provides the inverse encoding function which is
+needed during the model training.
+
+
+Installation
+=============
+
+It is recommanded to install *nerf* using the
+[Haskell Tool Stack][stack], which you will need to downoload and
+install on your machine beforehand. Then clone this repository into
+a local directory and use `stack` to install the library by running:
+
+ stack install
+
+
+Data formats
+============
+
+The only data encoding supported by Nerf is `UTF-8`.
+
+Training data
+-------------
+
+The current version of Nerf works with a simple data format in which:
+
+ * Each sentence is kept in a separate line,
+ * Named entities are represented with embedded beginning and ending tags,
+ * Contents of individual tags represent named entity types.
+
+For example:
+
+ Church of the Flying Spaghetti Monster .
+
+Text and label values should be escaped by prepending the `\` character before special
+`>`, `<`, `\` and ` ` (space) characters.
+
+Have a look in the `example` directory for an example of a file in the
+appropriate format.
+
+NER input data
+--------------
+
+Below is a list of data formats supported within the NER mode.
+
+### Raw text
+
+Nerf can be used to annotate raw text with named entites. The annotated data
+will be presented in the format which is also used for training and has already
+been described above. Each sentence should be supplied in a separate line --
+currently, Nerf doesn't perform any sentence-level segmentation.
+
+### XCES format
+
+It is also possible to annotate data stored in the XCES format.
+
+
+Training
+========
+
+Once you have an annotated data file `train.nes` (and, optionally, an evaluation
+material `eval.nes`) conformant with the format described above you can train
+the Nerf model using the following command:
+
+ nerf train train.nes -e eval.nes -o model.bin
+
+Run `nerf train --help` to learn more about the program arguments and possible
+training options.
+
+**WARNING**: The `-N` runtime option currently leads to errors in the training
+process and therefore should be not used for the time being.
+
+
+
+Dictionaries
+------------
+
+Nerf supports a list of NE-related dictionaries:
+
+ * [PoliMorf](http://zil.ipipan.waw.pl/PoliMorf),
+ * [NELexicon](http://nlp.pwr.wroc.pl/en/tools-and-resources/nelexicon),
+ * [Gazetteer for Polish Named Entities](http://clip.ipipan.waw.pl/Gazetteer),
+ * [PNET](http://zil.ipipan.waw.pl/PNET),
+ * [Prolexbase](http://zil.ipipan.waw.pl/Prolexbase).
+
+To use the particular dictionary during NER you have to supply it as a
+command line argument during the training process, for example:
+
+ nerf train train.nes --polimorf PoliMorf-0.6.1.tab
+
+
+Named entity recognition
+========================
+
+To annotate the `input.txt` data file using the trained `model.bin` model, run:
+
+ nerf ner model.bin < input.txt
+
+Annotated data will be printed to `stdout`. Data formats currently supported within
+the NER mode has been described above. Run `nerf ner --help` to learn more about the
+additional NER arguments.
+
+
+Server
+======
+
+Nerf provides also a client/server mode. It is handy when, for example,
+you need to annotate a large collection of small files. Loading Nerf model
+from a disk takes considerable amount of time which makes the tagging method
+described above very slow in such a setting.
+
+To start the Nerf server, run:
+
+ nerf server model.bin
+
+You can supply a custom port number using a `--port` option. For example,
+to run the server on the `10101` port, use the following command:
+
+ nerf server model.bin --port 10101
+
+To use the server in a multi-threaded environment, you need to specify the
+`-N` [RTS][ghc-rts] option. A set of options which usually yield good
+server performance is presented in the following example:
+
+ nerf server model.bin +RTS -N -A4M -qg1 -I0
+
+Run `nerf server --help` to learn more about possible server-mode options.
+
+The client mode works just like the tagging mode. The only difference is that,
+instead of supplying your client with a model, you need to specify the port number
+(in case you used a custom one when starting the server; otherwise, the default
+port number will be used).
+
+ nerf client --port 10101 < input.txt > output.nes
+
+Run `nerf client --help` to learn more about the possible client-mode options.
+
+
+[stack]: http://docs.haskellstack.org "Haskell Tool Stack"
+[ghc-rts]: http://www.haskell.org/ghc/docs/latest/html/users_guide/runtime-control.html "GHC runtime system options"
diff --git a/Setup.hs b/Setup.hs
new file mode 100644
--- /dev/null
+++ b/Setup.hs
@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain
diff --git a/Setup.lhs b/Setup.lhs
deleted file mode 100644
--- a/Setup.lhs
+++ /dev/null
@@ -1,4 +0,0 @@
-#! /usr/bin/env runhaskell
-
-> import Distribution.Simple
-> main = defaultMain
diff --git a/app/Main.hs b/app/Main.hs
new file mode 100644
--- /dev/null
+++ b/app/Main.hs
@@ -0,0 +1,362 @@
+{-# LANGUAGE DeriveDataTypeable #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE RecordWildCards #-}
+
+import System.Console.CmdArgs
+import System.IO
+ ( Handle, hGetBuffering, hSetBuffering
+ , stdout, BufferMode (..), hClose, hFlush )
+import System.IO.Unsafe (unsafePerformIO)
+import qualified System.IO.Temp as Temp
+import qualified Network as N
+import System.Directory (getDirectoryContents)
+import System.FilePath (takeBaseName, (), (<.>))
+import Control.Applicative ((<$>), (<*>))
+import Control.Arrow (second)
+import Control.Monad (forM_)
+import Data.Maybe (catMaybes)
+import Data.Binary (encodeFile, decodeFile)
+import Data.Text.Binary ()
+import Text.Named.Enamex (parseEnamex, showForest)
+import qualified Data.Foldable as F
+import qualified Data.Map as M
+import qualified Numeric.SGD as SGD
+import qualified Data.Text as T
+import qualified Data.Text.Lazy as L
+import qualified Data.Text.Lazy.IO as L
+import qualified Data.DAWG.Static as D
+
+import NLP.Nerf (train, ner, tryOx)
+import NLP.Nerf.Schema (defaultConf)
+import NLP.Nerf.Dict
+ ( extractPoliMorf, extractPNEG, extractNELexicon, extractProlexbase
+ , extractIntTriggers, extractExtTriggers, Dict )
+import NLP.Nerf.XCES as XCES
+import qualified NLP.Nerf.Server as S
+
+import NLP.Nerf.Compare ((.+.))
+import qualified NLP.Nerf.Compare as C
+
+
+-- | Default port number.
+portDefault :: Int
+portDefault = 10090
+
+
+---------------------------------------
+-- Command line options
+---------------------------------------
+
+
+-- | Data formats.
+data Format
+ = Text
+ | XCES
+ deriving (Data, Typeable, Show)
+
+
+data Nerf
+ = Train
+ { trainPath :: FilePath
+ , evalPath :: Maybe FilePath
+ , poliMorf :: Maybe FilePath
+ , prolex :: Maybe FilePath
+ , pneg :: Maybe FilePath
+ , neLex :: Maybe FilePath
+ , pnet :: Maybe FilePath
+ , iterNum :: Double
+ , batchSize :: Int
+ , regVar :: Double
+ , gain0 :: Double
+ , tau :: Double
+ , outNerf :: Maybe FilePath }
+ | CV
+ { dataDir :: FilePath
+ , poliMorf :: Maybe FilePath
+ , prolex :: Maybe FilePath
+ , pneg :: Maybe FilePath
+ , neLex :: Maybe FilePath
+ , pnet :: Maybe FilePath
+ , iterNum :: Double
+ , batchSize :: Int
+ , regVar :: Double
+ , gain0 :: Double
+ , tau :: Double
+ , outDir :: Maybe FilePath }
+ | NER
+ { inModel :: FilePath
+ , format :: Format }
+ | Server
+ { inModel :: FilePath
+ , port :: Int }
+ | Client
+ { format :: Format
+ , host :: String
+ , port :: Int }
+ | Ox
+ { dataPath :: FilePath
+ , poliMorf :: Maybe FilePath
+ , prolex :: Maybe FilePath
+ , pneg :: Maybe FilePath
+ , neLex :: Maybe FilePath
+ , pnet :: Maybe FilePath }
+ | Compare
+ { dataPath :: FilePath
+ , dataPath' :: FilePath }
+ deriving (Data, Typeable, Show)
+
+
+trainMode :: Nerf
+trainMode = Train
+ { trainPath = def &= argPos 0 &= typ "TRAIN-FILE"
+ , evalPath = def &= typFile &= help "Evaluation file"
+ , poliMorf = def &= typFile &= help "Path to PoliMorf"
+ , prolex = def &= typFile &= help "Path to Prolexbase"
+ , pneg = def &= typFile &= help "Path to PNEG-LMF"
+ , neLex = def &= typFile &= help "Path to NELexicon"
+ , pnet = def &= typFile &= help "Path to PNET"
+ , iterNum = 10 &= help "Number of SGD iterations"
+ , batchSize = 30 &= help "Batch size"
+ , regVar = 10.0 &= help "Regularization variance"
+ , gain0 = 1.0 &= help "Initial gain parameter"
+ , tau = 5.0 &= help "Initial tau parameter"
+ , outNerf = def &= typFile &= help "Output model file" }
+
+
+cvMode :: Nerf
+cvMode = CV
+ { dataDir = def &= argPos 0 &= typ "DATA-DIR"
+ , poliMorf = def &= typFile &= help "Path to PoliMorf"
+ , prolex = def &= typFile &= help "Path to Prolexbase"
+ , pneg = def &= typFile &= help "Path to PNEG-LMF"
+ , neLex = def &= typFile &= help "Path to NELexicon"
+ , pnet = def &= typFile &= help "Path to PNET"
+ , iterNum = 10 &= help "Number of SGD iterations"
+ , batchSize = 30 &= help "Batch size"
+ , regVar = 10.0 &= help "Regularization variance"
+ , gain0 = 1.0 &= help "Initial gain parameter"
+ , tau = 5.0 &= help "Initial tau parameter"
+ , outDir = def &= typFile &= help "Output model directory" }
+
+
+nerMode :: Nerf
+nerMode = NER
+ { inModel = def &= argPos 0 &= typ "MODEL-FILE"
+ , format = enum
+ [ Text &= help "Raw text"
+ , XCES &= help "XCES" ] }
+
+
+serverMode :: Nerf
+serverMode = Server
+ { inModel = def &= argPos 0 &= typ "MODEL-FILE"
+ , port = portDefault &= help "Port number" }
+
+
+clientMode :: Nerf
+clientMode = Client
+ { port = portDefault &= help "Port number"
+ , host = "localhost" &= help "Server host name"
+ , format = enum
+ [ Text &= help "Raw text"
+ , XCES &= help "XCES" ] }
+
+
+oxMode :: Nerf
+oxMode = Ox
+ { dataPath = def &= argPos 0 &= typ "DATA-FILE"
+ , poliMorf = def &= typFile &= help "Path to PoliMorf"
+ , prolex = def &= typFile &= help "Path to Prolexbase"
+ , pneg = def &= typFile &= help "Path to PNEG-LMF"
+ , neLex = def &= typFile &= help "Path to NELexicon"
+ , pnet = def &= typFile &= help "Path to PNET" }
+
+
+cmpMode :: Nerf
+cmpMode = Compare
+ { dataPath = def &= argPos 0 &= typ "REFERENCE"
+ , dataPath' = def &= argPos 1 &= typ "COMPARED" }
+
+
+argModes :: Mode (CmdArgs Nerf)
+argModes = cmdArgsMode $ modes
+ [trainMode, cvMode, nerMode, serverMode, clientMode, cmpMode, oxMode]
+
+
+data Resources = Resources
+ { poliDict :: Maybe Dict
+ , prolexDict :: Maybe Dict
+ , pnegDict :: Maybe Dict
+ , neLexDict :: Maybe Dict
+ , intDict :: Maybe Dict
+ , extDict :: Maybe Dict }
+
+
+extract :: Nerf -> IO Resources
+extract nerf = withBuffering stdout NoBuffering $ Resources
+ <$> extractDict "PoliMorf" extractPoliMorf (poliMorf nerf)
+ <*> extractDict "Prolexbase" extractProlexbase (prolex nerf)
+ <*> extractDict "PNEG" extractPNEG (pneg nerf)
+ <*> extractDict "NELexicon" extractNELexicon (neLex nerf)
+ <*> extractDict "internal triggers" extractIntTriggers (pnet nerf)
+ <*> extractDict "external triggers" extractExtTriggers (pnet nerf)
+
+
+withBuffering :: Handle -> BufferMode -> IO a -> IO a
+withBuffering h mode io = do
+ oldMode <- hGetBuffering h
+ hSetBuffering h mode
+ x <- io
+ hSetBuffering h oldMode
+ return x
+
+
+extractDict :: String -> (a -> IO Dict) -> Maybe a -> IO (Maybe Dict)
+extractDict msg f (Just x) = do
+ putStr $ "Reading " ++ msg ++ "..."
+ dict <- f x
+ let k = D.numStates dict
+ k `seq` putStrLn $ " Done"
+ putStrLn $ "Number of automaton states = " ++ show k
+ return (Just dict)
+extractDict _ _ Nothing = return Nothing
+
+
+main :: IO ()
+main = exec =<< cmdArgsRun argModes
+
+
+exec :: Nerf -> IO ()
+
+
+exec nerfArgs@Train{..} = do
+ Resources{..} <- extract nerfArgs
+ cfg <- defaultConf
+ (catMaybes [poliDict, prolexDict, pnegDict, neLexDict])
+ intDict extDict
+ nerf <- train sgdArgs cfg trainPath evalPath
+ flip F.traverse_ outNerf $ \path -> do
+ putStrLn $ "\nSaving model in " ++ path ++ "..."
+ encodeFile path nerf
+ where
+ sgdArgs = SGD.SgdArgs
+ { SGD.batchSize = batchSize
+ , SGD.regVar = regVar
+ , SGD.iterNum = iterNum
+ , SGD.gain0 = gain0
+ , SGD.tau = tau }
+
+
+exec nerfArgs@CV{..} = do
+ Resources{..} <- extract nerfArgs
+ cfg <- defaultConf
+ (catMaybes [poliDict, prolexDict, pnegDict, neLexDict])
+ intDict extDict
+ parts <- getParts dataDir
+ forM_ (enumDivs parts) $ \(evalPath, trainPaths) -> do
+ putStrLn $ "\nPart: " ++ evalPath
+ withParts trainPaths $ \trainPath -> do
+ nerf <- train sgdArgs cfg trainPath (Just evalPath)
+ flip F.traverse_ outDir $ \dir -> do
+ let path = dir takeBaseName evalPath <.> ".bin"
+ putStrLn $ "\nSaving model in " ++ path ++ "..."
+ encodeFile path nerf
+ where
+ sgdArgs = SGD.SgdArgs
+ { SGD.batchSize = batchSize
+ , SGD.regVar = regVar
+ , SGD.iterNum = iterNum
+ , SGD.gain0 = gain0
+ , SGD.tau = tau }
+
+
+exec NER{..} = case format of
+ Text -> do
+ nerf <- decodeFile inModel
+ inp <- L.lines <$> L.getContents
+ forM_ inp $ \sent -> do
+ let forest = ner nerf (L.unpack sent)
+ L.putStrLn (showForest forest)
+ XCES -> do
+ nerf <- decodeFile inModel
+ L.putStrLn . XCES.nerXCES (ner nerf) =<< L.getContents
+
+
+exec Server{..} = do
+ putStr "Loading model..." >> hFlush stdout
+ nerf <- decodeFile inModel
+ nerf `seq` putStrLn " done"
+ let portNum = N.PortNumber $ fromIntegral port
+ putStrLn $ "Listening on port " ++ show port
+ S.runNerfServer nerf portNum
+
+
+exec Client{..} = case format of
+ Text -> do
+ inp <- L.lines <$> L.getContents
+ forM_ inp $ \sent -> do
+ forest <- S.ner host portNum $ L.unpack sent
+ L.putStrLn (showForest forest)
+ XCES -> do
+ let nerRemote = unsafePerformIO . S.ner host portNum
+ L.putStrLn . XCES.nerXCES nerRemote =<< L.getContents
+ where
+ portNum = N.PortNumber $ fromIntegral port
+
+
+exec nerfArgs@Ox{..} = do
+ Resources{..} <- extract nerfArgs
+ cfg <- defaultConf
+ (catMaybes [poliDict, prolexDict, pnegDict, neLexDict])
+ intDict extDict
+ tryOx cfg dataPath
+
+
+exec Compare{..} = do
+ x <- parseEnamex <$> L.readFile dataPath
+ y <- parseEnamex <$> L.readFile dataPath'
+ let statMap = C.compare $ zip x y
+ forM_ (M.toList statMap) $ uncurry printStats
+ printStats "" (foldl1 (.+.) $ M.elems statMap)
+ where
+ printStats neType stats = do
+ putStrLn $ "# " ++ T.unpack neType
+ putStrLn $ "true positive: " ++ show (C.tp stats)
+ putStrLn $ "false positive: " ++ show (C.fp stats)
+ -- putStrLn $ "true negative: " ++ show (C.tn stats)
+ putStrLn $ "false negative: " ++ show (C.fn stats)
+
+
+-- readRaw :: FilePath -> IO [L.Text]
+-- readRaw = fmap L.lines . L.readFile
+
+
+----------------------------------------
+-- Cross-validation
+----------------------------------------
+
+
+-- | Get paths of the individual parts of the dataset
+-- stored in the given directory.
+getParts :: FilePath -> IO [FilePath]
+getParts path = do
+ xs <- filter (\x -> not (x `elem` [".", ".."]))
+ <$> getDirectoryContents path
+ return $ map (path ) xs
+
+
+-- | Take data from the given list of paths and store
+-- it all in a temporary file, than run the given handler.
+withParts :: [FilePath] -> (FilePath -> IO a) -> IO a
+withParts paths handler = Temp.withSystemTempFile "train." $ \tempPath _h -> do
+ hClose _h
+ forM_ paths $ \srcPath -> do
+ L.readFile srcPath >>= L.appendFile tempPath
+ handler tempPath
+
+
+-- | Enumerate subsequent partitionings of the dataset.
+enumDivs :: [a] -> [(a, [a])]
+enumDivs [] = []
+enumDivs (x:xs) = (x, xs) : map (second (x:)) (enumDivs xs)
diff --git a/nerf.cabal b/nerf.cabal
--- a/nerf.cabal
+++ b/nerf.cabal
@@ -1,81 +1,103 @@
-name: nerf
-version: 0.5.3
-synopsis: Nerf, the named entity recognition tool based on linear-chain CRFs
-description:
- The package provides the named entity recognition (NER) tool divided into a
- back-end library (see the "NLP.Nerf" module) and the front-end tool nerf.
- Using the library you can model and recognize named entities (NEs) which,
- for a particular sentence, take the form of forest with NE category values
- kept in internal nodes and sentence words kept in forest leaves.
- .
- To model NE forests we combine two different techniques. The IOB codec
- is used to translate to and fro between the original, forest representation
- of NEs and the sequence of atomic labels. In other words, it provides two
- isomorphic functions for encoding and decoding between both
- representations. Linear-chain conditional random fields, on the other hand,
- provide the framework for label modelling and tagging.
-license: BSD3
-license-file: LICENSE
-cabal-version: >= 1.6
-copyright: Copyright (c) 2012 IPI PAN
-author: Jakub Waszczuk
-maintainer: waszczuk.kuba@gmail.com
-stability: experimental
-category: Natural Language Processing
-homepage: https://github.com/kawu/nerf
-build-type: Simple
-
-library
- hs-source-dirs: src
-
- build-depends:
- base >= 4 && < 5
- , containers
- , vector
- , text
- , binary
- , bytestring >= 0.9 && < 0.11
- , text-binary >= 0.1 && < 0.2
- , tagsoup >= 0.13 && < 0.14
- , polysoup >= 0.2 && < 0.3
- , crf-chain1 >= 0.2 && < 0.3
- , data-named >= 0.5.1 && < 0.6
- , monad-ox >= 0.2 && < 0.3
- , sgd >= 0.2.1 && < 0.3
- , polimorf >= 0.6.0 && < 0.7
- , dawg >= 0.8.1 && < 0.9
- , tokenize == 0.1.3
- , mtl >= 2.1 && < 2.3
- , network >= 2.3 && < 2.7
- , cmdargs >= 0.10 && < 0.11
- , IntervalMap >= 0.3 && < 0.4
+cabal-version: 1.12
- exposed-modules:
- NLP.Nerf
- , NLP.Nerf.Types
- , NLP.Nerf.Schema
- , NLP.Nerf.Tokenize
- , NLP.Nerf.Dict
- , NLP.Nerf.Dict.Base
- , NLP.Nerf.Dict.PNEG
- , NLP.Nerf.Dict.PNET
- , NLP.Nerf.Dict.NELexicon
- , NLP.Nerf.Dict.Prolexbase
- , NLP.Nerf.Compare
- , NLP.Nerf.Server
- , NLP.Nerf.XCES
+-- This file has been generated from package.yaml by hpack version 0.31.1.
+--
+-- see: https://github.com/sol/hpack
+--
+-- hash: ad5f136e7ed38e4e292f8ffc7c67171e2a4bc994ea6c43fb33ec1babf6894a9d
- ghc-options: -Wall -O2
+name: nerf
+version: 0.5.4
+synopsis: Nerf, a named entity recognition tool based on linear-chain CRFs
+description: Please see the README on GitHub at
+category: Natural Language Processing
+homepage: https://github.com/kawu/nerf#readme
+bug-reports: https://github.com/kawu/nerf/issues
+author: Jakub Waszczuk
+maintainer: waszczuk.kuba@gmail.com
+copyright: 2012-2019 IPI PAN, Jakub Waszczuk
+license: BSD3
+license-file: LICENSE
+build-type: Simple
+extra-source-files:
+ README.md
source-repository head
- type: git
- location: https://github.com/kawu/nerf.git
+ type: git
+ location: https://github.com/kawu/nerf
+library
+ exposed-modules:
+ NLP.Nerf
+ NLP.Nerf.Compare
+ NLP.Nerf.Dict
+ NLP.Nerf.Dict.Base
+ NLP.Nerf.Dict.NELexicon
+ NLP.Nerf.Dict.PNEG
+ NLP.Nerf.Dict.PNET
+ NLP.Nerf.Dict.Prolexbase
+ NLP.Nerf.Schema
+ NLP.Nerf.Server
+ NLP.Nerf.Tokenize
+ NLP.Nerf.Types
+ NLP.Nerf.XCES
+ other-modules:
+ Paths_nerf
+ hs-source-dirs:
+ src
+ build-depends:
+ IntervalMap >=0.6 && <0.7
+ , base >=4.7 && <5
+ , binary
+ , bytestring >=0.9 && <0.11
+ , cmdargs >=0.10 && <0.11
+ , containers >=0.5 && <0.7
+ , crf-chain1 >=0.2 && <0.3
+ , data-named >=0.6.1 && <0.7
+ , dawg >=0.8.2 && <0.9
+ , monad-ox >=0.2 && <0.3
+ , mtl >=2.1 && <2.3
+ , network >=2.3 && <2.9
+ , polimorf >=0.7.4 && <0.8
+ , polysoup >=0.2 && <0.3
+ , sgd >=0.2.3 && <0.3
+ , tagsoup >=0.13 && <0.15
+ , text
+ , text-binary >=0.1 && <0.3
+ , tokenize ==0.3.0
+ , vector
+ default-language: Haskell2010
+
executable nerf
- hs-source-dirs: src, tools
+ main-is: Main.hs
+ other-modules:
+ Paths_nerf
+ hs-source-dirs:
+ app
+ ghc-options: -threaded -rtsopts
build-depends:
- filepath >= 1.3 && < 1.4,
- directory >= 1.1 && < 1.3,
- temporary >= 1.1 && < 1.2
- main-is: nerf.hs
- ghc-options: -Wall -O2 -threaded -rtsopts
+ IntervalMap >=0.6 && <0.7
+ , base >=4.7 && <5
+ , binary
+ , bytestring >=0.9 && <0.11
+ , cmdargs >=0.10 && <0.11
+ , containers >=0.5 && <0.7
+ , crf-chain1 >=0.2 && <0.3
+ , data-named >=0.6.1 && <0.7
+ , dawg >=0.8.2 && <0.9
+ , directory >=1.1 && <1.4
+ , filepath >=1.3 && <1.5
+ , monad-ox >=0.2 && <0.3
+ , mtl >=2.1 && <2.3
+ , nerf
+ , network >=2.3 && <2.9
+ , polimorf >=0.7.4 && <0.8
+ , polysoup >=0.2 && <0.3
+ , sgd >=0.2.3 && <0.3
+ , tagsoup >=0.13 && <0.15
+ , temporary >=1.1 && <1.4
+ , text
+ , text-binary >=0.1 && <0.3
+ , tokenize ==0.3.0
+ , vector
+ default-language: Haskell2010
diff --git a/src/NLP/Nerf.hs b/src/NLP/Nerf.hs
--- a/src/NLP/Nerf.hs
+++ b/src/NLP/Nerf.hs
@@ -11,6 +11,8 @@
, module NLP.Nerf.Types
) where
+import Prelude hiding (Word)
+
import Control.Applicative ((<$>), (<*>))
import Data.Binary (Binary, put, get)
import Data.Foldable (foldMap)
diff --git a/src/NLP/Nerf/Schema.hs b/src/NLP/Nerf/Schema.hs
--- a/src/NLP/Nerf/Schema.hs
+++ b/src/NLP/Nerf/Schema.hs
@@ -39,6 +39,8 @@
, dictB
) where
+import Prelude hiding (Word)
+
import Control.Applicative ((<$>), (<*>))
import Control.Monad (forM_, join)
import Data.Maybe (maybeToList)
diff --git a/src/NLP/Nerf/Server.hs b/src/NLP/Nerf/Server.hs
--- a/src/NLP/Nerf/Server.hs
+++ b/src/NLP/Nerf/Server.hs
@@ -7,6 +7,7 @@
, ner
) where
+import Prelude hiding (Word)
import Control.Applicative ((<$>))
import Control.Monad (forever, void)
diff --git a/src/NLP/Nerf/Tokenize.hs b/src/NLP/Nerf/Tokenize.hs
--- a/src/NLP/Nerf/Tokenize.hs
+++ b/src/NLP/Nerf/Tokenize.hs
@@ -16,6 +16,8 @@
) where
+import Prelude hiding (Word)
+
import Control.Arrow (second)
import Control.Monad ((>=>))
import qualified Data.Char as Char
@@ -141,7 +143,7 @@
where
replace im (Left x) = (im, Left x)
replace im (Right (ran, _)) =
- let rsXs = I.intersecting im ran
+ let rsXs = I.assocs $ I.intersecting im ran
im' = L.foldl' (flip I.delete) im (map fst rsXs)
in (im', Right rsXs)
diff --git a/src/NLP/Nerf/Types.hs b/src/NLP/Nerf/Types.hs
--- a/src/NLP/Nerf/Types.hs
+++ b/src/NLP/Nerf/Types.hs
@@ -7,6 +7,8 @@
, Lb
) where
+import Prelude hiding (Word)
+
import qualified Data.Text as T
import qualified Data.Named.IOB as IOB
diff --git a/src/NLP/Nerf/XCES.hs b/src/NLP/Nerf/XCES.hs
--- a/src/NLP/Nerf/XCES.hs
+++ b/src/NLP/Nerf/XCES.hs
@@ -10,6 +10,8 @@
) where
+import Prelude hiding (Word)
+
import qualified Data.Text.Lazy as L
import Data.List (intercalate, intersperse)
import Data.Char (isSpace)
diff --git a/tools/nerf.hs b/tools/nerf.hs
deleted file mode 100644
--- a/tools/nerf.hs
+++ /dev/null
@@ -1,362 +0,0 @@
-{-# LANGUAGE DeriveDataTypeable #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE RecordWildCards #-}
-
-import System.Console.CmdArgs
-import System.IO
- ( Handle, hGetBuffering, hSetBuffering
- , stdout, BufferMode (..), hClose, hFlush )
-import System.IO.Unsafe (unsafePerformIO)
-import qualified System.IO.Temp as Temp
-import qualified Network as N
-import System.Directory (getDirectoryContents)
-import System.FilePath (takeBaseName, (), (<.>))
-import Control.Applicative ((<$>), (<*>))
-import Control.Arrow (second)
-import Control.Monad (forM_)
-import Data.Maybe (catMaybes)
-import Data.Binary (encodeFile, decodeFile)
-import Data.Text.Binary ()
-import Text.Named.Enamex (parseEnamex, showForest)
-import qualified Data.Foldable as F
-import qualified Data.Map as M
-import qualified Numeric.SGD as SGD
-import qualified Data.Text as T
-import qualified Data.Text.Lazy as L
-import qualified Data.Text.Lazy.IO as L
-import qualified Data.DAWG.Static as D
-
-import NLP.Nerf (train, ner, tryOx)
-import NLP.Nerf.Schema (defaultConf)
-import NLP.Nerf.Dict
- ( extractPoliMorf, extractPNEG, extractNELexicon, extractProlexbase
- , extractIntTriggers, extractExtTriggers, Dict )
-import NLP.Nerf.XCES as XCES
-import qualified NLP.Nerf.Server as S
-
-import NLP.Nerf.Compare ((.+.))
-import qualified NLP.Nerf.Compare as C
-
-
--- | Default port number.
-portDefault :: Int
-portDefault = 10090
-
-
----------------------------------------
--- Command line options
----------------------------------------
-
-
--- | Data formats.
-data Format
- = Text
- | XCES
- deriving (Data, Typeable, Show)
-
-
-data Nerf
- = Train
- { trainPath :: FilePath
- , evalPath :: Maybe FilePath
- , poliMorf :: Maybe FilePath
- , prolex :: Maybe FilePath
- , pneg :: Maybe FilePath
- , neLex :: Maybe FilePath
- , pnet :: Maybe FilePath
- , iterNum :: Double
- , batchSize :: Int
- , regVar :: Double
- , gain0 :: Double
- , tau :: Double
- , outNerf :: Maybe FilePath }
- | CV
- { dataDir :: FilePath
- , poliMorf :: Maybe FilePath
- , prolex :: Maybe FilePath
- , pneg :: Maybe FilePath
- , neLex :: Maybe FilePath
- , pnet :: Maybe FilePath
- , iterNum :: Double
- , batchSize :: Int
- , regVar :: Double
- , gain0 :: Double
- , tau :: Double
- , outDir :: Maybe FilePath }
- | NER
- { inModel :: FilePath
- , format :: Format }
- | Server
- { inModel :: FilePath
- , port :: Int }
- | Client
- { format :: Format
- , host :: String
- , port :: Int }
- | Ox
- { dataPath :: FilePath
- , poliMorf :: Maybe FilePath
- , prolex :: Maybe FilePath
- , pneg :: Maybe FilePath
- , neLex :: Maybe FilePath
- , pnet :: Maybe FilePath }
- | Compare
- { dataPath :: FilePath
- , dataPath' :: FilePath }
- deriving (Data, Typeable, Show)
-
-
-trainMode :: Nerf
-trainMode = Train
- { trainPath = def &= argPos 0 &= typ "TRAIN-FILE"
- , evalPath = def &= typFile &= help "Evaluation file"
- , poliMorf = def &= typFile &= help "Path to PoliMorf"
- , prolex = def &= typFile &= help "Path to Prolexbase"
- , pneg = def &= typFile &= help "Path to PNEG-LMF"
- , neLex = def &= typFile &= help "Path to NELexicon"
- , pnet = def &= typFile &= help "Path to PNET"
- , iterNum = 10 &= help "Number of SGD iterations"
- , batchSize = 30 &= help "Batch size"
- , regVar = 10.0 &= help "Regularization variance"
- , gain0 = 1.0 &= help "Initial gain parameter"
- , tau = 5.0 &= help "Initial tau parameter"
- , outNerf = def &= typFile &= help "Output model file" }
-
-
-cvMode :: Nerf
-cvMode = CV
- { dataDir = def &= argPos 0 &= typ "DATA-DIR"
- , poliMorf = def &= typFile &= help "Path to PoliMorf"
- , prolex = def &= typFile &= help "Path to Prolexbase"
- , pneg = def &= typFile &= help "Path to PNEG-LMF"
- , neLex = def &= typFile &= help "Path to NELexicon"
- , pnet = def &= typFile &= help "Path to PNET"
- , iterNum = 10 &= help "Number of SGD iterations"
- , batchSize = 30 &= help "Batch size"
- , regVar = 10.0 &= help "Regularization variance"
- , gain0 = 1.0 &= help "Initial gain parameter"
- , tau = 5.0 &= help "Initial tau parameter"
- , outDir = def &= typFile &= help "Output model directory" }
-
-
-nerMode :: Nerf
-nerMode = NER
- { inModel = def &= argPos 0 &= typ "MODEL-FILE"
- , format = enum
- [ Text &= help "Raw text"
- , XCES &= help "XCES" ] }
-
-
-serverMode :: Nerf
-serverMode = Server
- { inModel = def &= argPos 0 &= typ "MODEL-FILE"
- , port = portDefault &= help "Port number" }
-
-
-clientMode :: Nerf
-clientMode = Client
- { port = portDefault &= help "Port number"
- , host = "localhost" &= help "Server host name"
- , format = enum
- [ Text &= help "Raw text"
- , XCES &= help "XCES" ] }
-
-
-oxMode :: Nerf
-oxMode = Ox
- { dataPath = def &= argPos 0 &= typ "DATA-FILE"
- , poliMorf = def &= typFile &= help "Path to PoliMorf"
- , prolex = def &= typFile &= help "Path to Prolexbase"
- , pneg = def &= typFile &= help "Path to PNEG-LMF"
- , neLex = def &= typFile &= help "Path to NELexicon"
- , pnet = def &= typFile &= help "Path to PNET" }
-
-
-cmpMode :: Nerf
-cmpMode = Compare
- { dataPath = def &= argPos 0 &= typ "REFERENCE"
- , dataPath' = def &= argPos 1 &= typ "COMPARED" }
-
-
-argModes :: Mode (CmdArgs Nerf)
-argModes = cmdArgsMode $ modes
- [trainMode, cvMode, nerMode, serverMode, clientMode, cmpMode, oxMode]
-
-
-data Resources = Resources
- { poliDict :: Maybe Dict
- , prolexDict :: Maybe Dict
- , pnegDict :: Maybe Dict
- , neLexDict :: Maybe Dict
- , intDict :: Maybe Dict
- , extDict :: Maybe Dict }
-
-
-extract :: Nerf -> IO Resources
-extract nerf = withBuffering stdout NoBuffering $ Resources
- <$> extractDict "PoliMorf" extractPoliMorf (poliMorf nerf)
- <*> extractDict "Prolexbase" extractProlexbase (prolex nerf)
- <*> extractDict "PNEG" extractPNEG (pneg nerf)
- <*> extractDict "NELexicon" extractNELexicon (neLex nerf)
- <*> extractDict "internal triggers" extractIntTriggers (pnet nerf)
- <*> extractDict "external triggers" extractExtTriggers (pnet nerf)
-
-
-withBuffering :: Handle -> BufferMode -> IO a -> IO a
-withBuffering h mode io = do
- oldMode <- hGetBuffering h
- hSetBuffering h mode
- x <- io
- hSetBuffering h oldMode
- return x
-
-
-extractDict :: String -> (a -> IO Dict) -> Maybe a -> IO (Maybe Dict)
-extractDict msg f (Just x) = do
- putStr $ "Reading " ++ msg ++ "..."
- dict <- f x
- let k = D.numStates dict
- k `seq` putStrLn $ " Done"
- putStrLn $ "Number of automaton states = " ++ show k
- return (Just dict)
-extractDict _ _ Nothing = return Nothing
-
-
-main :: IO ()
-main = exec =<< cmdArgsRun argModes
-
-
-exec :: Nerf -> IO ()
-
-
-exec nerfArgs@Train{..} = do
- Resources{..} <- extract nerfArgs
- cfg <- defaultConf
- (catMaybes [poliDict, prolexDict, pnegDict, neLexDict])
- intDict extDict
- nerf <- train sgdArgs cfg trainPath evalPath
- flip F.traverse_ outNerf $ \path -> do
- putStrLn $ "\nSaving model in " ++ path ++ "..."
- encodeFile path nerf
- where
- sgdArgs = SGD.SgdArgs
- { SGD.batchSize = batchSize
- , SGD.regVar = regVar
- , SGD.iterNum = iterNum
- , SGD.gain0 = gain0
- , SGD.tau = tau }
-
-
-exec nerfArgs@CV{..} = do
- Resources{..} <- extract nerfArgs
- cfg <- defaultConf
- (catMaybes [poliDict, prolexDict, pnegDict, neLexDict])
- intDict extDict
- parts <- getParts dataDir
- forM_ (enumDivs parts) $ \(evalPath, trainPaths) -> do
- putStrLn $ "\nPart: " ++ evalPath
- withParts trainPaths $ \trainPath -> do
- nerf <- train sgdArgs cfg trainPath (Just evalPath)
- flip F.traverse_ outDir $ \dir -> do
- let path = dir takeBaseName evalPath <.> ".bin"
- putStrLn $ "\nSaving model in " ++ path ++ "..."
- encodeFile path nerf
- where
- sgdArgs = SGD.SgdArgs
- { SGD.batchSize = batchSize
- , SGD.regVar = regVar
- , SGD.iterNum = iterNum
- , SGD.gain0 = gain0
- , SGD.tau = tau }
-
-
-exec NER{..} = case format of
- Text -> do
- nerf <- decodeFile inModel
- inp <- L.lines <$> L.getContents
- forM_ inp $ \sent -> do
- let forest = ner nerf (L.unpack sent)
- L.putStrLn (showForest forest)
- XCES -> do
- nerf <- decodeFile inModel
- L.putStrLn . XCES.nerXCES (ner nerf) =<< L.getContents
-
-
-exec Server{..} = do
- putStr "Loading model..." >> hFlush stdout
- nerf <- decodeFile inModel
- nerf `seq` putStrLn " done"
- let portNum = N.PortNumber $ fromIntegral port
- putStrLn $ "Listening on port " ++ show port
- S.runNerfServer nerf portNum
-
-
-exec Client{..} = case format of
- Text -> do
- inp <- L.lines <$> L.getContents
- forM_ inp $ \sent -> do
- forest <- S.ner host portNum $ L.unpack sent
- L.putStrLn (showForest forest)
- XCES -> do
- let nerRemote = unsafePerformIO . S.ner host portNum
- L.putStrLn . XCES.nerXCES nerRemote =<< L.getContents
- where
- portNum = N.PortNumber $ fromIntegral port
-
-
-exec nerfArgs@Ox{..} = do
- Resources{..} <- extract nerfArgs
- cfg <- defaultConf
- (catMaybes [poliDict, prolexDict, pnegDict, neLexDict])
- intDict extDict
- tryOx cfg dataPath
-
-
-exec Compare{..} = do
- x <- parseEnamex <$> L.readFile dataPath
- y <- parseEnamex <$> L.readFile dataPath'
- let statMap = C.compare $ zip x y
- forM_ (M.toList statMap) $ uncurry printStats
- printStats "" (foldl1 (.+.) $ M.elems statMap)
- where
- printStats neType stats = do
- putStrLn $ "# " ++ T.unpack neType
- putStrLn $ "true positive: " ++ show (C.tp stats)
- putStrLn $ "false positive: " ++ show (C.fp stats)
- -- putStrLn $ "true negative: " ++ show (C.tn stats)
- putStrLn $ "false negative: " ++ show (C.fn stats)
-
-
--- readRaw :: FilePath -> IO [L.Text]
--- readRaw = fmap L.lines . L.readFile
-
-
-----------------------------------------
--- Cross-validation
-----------------------------------------
-
-
--- | Get paths of the individual parts of the dataset
--- stored in the given directory.
-getParts :: FilePath -> IO [FilePath]
-getParts path = do
- xs <- filter (\x -> not (x `elem` [".", ".."]))
- <$> getDirectoryContents path
- return $ map (path ) xs
-
-
--- | Take data from the given list of paths and store
--- it all in a temporary file, than run the given handler.
-withParts :: [FilePath] -> (FilePath -> IO a) -> IO a
-withParts paths handler = Temp.withSystemTempFile "train." $ \tempPath _h -> do
- hClose _h
- forM_ paths $ \srcPath -> do
- L.readFile srcPath >>= L.appendFile tempPath
- handler tempPath
-
-
--- | Enumerate subsequent partitionings of the dataset.
-enumDivs :: [a] -> [(a, [a])]
-enumDivs [] = []
-enumDivs (x:xs) = (x, xs) : map (second (x:)) (enumDivs xs)