sequor 0.2.3 → 0.3.0
raw patch · 12 files changed
+249/−10 lines, 12 filesnew-component:exe:augment
Files
- Makefile +47/−6
- README +4/−1
- bin/seminer +22/−0
- data/seminer/deu.eci.unlabeled.500+1000.classes too large to diff
- data/seminer/deu.train.lrec-2010-raw.model too large to diff
- data/seminer/deu.train.lrec-2010.model too large to diff
- data/seminer/lrec-2010-raw.features +19/−0
- data/seminer/lrec-2010.features +24/−0
- lib/seminer/README +36/−0
- lib/seminer/augment.hs +59/−0
- lib/seminer/seminer +22/−0
- sequor.cabal +16/−3
Makefile view
@@ -1,20 +1,61 @@ INCLUDES=lib -all:run- --run: - ghc --make -O2 -isrc:$(INCLUDES) -o bin/sequor src/ghc_rts_opts.c src/Main.hs- cabal: runghc Setup.lhs configure --user --prefix=`pwd` --bindir=`pwd`/bin/ &&\ runghc Setup.lhs build && runghc Setup.lhs install + cabal-static: runghc Setup.lhs configure --user --prefix=`pwd` --bindir=`pwd`/bin/ \ --ghc-option="-optl-static" --ghc-option="-optl-pthread" &&\ runghc Setup.lhs build && runghc Setup.lhs install -rm -rf dist -rm -rf share++train-seminer: data/seminer/deu.train.lrec-2010-raw.model \+ data/seminer/deu.train.lrec-2010.model ++data/seminer/deu.train.lrec-2010.model: data/seminer/lrec-2010.features \+ data/seminer/deu.train.bio.brown \+ data/seminer/deu.testa.bio.brown+ ./bin/sequor train data/seminer/lrec-2010.features \+ data/seminer/deu.train.bio.brown \+ data/seminer/deu.train.lrec-2010.model --hash \+ --heldout data/seminer/deu.testa.bio.brown --iter 20 \+ --min-count 100 --rate 0.001 --beam 9++data/seminer/deu.train.lrec-2010-raw.model: data/seminer/lrec-2010-raw.features \+ data/seminer/deu.train.bio.raw.brown \+ data/seminer/deu.testa.bio.raw.brown+ ./bin/sequor train data/seminer/lrec-2010-raw.features \+ data/seminer/deu.train.bio.raw.brown \+ data/seminer/deu.train.lrec-2010-raw.model --hash \+ --heldout data/seminer/deu.testa.bio.raw.brown --iter 20 \+ --min-count 100 --rate 0.001 --beam 9+++data/seminer/deu.train.bio.raw.brown: data/seminer/deu.train.bio \+ data/seminer/deu.eci.unlabeled.500+1000.classes+ cut -d' ' -f1,5 data/seminer/deu.train.bio |\+ ./bin/augment brown True data/seminer/deu.eci.unlabeled.500+1000.classes\+ > data/seminer/deu.train.bio.raw.brown++data/seminer/deu.testa.bio.raw.brown: data/seminer/deu.testa.bio \+ data/seminer/deu.eci.unlabeled.500+1000.classes+ cut -d' ' -f1,5 data/seminer/deu.testa.bio |\+ ./bin/augment brown True data/seminer/deu.eci.unlabeled.500+1000.classes\+ > data/seminer/deu.testa.bio.raw.brown++data/seminer/deu.train.bio.brown: data/seminer/deu.train.bio \+ data/seminer/deu.eci.unlabeled.500+1000.classes+ cat data/seminer/deu.train.bio |\+ ./bin/augment brown True data/seminer/deu.eci.unlabeled.500+1000.classes\+ > data/seminer/deu.train.bio.brown++data/seminer/deu.testa.bio.brown: data/seminer/deu.testa.bio \+ data/seminer/deu.eci.unlabeled.500+1000.classes+ cat data/seminer/deu.testa.bio |\+ ./bin/augment brown True data/seminer/deu.eci.unlabeled.500+1000.classes\+ > data/seminer/deu.testa.bio.brown clean: -find . -name '*.o' | xargs rm
README view
@@ -1,4 +1,4 @@-sequor 0.2.2+sequor 0.3.0 AUTHOR: Grzegorz Chrupała <gchrupala@lsv.uni-saarland.de> @@ -6,6 +6,9 @@ perceptron. Sequor has a flexible feature template language and is meant mainly for NLP applications such as Part of Speech tagging, syntactic chunking or Named Entity labeling.++This version of Sequor includes SemiNER, a named-entity labeler, with+pre-trained models for German. For details see ./lib/seminer/README INSTALLATION
+ bin/seminer view
@@ -0,0 +1,22 @@+#! /bin/bash++if [ -z $SEMINER_ROOT ];+then SEMINER_ROOT=`pwd`+fi++case $1 in + "raw")+ cut -d' ' -f1 |\+ $SEMINER_ROOT/bin/augment brown False \+ $SEMINER_ROOT/data/seminer/deu.eci.unlabeled.500+1000.classes |\+ $SEMINER_ROOT/bin/sequor predict \+ $SEMINER_ROOT/data/seminer/deu.train.lrec-2010-raw.model+ ;;+ "full")+ cut -d' ' -f1,2,3,4 |\+ $SEMINER_ROOT/bin/augment brown False \+ $SEMINER_ROOT/data/seminer/deu.eci.unlabeled.500+1000.classes |\+ $SEMINER_ROOT/bin/sequor predict \+ $SEMINER_ROOT/data/seminer/deu.train.lrec-2010.model+ ;;+esac
+ data/seminer/deu.eci.unlabeled.500+1000.classes view
file too large to diff
+ data/seminer/deu.train.lrec-2010-raw.model view
file too large to diff
+ data/seminer/deu.train.lrec-2010.model view
file too large to diff
+ data/seminer/lrec-2010-raw.features view
@@ -0,0 +1,19 @@+Cat [ Index (Cell 0 0)+ , Lower (Cell 0 0)+ , Suffix 3 (Cell 0 0)+ , Suffix 2 (Cell 0 0)+ , Suffix 1 (Cell 0 0)+ , WordShape (Cell 0 0)+ , MarkNull (Cell -1 0)+ , Cell -2 0+ , Cell 1 0+ , Cell 2 0+ , Cell 0 2+ , Cart (Cell 0 1)+ (Cat [ Lower (Cell 0 0)+ , Suffix 3 (Cell 0 0)+ , Suffix 2 (Cell 0 0)+ , Suffix 1 (Cell 0 0)+ , WordShape (Cell 0 0) + ])+ ]
+ data/seminer/lrec-2010.features view
@@ -0,0 +1,24 @@+Cat [ Index (Cell 0 0)+ , Lower (Cell 0 0)+ , Cell 0 1+ , Suffix 3 (Cell 0 0)+ , Suffix 2 (Cell 0 0)+ , Suffix 1 (Cell 0 0)+ , WordShape (Cell 0 0)+ , Cell 0 2+ , Cell 0 3+ , MarkNull (Cell -1 0)+ , Cell -2 0+ , Cell 1 0+ , Cell 2 0+ , Cell 0 5+ , Cart (Cell 0 4)+ (Cat [ Lower (Cell 0 0)+ , Suffix 3 (Cell 0 0)+ , Suffix 2 (Cell 0 0)+ , Suffix 1 (Cell 0 0)+ , WordShape (Cell 0 0) + , Cell 0 2+ , Cell 0 3+ ])+ ]
+ lib/seminer/README view
@@ -0,0 +1,36 @@+SemiNER - Named Entity Labeler++Grzegorz Chrupala <gchrupala@lsv.uni-saarland.de>++Version 0.3++SemiNER is a tool for Named Entity labeleling. This +release includes models trained on the German CoNLL data with+features extracted from a large unlabeled German corpus.++The easiest way to start using SemiNER is to run the following command+from the top-level sequor directory:++> cabal install --prefix=`pwd`++This assumes that you have already installed the Haskell platform from+http://www.haskell.org/platform++Contact the author me if you encounter problems.++USAGE:++There are two pretrained models: full (which uses all the features+from training data, including lemmas, POS tags and chunk tags) and raw+(which only uses word features and cluster id features. You don't need+to run any additional preprocessing steps to run the raw model.++To label new data using the raw pre-trained model:+> bin/seminer raw < INPUT-FILE > OUTPUT-FILE+To label new data using the full pre-trained model:+> bin/seminer full < INPUT-FILE > OUTPUT-FILE+++The CoNLL input format is one token per line, sentences separated by a+blank line.+
+ lib/seminer/augment.hs view
@@ -0,0 +1,59 @@+{-# LANGUAGE OverloadedStrings #-}+module Main (main)+where+import qualified Helper.Text as Text+import Helper.Text(Txt)+import Data.List (unzip3,zipWith4)+import qualified Data.Map as Map+import System.Environment (getArgs)+import Helper.Utils (splitWith)++main :: IO ()+main = do+ command:args <- getArgs+ case command of + "brown" -> do+ let [isLabeled,dictf] = args+ dict <- readBrownDict `fmap` Text.readFile dictf+ let size = length . snd . Map.findMax $ dict+ txt <- Text.getContents+ Text.putStr+ . processStream (read isLabeled)+ (map (\w -> Map.findWithDefault + (replicate size "<NONE>")+ w + dict))+ $ txt++processStream :: Bool -> ([Txt] -> [[Txt]]) -> Txt -> Txt +processStream l f = + Text.unlines+ . map Text.unlines+ . map ( map Text.unwords . augmentSentence l f )+ . splitWith null+ . map Text.words+ . Text.lines++augmentSentence :: Bool -> ([Txt] -> [[Txt]]) -> [[Txt]] -> [[Txt]]+augmentSentence l f toks =+ let (ws,fs,ys) = unzip3 $ map (parseTok l) toks+ zs = f ws+ in zipWith4 (\ w f y z -> [w]++ f ++ z ++ [y])+ ws + fs + ys + zs++parseTok :: Bool -> [Txt] -> (Txt,[Txt],Txt)+parseTok True [w] = error $ "Invalid token " ++ show w+parseTok True [w,y] = (w,[],y)+parseTok True (w:fs) = (w,init fs,last fs)+parseTok False [w] = (w,[],"")+parseTok False (w:fs) = (w,fs,"")++readBrownDict :: Txt -> Map.Map Txt [Txt]+readBrownDict str = + Map.fromList [ let ws = Text.words $ ln + in (last ws , init . init $ ws)+ | ln <- Text.lines str ]+
+ lib/seminer/seminer view
@@ -0,0 +1,22 @@+#! /bin/bash++if [ -z $SEMINER_ROOT ];+then SEMINER_ROOT=`pwd`+fi++case $1 in + "raw")+ cut -d' ' -f1 |\+ $SEMINER_ROOT/bin/augment brown False \+ $SEMINER_ROOT/data/seminer/deu.eci.unlabeled.500+1000.classes |\+ $SEMINER_ROOT/bin/sequor predict \+ $SEMINER_ROOT/data/seminer/deu.train.lrec-2010-raw.model+ ;;+ "full")+ cut -d' ' -f1,2,3,4 |\+ $SEMINER_ROOT/bin/augment brown False \+ $SEMINER_ROOT/data/seminer/deu.eci.unlabeled.500+1000.classes |\+ $SEMINER_ROOT/bin/sequor predict \+ $SEMINER_ROOT/data/seminer/deu.train.lrec-2010.model+ ;;+esac
sequor.cabal view
@@ -1,5 +1,5 @@ Name: sequor-Version: 0.2.3+Version: 0.3.0 Description: A sequence labeler based on Collins's sequence perceptron. Synopsis: A sequence labeler based on Collins's sequence perceptron. Homepage: http://code.google.com/p/sequor/@@ -10,11 +10,17 @@ Build-Type: Simple Cabal-Version: >=1.2 Category: Natural Language Processing-Extra-source-files: README, Makefile+Extra-source-files: README, Makefile, lib/seminer/README,+ lib/seminer/seminer, lib/seminer/seminer, bin/seminer Data-dir: data Data-files: all.features, example.features, mlcomp.features, mlcomp2.features, - train.conll, devel.conll, test.conll+ train.conll, devel.conll, test.conll,+ seminer/lrec-2010.features,+ seminer/lrec-2010-raw.features,+ seminer/deu.eci.unlabeled.500+1000.classes,+ seminer/deu.train.lrec-2010.model,+ seminer/deu.train.lrec-2010-raw.model Executable sequor Main-is: Main.hs@@ -30,4 +36,11 @@ hs-source-dirs: src,lib ghc-options: -O2 -rtsopts c-sources: src/ghc_rts_opts.c ++Executable augment+ Main-is: augment.hs+ Other-modules: Helper.Utils, Helper.Text+ Build-Depends: base >= 3 && < 5, containers >= 0.2, text >= 0.10+ hs-source-dirs: lib, lib/seminer+ ghc-options: -O2 -rtsopts