diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,20 +1,61 @@ INCLUDES=lib -all:run - - -run: - ghc --make -O2 -isrc:$(INCLUDES) -o bin/sequor src/ghc_rts_opts.c src/Main.hs - cabal: runghc Setup.lhs configure --user --prefix=`pwd` --bindir=`pwd`/bin/ &&\ runghc Setup.lhs build && runghc Setup.lhs install + cabal-static: runghc Setup.lhs configure --user --prefix=`pwd` --bindir=`pwd`/bin/ \ --ghc-option="-optl-static" --ghc-option="-optl-pthread" &&\ runghc Setup.lhs build && runghc Setup.lhs install -rm -rf dist -rm -rf share + +train-seminer: data/seminer/deu.train.lrec-2010-raw.model \ + data/seminer/deu.train.lrec-2010.model + +data/seminer/deu.train.lrec-2010.model: data/seminer/lrec-2010.features \ + data/seminer/deu.train.bio.brown \ + data/seminer/deu.testa.bio.brown + ./bin/sequor train data/seminer/lrec-2010.features \ + data/seminer/deu.train.bio.brown \ + data/seminer/deu.train.lrec-2010.model --hash \ + --heldout data/seminer/deu.testa.bio.brown --iter 20 \ + --min-count 100 --rate 0.001 --beam 9 + +data/seminer/deu.train.lrec-2010-raw.model: data/seminer/lrec-2010-raw.features \ + data/seminer/deu.train.bio.raw.brown \ + data/seminer/deu.testa.bio.raw.brown + ./bin/sequor train data/seminer/lrec-2010-raw.features \ + data/seminer/deu.train.bio.raw.brown \ + data/seminer/deu.train.lrec-2010-raw.model --hash \ + --heldout data/seminer/deu.testa.bio.raw.brown --iter 20 \ + --min-count 100 --rate 0.001 --beam 9 + + +data/seminer/deu.train.bio.raw.brown: data/seminer/deu.train.bio \ + data/seminer/deu.eci.unlabeled.500+1000.classes + cut -d' ' -f1,5 data/seminer/deu.train.bio |\ + ./bin/augment brown True data/seminer/deu.eci.unlabeled.500+1000.classes\ + > data/seminer/deu.train.bio.raw.brown + +data/seminer/deu.testa.bio.raw.brown: data/seminer/deu.testa.bio \ + data/seminer/deu.eci.unlabeled.500+1000.classes + cut -d' ' -f1,5 data/seminer/deu.testa.bio |\ + ./bin/augment brown True data/seminer/deu.eci.unlabeled.500+1000.classes\ + > data/seminer/deu.testa.bio.raw.brown + +data/seminer/deu.train.bio.brown: data/seminer/deu.train.bio \ + data/seminer/deu.eci.unlabeled.500+1000.classes + cat data/seminer/deu.train.bio |\ + ./bin/augment brown True data/seminer/deu.eci.unlabeled.500+1000.classes\ + > data/seminer/deu.train.bio.brown + +data/seminer/deu.testa.bio.brown: data/seminer/deu.testa.bio \ + data/seminer/deu.eci.unlabeled.500+1000.classes + cat data/seminer/deu.testa.bio |\ + ./bin/augment brown True data/seminer/deu.eci.unlabeled.500+1000.classes\ + > data/seminer/deu.testa.bio.brown clean: -find . -name '*.o' | xargs rm diff --git a/README b/README --- a/README +++ b/README @@ -1,4 +1,4 @@ -sequor 0.2.2 +sequor 0.3.0 AUTHOR: Grzegorz Chrupała @@ -6,6 +6,9 @@ perceptron. Sequor has a flexible feature template language and is meant mainly for NLP applications such as Part of Speech tagging, syntactic chunking or Named Entity labeling. + +This version of Sequor includes SemiNER, a named-entity labeler, with +pre-trained models for German. For details see ./lib/seminer/README INSTALLATION diff --git a/bin/seminer b/bin/seminer new file mode 100644 --- /dev/null +++ b/bin/seminer @@ -0,0 +1,22 @@ +#! /bin/bash + +if [ -z $SEMINER_ROOT ]; +then SEMINER_ROOT=`pwd` +fi + +case $1 in + "raw") + cut -d' ' -f1 |\ + $SEMINER_ROOT/bin/augment brown False \ + $SEMINER_ROOT/data/seminer/deu.eci.unlabeled.500+1000.classes |\ + $SEMINER_ROOT/bin/sequor predict \ + $SEMINER_ROOT/data/seminer/deu.train.lrec-2010-raw.model + ;; + "full") + cut -d' ' -f1,2,3,4 |\ + $SEMINER_ROOT/bin/augment brown False \ + $SEMINER_ROOT/data/seminer/deu.eci.unlabeled.500+1000.classes |\ + $SEMINER_ROOT/bin/sequor predict \ + $SEMINER_ROOT/data/seminer/deu.train.lrec-2010.model + ;; +esac diff --git a/data/seminer/deu.eci.unlabeled.500+1000.classes b/data/seminer/deu.eci.unlabeled.500+1000.classes new file mode 100644 # file too large to diff: data/seminer/deu.eci.unlabeled.500+1000.classes diff --git a/data/seminer/deu.train.lrec-2010-raw.model b/data/seminer/deu.train.lrec-2010-raw.model new file mode 100644 # file too large to diff: data/seminer/deu.train.lrec-2010-raw.model diff --git a/data/seminer/deu.train.lrec-2010.model b/data/seminer/deu.train.lrec-2010.model new file mode 100644 # file too large to diff: data/seminer/deu.train.lrec-2010.model diff --git a/data/seminer/lrec-2010-raw.features b/data/seminer/lrec-2010-raw.features new file mode 100644 --- /dev/null +++ b/data/seminer/lrec-2010-raw.features @@ -0,0 +1,19 @@ +Cat [ Index (Cell 0 0) + , Lower (Cell 0 0) + , Suffix 3 (Cell 0 0) + , Suffix 2 (Cell 0 0) + , Suffix 1 (Cell 0 0) + , WordShape (Cell 0 0) + , MarkNull (Cell -1 0) + , Cell -2 0 + , Cell 1 0 + , Cell 2 0 + , Cell 0 2 + , Cart (Cell 0 1) + (Cat [ Lower (Cell 0 0) + , Suffix 3 (Cell 0 0) + , Suffix 2 (Cell 0 0) + , Suffix 1 (Cell 0 0) + , WordShape (Cell 0 0) + ]) + ] diff --git a/data/seminer/lrec-2010.features b/data/seminer/lrec-2010.features new file mode 100644 --- /dev/null +++ b/data/seminer/lrec-2010.features @@ -0,0 +1,24 @@ +Cat [ Index (Cell 0 0) + , Lower (Cell 0 0) + , Cell 0 1 + , Suffix 3 (Cell 0 0) + , Suffix 2 (Cell 0 0) + , Suffix 1 (Cell 0 0) + , WordShape (Cell 0 0) + , Cell 0 2 + , Cell 0 3 + , MarkNull (Cell -1 0) + , Cell -2 0 + , Cell 1 0 + , Cell 2 0 + , Cell 0 5 + , Cart (Cell 0 4) + (Cat [ Lower (Cell 0 0) + , Suffix 3 (Cell 0 0) + , Suffix 2 (Cell 0 0) + , Suffix 1 (Cell 0 0) + , WordShape (Cell 0 0) + , Cell 0 2 + , Cell 0 3 + ]) + ] diff --git a/lib/seminer/README b/lib/seminer/README new file mode 100644 --- /dev/null +++ b/lib/seminer/README @@ -0,0 +1,36 @@ +SemiNER - Named Entity Labeler + +Grzegorz Chrupala + +Version 0.3 + +SemiNER is a tool for Named Entity labeleling. This +release includes models trained on the German CoNLL data with +features extracted from a large unlabeled German corpus. + +The easiest way to start using SemiNER is to run the following command +from the top-level sequor directory: + +> cabal install --prefix=`pwd` + +This assumes that you have already installed the Haskell platform from +http://www.haskell.org/platform + +Contact the author me if you encounter problems. + +USAGE: + +There are two pretrained models: full (which uses all the features +from training data, including lemmas, POS tags and chunk tags) and raw +(which only uses word features and cluster id features. You don't need +to run any additional preprocessing steps to run the raw model. + +To label new data using the raw pre-trained model: +> bin/seminer raw < INPUT-FILE > OUTPUT-FILE +To label new data using the full pre-trained model: +> bin/seminer full < INPUT-FILE > OUTPUT-FILE + + +The CoNLL input format is one token per line, sentences separated by a +blank line. + diff --git a/lib/seminer/augment.hs b/lib/seminer/augment.hs new file mode 100644 --- /dev/null +++ b/lib/seminer/augment.hs @@ -0,0 +1,59 @@ +{-# LANGUAGE OverloadedStrings #-} +module Main (main) +where +import qualified Helper.Text as Text +import Helper.Text(Txt) +import Data.List (unzip3,zipWith4) +import qualified Data.Map as Map +import System.Environment (getArgs) +import Helper.Utils (splitWith) + +main :: IO () +main = do + command:args <- getArgs + case command of + "brown" -> do + let [isLabeled,dictf] = args + dict <- readBrownDict `fmap` Text.readFile dictf + let size = length . snd . Map.findMax $ dict + txt <- Text.getContents + Text.putStr + . processStream (read isLabeled) + (map (\w -> Map.findWithDefault + (replicate size "") + w + dict)) + $ txt + +processStream :: Bool -> ([Txt] -> [[Txt]]) -> Txt -> Txt +processStream l f = + Text.unlines + . map Text.unlines + . map ( map Text.unwords . augmentSentence l f ) + . splitWith null + . map Text.words + . Text.lines + +augmentSentence :: Bool -> ([Txt] -> [[Txt]]) -> [[Txt]] -> [[Txt]] +augmentSentence l f toks = + let (ws,fs,ys) = unzip3 $ map (parseTok l) toks + zs = f ws + in zipWith4 (\ w f y z -> [w]++ f ++ z ++ [y]) + ws + fs + ys + zs + +parseTok :: Bool -> [Txt] -> (Txt,[Txt],Txt) +parseTok True [w] = error $ "Invalid token " ++ show w +parseTok True [w,y] = (w,[],y) +parseTok True (w:fs) = (w,init fs,last fs) +parseTok False [w] = (w,[],"") +parseTok False (w:fs) = (w,fs,"") + +readBrownDict :: Txt -> Map.Map Txt [Txt] +readBrownDict str = + Map.fromList [ let ws = Text.words $ ln + in (last ws , init . init $ ws) + | ln <- Text.lines str ] + diff --git a/lib/seminer/seminer b/lib/seminer/seminer new file mode 100644 --- /dev/null +++ b/lib/seminer/seminer @@ -0,0 +1,22 @@ +#! /bin/bash + +if [ -z $SEMINER_ROOT ]; +then SEMINER_ROOT=`pwd` +fi + +case $1 in + "raw") + cut -d' ' -f1 |\ + $SEMINER_ROOT/bin/augment brown False \ + $SEMINER_ROOT/data/seminer/deu.eci.unlabeled.500+1000.classes |\ + $SEMINER_ROOT/bin/sequor predict \ + $SEMINER_ROOT/data/seminer/deu.train.lrec-2010-raw.model + ;; + "full") + cut -d' ' -f1,2,3,4 |\ + $SEMINER_ROOT/bin/augment brown False \ + $SEMINER_ROOT/data/seminer/deu.eci.unlabeled.500+1000.classes |\ + $SEMINER_ROOT/bin/sequor predict \ + $SEMINER_ROOT/data/seminer/deu.train.lrec-2010.model + ;; +esac diff --git a/sequor.cabal b/sequor.cabal --- a/sequor.cabal +++ b/sequor.cabal @@ -1,5 +1,5 @@ Name: sequor -Version: 0.2.3 +Version: 0.3.0 Description: A sequence labeler based on Collins's sequence perceptron. Synopsis: A sequence labeler based on Collins's sequence perceptron. Homepage: http://code.google.com/p/sequor/ @@ -10,11 +10,17 @@ Build-Type: Simple Cabal-Version: >=1.2 Category: Natural Language Processing -Extra-source-files: README, Makefile +Extra-source-files: README, Makefile, lib/seminer/README, + lib/seminer/seminer, lib/seminer/seminer, bin/seminer Data-dir: data Data-files: all.features, example.features, mlcomp.features, mlcomp2.features, - train.conll, devel.conll, test.conll + train.conll, devel.conll, test.conll, + seminer/lrec-2010.features, + seminer/lrec-2010-raw.features, + seminer/deu.eci.unlabeled.500+1000.classes, + seminer/deu.train.lrec-2010.model, + seminer/deu.train.lrec-2010-raw.model Executable sequor Main-is: Main.hs @@ -30,4 +36,11 @@ hs-source-dirs: src,lib ghc-options: -O2 -rtsopts c-sources: src/ghc_rts_opts.c + +Executable augment + Main-is: augment.hs + Other-modules: Helper.Utils, Helper.Text + Build-Depends: base >= 3 && < 5, containers >= 0.2, text >= 0.10 + hs-source-dirs: lib, lib/seminer + ghc-options: -O2 -rtsopts