morfette 0.3.6 → 0.4.2
raw patch · 14 files changed
+180/−117 lines, 14 filesbinary-added
Files
- INSTALL +0/−13
- Makefile +23/−64
- README +91/−23
- data/es/model/conf.model too large to diff
- data/es/model/models.model too large to diff
- data/es/model/mwe.model binary
- data/fr/model/conf.model too large to diff
- data/fr/model/models.model too large to diff
- data/fr/model/mwe.model binary
- morfette.cabal +6/−3
- src/GramLab/Morfette/Models2.hs +12/−5
- src/GramLab/Morfette/Utils.hs +10/−3
- src/Lemma.hs +21/−3
- src/POS.hs +17/−3
− INSTALL
@@ -1,13 +0,0 @@-=BUILDING INSTRUCTIONS=--You will need the GHC compiler <http://www.haskell.org/ghc/>. Morfette-depends on the following additional Haskell libraries:- * utf8-string: http://hackage.haskell.org/package/utf8-string- * binary: http://hackage.haskell.org/package/binary--Once you have GHC and the libraries, the rest is straightforward. -Unpack morfette and run the following commands:-- > runghc Setup configure --prefix=path/to/install-dir - > runghc Setup build- > runghc Setup install
Makefile view
@@ -1,79 +1,38 @@-defaults: all +# Train Spanish model +data/es/model: data/es/train.tokpos data/es/freeling-dict/dicc-2.0.src+ morfette train --iter-pos 10 --iter-lemma 3 \+ --dict-file data/es/freeling-dict/dicc-2.0.src \+ --language-conf es data/es/train.tokpos $@ +RTS -s -H2G +# Eval Spanish model +data/es/%.predict: data/es/model data/es/%.tokpos+ cut -f1 -d' ' data/es/$*.tokpos | morfette predict data/es/model > $@ -#program name-MORFETTE=dist/build/morfette/morfette+%-es-eval: data/es/train.tokpos data/es/%.tokpos data/es/%.predict+ morfette eval data/es/train.tokpos data/es/$*.tokpos data/es/$*.predict \+ --ignore-punctuation --ignore-case -#parameters for training and eval-# modify to suit your needs-# for french TYPE can be either ftb4 or ftbmax--TYPE=ftb4-TRAINDATADIR=DATA/-PREF=${TRAINDATADIR}/${TYPE}-TRAINSET=${PREF}/ftb_1.pos.utf8.morpheteready-DEVSET=${PREF}/ftb_2.pos.utf8.morpheteready-GOLDSET=${PREF}/ftb_3.pos.utf8.morpheteready--LEXICON=${TRAINDATADIR}/lexicon/lexicon.ftb4.4morfette.csv-ITERPOS=10-ITERLEMMA=3-MODELNAME=${PREF}/${TYPE}.${ITERPOS}x${ITERLEMMA}.model+# Train French model -all: configure build- - -check: train eval_dev+data/fr/model: data/fr/ftb_1.tokpos data/fr/lexicon.ftb4.4morfette.csv+ morfette train --iter-pos 10 --iter-lemma 2 \+ --dict-file data/fr/lexicon.ftb4.4morfette.csv \+ --language-conf fr data/fr/ftb_1.tokpos $@ +RTS -s -H2G -# note that if you want to install in your own home directory-# add a --prefix=DIRTOINSTALL option right after the --user-# as in -# runghc Setup.lhs configure --user --prefix=/home/foo/bar-# the install target will then install the morfette compiled binary-# in /home/foo/bar/bin (bin must exist)--configure:- runghc Setup.lhs configure --user---build: configure- runghc Setup.lhs build- -install: configure build- runghc Setup.lhs install- - -install_home:- - -clean:- runghc Setup.lhs clean- +# Eval French model +data/fr/%.predict: data/fr/model data/fr/%.tokpos+ cut -f1 -d' ' data/fr/$*.tokpos | morfette predict data/fr/model > $@ +%-fr-eval: data/fr/ftb_1.tokpos data/fr/%.tokpos data/fr/%.predict+ morfette eval data/fr/ftb_1.tokpos data/fr/$*.tokpos data/fr/$*.predict \+ --ignore-punctuation --ignore-case -train:- ${MORFETTE} train --iter-pos=${ITERPOS} --iter-lemma=${ITERLEMMA} \- ${TRAINSET} ${MODELNAME} \- --dict=${LEXICON} +RTS -K100m -sstderr- - -eval_dev:- #${MORFETTE} predict ${MODELNAME} < ${DEVSET} > ${DEVSET}.tagged- ${MORFETTE} eval --ignore-case /dev/null ${DEVSET} ${DEVSET}.tagged |\- tee ${DEVSET}.tagged.result.${TYPE}.${ITERPOS}x${ITERLEMMA}.model- - --eval_gold:- ${MORFETTE} predict ${MODELNAME} < ${GOLDSET} > ${GOLDSET}.tagged- ${MORFETTE} eval --ignore-case ${TRAINSET} ${GOLDSET} ${GOLDSET}.tagged |\- tee ${GOLDSET}.tagged.result.${TYPE}.${ITERPOS}x${ITERLEMMA}.model- +.PHONY: devel-es-eval ftb_2-fr-eval
README view
@@ -1,11 +1,12 @@-=INTRODUCTION=+INTRODUCTION+============ -Morfette website: http://sites.google.com/site/morfetteweb/+Morfette website: https://code.google.com/p/morfette/ Morfette is a tool for supervised learning of inflectional morphology. Given a corpus of sentences annotated with lemmas and morphological labels, and optionally a lexicon, morfette -learns how to morphologically analyse new sentences. +learns how to morphologically analyze new sentences. In the learning stage Morfette fits two separate logistic regression models: one for morphological tagging and one for lemmatization. The@@ -18,33 +19,61 @@ operations which are needed to transform the inflected word form into the corresponding lemma. -The basic approach is described in (Chrupala et al 2008 and Chrupala 2008). +The basic approach is described in (Chrupała et al 2008 and Chrupała 2008). The current version of Morfette uses an averaged perceptron to fit the models, rather than Maximum Entropy training. The lemmatization -classes are Edit-Tree-based as described in (Chrupala 2008).+classes are Edit-Tree-based as described in (Chrupała 2008). -=LICENSE= +LICENSE+=======+ The source code in the src directory is licensed under the BSD license. -=INSTALLATION=-Pre-built binaries are available from the project website. -If they don't work on your system you will-need to build from source, using the GHC Haskell compiler. Build-instructions are in [INSTALL]+INSTALLATION+============ -=USAGE=+The easiest way to install Morfette is to first install the Haskell+Platform <http://www.haskell.org/platform/> then execute the following+commands from within the morfette directory:++> cabal update+> cabal install --bindir=$HOME/bin++This will compile Morfette and install the executable in $HOME/bin. ++Cabal can also download Morfette from the source code repository+Hackage:++> cabal install morfette --bindir=$HOME/bin --datadir=$HOME/share++This will download Morfette, compile it, install the executable in+$HOME/bin, and install the data files in a $HOME/share.++There are also pre-built binaries available from the project website. ++USAGE+=====+ Usage: morfette command [OPTION...] [ARG...] train: train models train [OPTION...] TRAIN-FILE MODEL-DIR --dict-file=PATH path to optional dictionary --language-configuration=es|pl|tr|.. language configuration- --class-entropy-prune-threshold=NUM class prune threshold+ --iter-pos=NUM iterations for POS model+ --iter-lemma=NUM iterations for Lemma model +extract-features:+ extract features+extract-features [OPTION...] MODEL-DIR + --dict-file=PATH path to optional dictionary+ --model-id=pos|lemma model id (`pos' or `lemma')+ predict: predict postags and lemmas using saved model data predict [OPTION...] MODEL-DIR - --beam=+INT beam size to use- --tokenize tokenize input+ --beam=+INT beam size to use+ --tokenize tokenize input+ --multi=+INT n-best output eval: evaluate morpho-tagging and lemmatization results eval [OPTION...] TRAIN-FILE GOLD-FILE TEST-FILE @@ -54,15 +83,32 @@ --ignore-punctuation ignore punctuation for evaluation --ignore-pos=POS-prefix ignore POS starting with POS-prefix for evaluation +version: show version+version [OPTION...] -=EXAMPLE USAGE=+EXAMPLE USAGE+=============+ To train a new model:-morfette train --dict-file=DICT TRAINING-FILE MODEL-DIR +RTS -K100m +> morfette train --dict-file=DICT TRAINING-FILE MODEL-DIR + To use the model in MODEL-DIR to analyze new data:-morfette predict MODEL-DIR < TEST-DATA > ANALYZED-TEST-DATA -=DATA FORMAT=+> morfette predict MODEL-DIR < TEST-DATA > ANALYZED-TEST-DATA++PRETRAINED MODELS+=================++Pretrained models for Spanish and French are available in the data+directory: data/es/model and data/fr/model. For example you can use+the Spanish model like this:++> morfette predict data/es/model < TEST-DATA > ANALYZER-TEST-DATA++DATA FORMAT+===========+ Morfette expects both training and testing data to be tokenized and split into sentences. The format of training data look like this: @@ -104,14 +150,36 @@ la misma +Optionally, Morfette can also use vector representations of words for+training and prediction, in addition to character strings, as+described in [3]. In this case, the second column of is used for the+vectors. Inside this field vector components are separated by commas: -=References=-[1] Grzegorz Chrupala, Georgiana Dinu and Josef van Genabith. 2008.+Gómez 0.1,0.0,0.9 Gómez np0000p+sostiene 0.0,0.0,0.2 sostener vmip3s0++For prediction the format is similar, but only the first two columns+are used:++Gómez 0.1,0.0,0.9+sostiene 0.0,0.0,0.2++Each vector should have the same number of components.+++References+==========++[1] Grzegorz Chrupała, Georgiana Dinu and Josef van Genabith. 2008. Learning Morphology with Morfette. In Proceedings of LREC 2008. http://www.lrec-conf.org/proceedings/lrec2008/pdf/594_paper.pdf -[2] Grzegorz Chrupala. 2008. Towards a Machine-Learning Architecture+[2] Grzegorz Chrupała. 2008. Towards a Machine-Learning Architecture for Lexical Functional Grammar Parsing. Chapter 6. PhD dissertation, Dublin City University. - http://www.lsv.uni-saarland.de/personalPages/gchrupala/papers/phd.pdf+ http://grzegorz.chrupala.me/papers/phd.pdf++[3] Grzegorz Chrupała. 2010. Efficient induction of probabilistic word+ classes with LDA. IJCNLP. + http://grzegorz.chrupala.me/papers/ijcnlp-2011.pdf
+ data/es/model/conf.model view
file too large to diff
+ data/es/model/models.model view
file too large to diff
+ data/es/model/mwe.model view
binary file changed (absent → 10426 bytes)
+ data/fr/model/conf.model view
file too large to diff
+ data/fr/model/models.model view
file too large to diff
+ data/fr/model/mwe.model view
binary file changed (absent → 52582 bytes)
morfette.cabal view
@@ -1,5 +1,5 @@ Name: morfette-Version: 0.3.6+Version: 0.4.2 Homepage: http://sites.google.com/site/morfetteweb/ Synopsis: A tool for supervised learning of morphology Description: Morfette is a tool for supervised learning of inflectional@@ -12,8 +12,11 @@ Maintainer: Grzegorz Chrupała <grzegorz.chrupala@gmail.com> build-type: Simple category: Natural Language Processing-Extra-source-files: README, INSTALL, Makefile+Extra-source-files: README, Makefile cabal-version: >= 1.2+Data-dir: data+Data-files: es/model/conf.model, es/model/models.model, es/model/mwe.model,+ fr/model/conf.model, fr/model/models.model, fr/model/mwe.model Executable morfette hs-source-dirs: src@@ -39,6 +42,6 @@ vector >= 0.10, text >= 0.11 cc-options: -Wall c-sources: src/ghc_rts_opts.c - ghc-options: -O2 -rtsopts+ ghc-options: -O2 -rtsopts -with-rtsopts=-K128m -- uncomment the following line to link statically -- ld-options: -static -pthread
src/GramLab/Morfette/Models2.hs view
@@ -35,7 +35,8 @@ FS { label :: Row t a -> a , features :: LZipper (Row t a) (Row t a) (Row t a) -> [Feature String Double] , preprune :: ProbDist a -> ProbDist a- , check :: LZipper (Row t a) (Row t a) (Row t a) -> a -> Bool + , check :: LZipper (Row t a) (Row t a) (Row t a) -> a -> Bool+ , pruneUniqLabels :: Bool , trainSettings :: M.TrainSettings } @@ -96,14 +97,20 @@ -> [M.Model a Int String Double] train fspecs sents = flip map fspecs- $ \fs -> let yxs = concatMap (sentToExamples fs) $ sents- ys = uniq . map fst $ yxs- zs = concat [ take (length s) + $ \fs -> let yxs_all = concatMap (sentToExamples fs) $ sents+ zs_all = concat [ take (length s) . iterate slide . fromList $ s | s <- sents ]- yss = [ [ y | y <- ys , check fs z y ] + ys = (if pruneUniqLabels fs then Map.filter (>1) else id)+ . Map.fromListWith (+)+ . map (\ (y, _) -> (y, 1))+ $ yxs_all+ (yxs, zs) = unzip+ . filter (flip Map.member ys . fst . fst)+ $ zip yxs_all zs_all + yss = [ [ y | y <- Map.keys ys , check fs z y ] | z <- zs ] in M.train (trainSettings fs) yss yxs
src/GramLab/Morfette/Utils.hs view
@@ -64,6 +64,7 @@ | IterPOS Int | IterLemma Int | ModelId String+ | PruneUniqLabels deriving Eq data Input = Input { inputForm :: !String, inputEmb :: !(Maybe Emb) }@@ -101,9 +102,13 @@ , Option [] ["iter-lemma"] (ReqArg (IterLemma . read) "NUM") "iterations for Lemma model"- , Option [] ["cluster-file"]+ , Option [] ["prune-unique-labels"] + (NoArg PruneUniqLabels)+ "discard labels occurring only once"+ {- , Option [] ["cluster-file"] (ReqArg ClusterFile "PATH") "path to optional cluster file"+ -} ] [ "TRAIN-FILE", "MODEL-DIR" ]) , ("extract-features", CommandSpec (extractFeatures fs fspecs)@@ -278,7 +283,7 @@ [f] -> f i_l = case [f | IterLemma f <- flags ] of [] -> M.iter lemmaTrainSettings - [f] -> f + [f] -> f sentences = toksToSentences prepr toks createDirectoryIfMissing True modeldir let models = Models.train (map (\(i,fs) -> @@ -287,7 +292,9 @@ in fs' { Models.trainSettings = ts { M.entropyTh = g , M.iter = i- } })+ }+ , Models.pruneUniqLabels = PruneUniqLabels `elem` flags+ }) $ zip [i_p,i_l] fspecs) $ sentences B.writeFile (modelFile modeldir) (encode models)
src/Lemma.hs view
@@ -7,11 +7,15 @@ import GramLab.Morfette.Utils (ROW, Input(..), Output(..)) import Debug.Trace import Data.Maybe (fromMaybe)+import qualified Data.Vector.Unboxed as U+import Data.Ord +import Data.List featureSpec global = FS { label = theLabel , features = theFeatures global , preprune = mkPreprune 0.3- , check = theCheck + , check = theCheck+ , pruneUniqLabels = False , trainSettings = lemmaTrainSettings } theCheck :: LZipper ROW ROW ROW -> Output -> Bool@@ -42,7 +46,7 @@ -> LZipper ROW ROW ROW -> [Feature String Double] theFeatures global tic = focusFeatures (focus tic)- where focusFeatures (Just (Row { input = Input { inputForm = form } + where focusFeatures (Just (Row { input = Input { inputForm = form , inputEmb = mv } , output = (POS label:_) })) = [ Sym $ low form, Sym $ label , Sym $ spellingSpec form@@ -51,13 +55,27 @@ ] ++ prefixes maxPrefix (low form) ++ suffixes maxSuffix (low form)+ ++ embedding mv focusFeatures other = error $ "Lemma.theFeatures: " ++ show other low = lowercase lexmap w = Set $ map (show . make w . fst) $ Map.findWithDefault [] w (dictLex global) cluster w = case Map.lookup w (clusterDict global) of Nothing -> Null Just c -> Sym c-+ embedding Nothing = []+ embedding (Just v) = [ Num $ if abs n < 0.2 then 0 else n | n <- U.toList v ]+{- + embedding (Just v) = let izs = take 10 + . sortBy (flip $ comparing snd) + . zip [0..] + . U.toList + $ v + in [ Sym $ if n > 0.5 + then show i ++ "=True" + else show i ++ "=False" + | (i,n) <- izs ]+ +-} decode str = case reads str of [(s,"")] -> s other -> error ("Lemma.decode: no parse: " ++ str)
src/POS.hs view
@@ -7,11 +7,14 @@ import qualified Data.Map as Map import Lemma (apply) import qualified Data.Vector.Unboxed as U+import Data.Ord +import Data.List featureSpec global = FS { label = theLabel , features = theFeatures global , preprune = mkPreprune 0.3- , check = \_ _ -> True + , check = \_ _ -> True+ , pruneUniqLabels = False , trainSettings = posTrainSettings } leftCtx = 2@@ -62,5 +65,16 @@ getpos Nothing = "" getpos (Just (Row { output = (POS label:_) })) = head (splitPOS (lang global) label) embedding Nothing = []- embedding (Just v) = [ Num n | n <- U.toList v ]-+ embedding (Just v) = [ Num $ if abs n < 0.2 then 0 else n | n <- U.toList v ]+{-+ + embedding (Just v) = let izs = take 10 + . sortBy (flip $ comparing snd) + . zip [0..] + . U.toList + $ v + in [ Sym $ if n > 0.5 + then show i ++ "=True" + else show i ++ "=False" + | (i,n) <- izs ]+-}