diff --git a/INSTALL b/INSTALL
deleted file mode 100644
--- a/INSTALL
+++ /dev/null
@@ -1,13 +0,0 @@
-=BUILDING INSTRUCTIONS=
-
-You will need the GHC compiler . Morfette
-depends on the following additional Haskell libraries:
- * utf8-string: http://hackage.haskell.org/package/utf8-string
- * binary: http://hackage.haskell.org/package/binary
-
-Once you have GHC and the libraries, the rest is straightforward.
-Unpack morfette and run the following commands:
-
- > runghc Setup configure --prefix=path/to/install-dir
- > runghc Setup build
- > runghc Setup install
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -1,79 +1,38 @@
-defaults: all
+# Train Spanish model
+data/es/model: data/es/train.tokpos data/es/freeling-dict/dicc-2.0.src
+ morfette train --iter-pos 10 --iter-lemma 3 \
+ --dict-file data/es/freeling-dict/dicc-2.0.src \
+ --language-conf es data/es/train.tokpos $@ +RTS -s -H2G
+# Eval Spanish model
+data/es/%.predict: data/es/model data/es/%.tokpos
+ cut -f1 -d' ' data/es/$*.tokpos | morfette predict data/es/model > $@
-#program name
-MORFETTE=dist/build/morfette/morfette
+%-es-eval: data/es/train.tokpos data/es/%.tokpos data/es/%.predict
+ morfette eval data/es/train.tokpos data/es/$*.tokpos data/es/$*.predict \
+ --ignore-punctuation --ignore-case
-#parameters for training and eval
-# modify to suit your needs
-# for french TYPE can be either ftb4 or ftbmax
-
-TYPE=ftb4
-TRAINDATADIR=DATA/
-PREF=${TRAINDATADIR}/${TYPE}
-TRAINSET=${PREF}/ftb_1.pos.utf8.morpheteready
-DEVSET=${PREF}/ftb_2.pos.utf8.morpheteready
-GOLDSET=${PREF}/ftb_3.pos.utf8.morpheteready
-
-LEXICON=${TRAINDATADIR}/lexicon/lexicon.ftb4.4morfette.csv
-ITERPOS=10
-ITERLEMMA=3
-MODELNAME=${PREF}/${TYPE}.${ITERPOS}x${ITERLEMMA}.model
+# Train French model
-all: configure build
-
-
-check: train eval_dev
+data/fr/model: data/fr/ftb_1.tokpos data/fr/lexicon.ftb4.4morfette.csv
+ morfette train --iter-pos 10 --iter-lemma 2 \
+ --dict-file data/fr/lexicon.ftb4.4morfette.csv \
+ --language-conf fr data/fr/ftb_1.tokpos $@ +RTS -s -H2G
-# note that if you want to install in your own home directory
-# add a --prefix=DIRTOINSTALL option right after the --user
-# as in
-# runghc Setup.lhs configure --user --prefix=/home/foo/bar
-# the install target will then install the morfette compiled binary
-# in /home/foo/bar/bin (bin must exist)
-
-configure:
- runghc Setup.lhs configure --user
-
-
-build: configure
- runghc Setup.lhs build
-
-install: configure build
- runghc Setup.lhs install
-
-
-install_home:
-
-
-clean:
- runghc Setup.lhs clean
-
+# Eval French model
+data/fr/%.predict: data/fr/model data/fr/%.tokpos
+ cut -f1 -d' ' data/fr/$*.tokpos | morfette predict data/fr/model > $@
+%-fr-eval: data/fr/ftb_1.tokpos data/fr/%.tokpos data/fr/%.predict
+ morfette eval data/fr/ftb_1.tokpos data/fr/$*.tokpos data/fr/$*.predict \
+ --ignore-punctuation --ignore-case
-train:
- ${MORFETTE} train --iter-pos=${ITERPOS} --iter-lemma=${ITERLEMMA} \
- ${TRAINSET} ${MODELNAME} \
- --dict=${LEXICON} +RTS -K100m -sstderr
-
-
-eval_dev:
- #${MORFETTE} predict ${MODELNAME} < ${DEVSET} > ${DEVSET}.tagged
- ${MORFETTE} eval --ignore-case /dev/null ${DEVSET} ${DEVSET}.tagged |\
- tee ${DEVSET}.tagged.result.${TYPE}.${ITERPOS}x${ITERLEMMA}.model
-
-
-
-eval_gold:
- ${MORFETTE} predict ${MODELNAME} < ${GOLDSET} > ${GOLDSET}.tagged
- ${MORFETTE} eval --ignore-case ${TRAINSET} ${GOLDSET} ${GOLDSET}.tagged |\
- tee ${GOLDSET}.tagged.result.${TYPE}.${ITERPOS}x${ITERLEMMA}.model
-
+.PHONY: devel-es-eval ftb_2-fr-eval
diff --git a/README b/README
--- a/README
+++ b/README
@@ -1,11 +1,12 @@
-=INTRODUCTION=
+INTRODUCTION
+============
-Morfette website: http://sites.google.com/site/morfetteweb/
+Morfette website: https://code.google.com/p/morfette/
Morfette is a tool for supervised learning of inflectional
morphology. Given a corpus of sentences annotated with lemmas
and morphological labels, and optionally a lexicon, morfette
-learns how to morphologically analyse new sentences.
+learns how to morphologically analyze new sentences.
In the learning stage Morfette fits two separate logistic regression
models: one for morphological tagging and one for lemmatization. The
@@ -18,33 +19,61 @@
operations which are needed to transform the inflected word form into
the corresponding lemma.
-The basic approach is described in (Chrupala et al 2008 and Chrupala 2008).
+The basic approach is described in (Chrupała et al 2008 and Chrupała 2008).
The current version of Morfette uses an averaged perceptron to
fit the models, rather than Maximum Entropy training. The lemmatization
-classes are Edit-Tree-based as described in (Chrupala 2008).
+classes are Edit-Tree-based as described in (Chrupała 2008).
-=LICENSE=
+LICENSE
+=======
+
The source code in the src directory is licensed under
the BSD license.
-=INSTALLATION=
-Pre-built binaries are available from the project website.
-If they don't work on your system you will
-need to build from source, using the GHC Haskell compiler. Build
-instructions are in [INSTALL]
+INSTALLATION
+============
-=USAGE=
+The easiest way to install Morfette is to first install the Haskell
+Platform then execute the following
+commands from within the morfette directory:
+
+> cabal update
+> cabal install --bindir=$HOME/bin
+
+This will compile Morfette and install the executable in $HOME/bin.
+
+Cabal can also download Morfette from the source code repository
+Hackage:
+
+> cabal install morfette --bindir=$HOME/bin --datadir=$HOME/share
+
+This will download Morfette, compile it, install the executable in
+$HOME/bin, and install the data files in a $HOME/share.
+
+There are also pre-built binaries available from the project website.
+
+USAGE
+=====
+
Usage: morfette command [OPTION...] [ARG...]
train: train models
train [OPTION...] TRAIN-FILE MODEL-DIR
--dict-file=PATH path to optional dictionary
--language-configuration=es|pl|tr|.. language configuration
- --class-entropy-prune-threshold=NUM class prune threshold
+ --iter-pos=NUM iterations for POS model
+ --iter-lemma=NUM iterations for Lemma model
+extract-features:
+ extract features
+extract-features [OPTION...] MODEL-DIR
+ --dict-file=PATH path to optional dictionary
+ --model-id=pos|lemma model id (`pos' or `lemma')
+
predict: predict postags and lemmas using saved model data
predict [OPTION...] MODEL-DIR
- --beam=+INT beam size to use
- --tokenize tokenize input
+ --beam=+INT beam size to use
+ --tokenize tokenize input
+ --multi=+INT n-best output
eval: evaluate morpho-tagging and lemmatization results
eval [OPTION...] TRAIN-FILE GOLD-FILE TEST-FILE
@@ -54,15 +83,32 @@
--ignore-punctuation ignore punctuation for evaluation
--ignore-pos=POS-prefix ignore POS starting with POS-prefix for evaluation
+version: show version
+version [OPTION...]
-=EXAMPLE USAGE=
+EXAMPLE USAGE
+=============
+
To train a new model:
-morfette train --dict-file=DICT TRAINING-FILE MODEL-DIR +RTS -K100m
+> morfette train --dict-file=DICT TRAINING-FILE MODEL-DIR
+
To use the model in MODEL-DIR to analyze new data:
-morfette predict MODEL-DIR < TEST-DATA > ANALYZED-TEST-DATA
-=DATA FORMAT=
+> morfette predict MODEL-DIR < TEST-DATA > ANALYZED-TEST-DATA
+
+PRETRAINED MODELS
+=================
+
+Pretrained models for Spanish and French are available in the data
+directory: data/es/model and data/fr/model. For example you can use
+the Spanish model like this:
+
+> morfette predict data/es/model < TEST-DATA > ANALYZER-TEST-DATA
+
+DATA FORMAT
+===========
+
Morfette expects both training and testing data to be tokenized and
split into sentences. The format of training data look like this:
@@ -104,14 +150,36 @@
la
misma
+Optionally, Morfette can also use vector representations of words for
+training and prediction, in addition to character strings, as
+described in [3]. In this case, the second column of is used for the
+vectors. Inside this field vector components are separated by commas:
-=References=
-[1] Grzegorz Chrupala, Georgiana Dinu and Josef van Genabith. 2008.
+Gómez 0.1,0.0,0.9 Gómez np0000p
+sostiene 0.0,0.0,0.2 sostener vmip3s0
+
+For prediction the format is similar, but only the first two columns
+are used:
+
+Gómez 0.1,0.0,0.9
+sostiene 0.0,0.0,0.2
+
+Each vector should have the same number of components.
+
+
+References
+==========
+
+[1] Grzegorz Chrupała, Georgiana Dinu and Josef van Genabith. 2008.
Learning Morphology with Morfette. In Proceedings of LREC 2008.
http://www.lrec-conf.org/proceedings/lrec2008/pdf/594_paper.pdf
-[2] Grzegorz Chrupala. 2008. Towards a Machine-Learning Architecture
+[2] Grzegorz Chrupała. 2008. Towards a Machine-Learning Architecture
for Lexical Functional Grammar Parsing. Chapter 6. PhD
dissertation, Dublin City
University.
- http://www.lsv.uni-saarland.de/personalPages/gchrupala/papers/phd.pdf
+ http://grzegorz.chrupala.me/papers/phd.pdf
+
+[3] Grzegorz Chrupała. 2010. Efficient induction of probabilistic word
+ classes with LDA. IJCNLP.
+ http://grzegorz.chrupala.me/papers/ijcnlp-2011.pdf
diff --git a/data/es/model/conf.model b/data/es/model/conf.model
new file mode 100644
# file too large to diff: data/es/model/conf.model
diff --git a/data/es/model/models.model b/data/es/model/models.model
new file mode 100644
# file too large to diff: data/es/model/models.model
diff --git a/data/es/model/mwe.model b/data/es/model/mwe.model
new file mode 100644
Binary files /dev/null and b/data/es/model/mwe.model differ
diff --git a/data/fr/model/conf.model b/data/fr/model/conf.model
new file mode 100644
# file too large to diff: data/fr/model/conf.model
diff --git a/data/fr/model/models.model b/data/fr/model/models.model
new file mode 100644
# file too large to diff: data/fr/model/models.model
diff --git a/data/fr/model/mwe.model b/data/fr/model/mwe.model
new file mode 100644
Binary files /dev/null and b/data/fr/model/mwe.model differ
diff --git a/morfette.cabal b/morfette.cabal
--- a/morfette.cabal
+++ b/morfette.cabal
@@ -1,5 +1,5 @@
Name: morfette
-Version: 0.3.6
+Version: 0.4.2
Homepage: http://sites.google.com/site/morfetteweb/
Synopsis: A tool for supervised learning of morphology
Description: Morfette is a tool for supervised learning of inflectional
@@ -12,8 +12,11 @@
Maintainer: Grzegorz Chrupała
build-type: Simple
category: Natural Language Processing
-Extra-source-files: README, INSTALL, Makefile
+Extra-source-files: README, Makefile
cabal-version: >= 1.2
+Data-dir: data
+Data-files: es/model/conf.model, es/model/models.model, es/model/mwe.model,
+ fr/model/conf.model, fr/model/models.model, fr/model/mwe.model
Executable morfette
hs-source-dirs: src
@@ -39,6 +42,6 @@
vector >= 0.10, text >= 0.11
cc-options: -Wall
c-sources: src/ghc_rts_opts.c
- ghc-options: -O2 -rtsopts
+ ghc-options: -O2 -rtsopts -with-rtsopts=-K128m
-- uncomment the following line to link statically
-- ld-options: -static -pthread
diff --git a/src/GramLab/Morfette/Models2.hs b/src/GramLab/Morfette/Models2.hs
--- a/src/GramLab/Morfette/Models2.hs
+++ b/src/GramLab/Morfette/Models2.hs
@@ -35,7 +35,8 @@
FS { label :: Row t a -> a
, features :: LZipper (Row t a) (Row t a) (Row t a) -> [Feature String Double]
, preprune :: ProbDist a -> ProbDist a
- , check :: LZipper (Row t a) (Row t a) (Row t a) -> a -> Bool
+ , check :: LZipper (Row t a) (Row t a) (Row t a) -> a -> Bool
+ , pruneUniqLabels :: Bool
, trainSettings :: M.TrainSettings }
@@ -96,14 +97,20 @@
-> [M.Model a Int String Double]
train fspecs sents =
flip map fspecs
- $ \fs -> let yxs = concatMap (sentToExamples fs) $ sents
- ys = uniq . map fst $ yxs
- zs = concat [ take (length s)
+ $ \fs -> let yxs_all = concatMap (sentToExamples fs) $ sents
+ zs_all = concat [ take (length s)
. iterate slide
. fromList
$ s
| s <- sents ]
- yss = [ [ y | y <- ys , check fs z y ]
+ ys = (if pruneUniqLabels fs then Map.filter (>1) else id)
+ . Map.fromListWith (+)
+ . map (\ (y, _) -> (y, 1))
+ $ yxs_all
+ (yxs, zs) = unzip
+ . filter (flip Map.member ys . fst . fst)
+ $ zip yxs_all zs_all
+ yss = [ [ y | y <- Map.keys ys , check fs z y ]
| z <- zs ]
in M.train (trainSettings fs) yss yxs
diff --git a/src/GramLab/Morfette/Utils.hs b/src/GramLab/Morfette/Utils.hs
--- a/src/GramLab/Morfette/Utils.hs
+++ b/src/GramLab/Morfette/Utils.hs
@@ -64,6 +64,7 @@
| IterPOS Int
| IterLemma Int
| ModelId String
+ | PruneUniqLabels
deriving Eq
data Input = Input { inputForm :: !String, inputEmb :: !(Maybe Emb) }
@@ -101,9 +102,13 @@
, Option [] ["iter-lemma"]
(ReqArg (IterLemma . read) "NUM")
"iterations for Lemma model"
- , Option [] ["cluster-file"]
+ , Option [] ["prune-unique-labels"]
+ (NoArg PruneUniqLabels)
+ "discard labels occurring only once"
+ {- , Option [] ["cluster-file"]
(ReqArg ClusterFile "PATH")
"path to optional cluster file"
+ -}
]
[ "TRAIN-FILE", "MODEL-DIR" ])
, ("extract-features", CommandSpec (extractFeatures fs fspecs)
@@ -278,7 +283,7 @@
[f] -> f
i_l = case [f | IterLemma f <- flags ] of
[] -> M.iter lemmaTrainSettings
- [f] -> f
+ [f] -> f
sentences = toksToSentences prepr toks
createDirectoryIfMissing True modeldir
let models = Models.train (map (\(i,fs) ->
@@ -287,7 +292,9 @@
in fs' { Models.trainSettings =
ts { M.entropyTh = g
, M.iter = i
- } })
+ }
+ , Models.pruneUniqLabels = PruneUniqLabels `elem` flags
+ })
$ zip [i_p,i_l] fspecs)
$ sentences
B.writeFile (modelFile modeldir) (encode models)
diff --git a/src/Lemma.hs b/src/Lemma.hs
--- a/src/Lemma.hs
+++ b/src/Lemma.hs
@@ -7,11 +7,15 @@
import GramLab.Morfette.Utils (ROW, Input(..), Output(..))
import Debug.Trace
import Data.Maybe (fromMaybe)
+import qualified Data.Vector.Unboxed as U
+import Data.Ord
+import Data.List
featureSpec global = FS { label = theLabel
, features = theFeatures global
, preprune = mkPreprune 0.3
- , check = theCheck
+ , check = theCheck
+ , pruneUniqLabels = False
, trainSettings = lemmaTrainSettings }
theCheck :: LZipper ROW ROW ROW -> Output -> Bool
@@ -42,7 +46,7 @@
-> LZipper ROW ROW ROW
-> [Feature String Double]
theFeatures global tic = focusFeatures (focus tic)
- where focusFeatures (Just (Row { input = Input { inputForm = form }
+ where focusFeatures (Just (Row { input = Input { inputForm = form , inputEmb = mv }
, output = (POS label:_) })) =
[ Sym $ low form, Sym $ label
, Sym $ spellingSpec form
@@ -51,13 +55,27 @@
]
++ prefixes maxPrefix (low form)
++ suffixes maxSuffix (low form)
+ ++ embedding mv
focusFeatures other = error $ "Lemma.theFeatures: " ++ show other
low = lowercase
lexmap w = Set $ map (show . make w . fst) $ Map.findWithDefault [] w (dictLex global)
cluster w = case Map.lookup w (clusterDict global) of
Nothing -> Null
Just c -> Sym c
-
+ embedding Nothing = []
+ embedding (Just v) = [ Num $ if abs n < 0.2 then 0 else n | n <- U.toList v ]
+{-
+ embedding (Just v) = let izs = take 10
+ . sortBy (flip $ comparing snd)
+ . zip [0..]
+ . U.toList
+ $ v
+ in [ Sym $ if n > 0.5
+ then show i ++ "=True"
+ else show i ++ "=False"
+ | (i,n) <- izs ]
+
+-}
decode str = case reads str of
[(s,"")] -> s
other -> error ("Lemma.decode: no parse: " ++ str)
diff --git a/src/POS.hs b/src/POS.hs
--- a/src/POS.hs
+++ b/src/POS.hs
@@ -7,11 +7,14 @@
import qualified Data.Map as Map
import Lemma (apply)
import qualified Data.Vector.Unboxed as U
+import Data.Ord
+import Data.List
featureSpec global = FS { label = theLabel
, features = theFeatures global
, preprune = mkPreprune 0.3
- , check = \_ _ -> True
+ , check = \_ _ -> True
+ , pruneUniqLabels = False
, trainSettings = posTrainSettings }
leftCtx = 2
@@ -62,5 +65,16 @@
getpos Nothing = ""
getpos (Just (Row { output = (POS label:_) })) = head (splitPOS (lang global) label)
embedding Nothing = []
- embedding (Just v) = [ Num n | n <- U.toList v ]
-
+ embedding (Just v) = [ Num $ if abs n < 0.2 then 0 else n | n <- U.toList v ]
+{-
+
+ embedding (Just v) = let izs = take 10
+ . sortBy (flip $ comparing snd)
+ . zip [0..]
+ . U.toList
+ $ v
+ in [ Sym $ if n > 0.5
+ then show i ++ "=True"
+ else show i ++ "=False"
+ | (i,n) <- izs ]
+-}