WordAlignment 0.1.0.0 → 0.2.0.0
raw patch · 41 files changed
+2893/−1266 lines, 41 filesdep +DPutilsdep +aesondep +bimapsdep −QuickCheckdep −ascii-progressdep −control-monad-omegadep ~ADPfusiondep ~AlignmentAlgorithmsdep ~FormalGrammars
Dependencies added: DPutils, aeson, bimaps, data-default, filepath, mtl, pipes, split, tasty, tasty-quickcheck, tasty-silver, tasty-th
Dependencies removed: QuickCheck, ascii-progress, control-monad-omega, stringable, test-framework, test-framework-quickcheck2, test-framework-th
Dependency ranges changed: ADPfusion, AlignmentAlgorithms, FormalGrammars, NaturalLanguageAlphabets, PrimitiveArray, attoparsec, base, cmdargs, deepseq, file-embed, fmlist, hashable, intern, lens, parallel, primitive, strict, text, text-format, tuple-th, unordered-containers, vector
Files
- Linguistics/Bigram.hs +0/−131
- Linguistics/Common.hs +0/−90
- Linguistics/TwoWay/Bigram.hs +0/−167
- Linguistics/TwoWay/Simple.hs +0/−150
- Linguistics/Word.hs +0/−77
- Linguistics/WordAlignment.hs +90/−0
- Linguistics/WordAlignment/AlignmentBuilder.hs +84/−0
- Linguistics/WordAlignment/Bigram.hs +152/−0
- Linguistics/WordAlignment/Common.hs +79/−0
- Linguistics/WordAlignment/FastLookups.hs +33/−0
- Linguistics/WordAlignment/PipedPairs.hs +179/−0
- Linguistics/WordAlignment/TwoWay/Global/Bigram.hs +136/−0
- Linguistics/WordAlignment/TwoWay/Global/Simple.hs +104/−0
- Linguistics/WordAlignment/Word.hs +147/−0
- README.md +193/−77
- WordAlignment.cabal +120/−56
- changelog.md +12/−0
- scores/defaultBigramScores +14/−0
- scores/defaultSimpleScores +22/−0
- src/WordAlign.hs +198/−515
- stack.yaml +20/−0
- tests/PTSP800.bgdef +14/−0
- tests/PTSP800.bgms +779/−0
- tests/PTSP800.ugdef +22/−0
- tests/PTpoSPpampa.words +2/−0
- tests/PTsoloSPsuelo.words +2/−0
- tests/example.bgdef +14/−0
- tests/example.bgms +8/−0
- tests/example.ugdef +22/−0
- tests/example.words +6/−0
- tests/global-bigram-PTSP800-PTpoSPpampa-1.golden +15/−0
- tests/global-bigram-PTSP800-PTsoloSPsuelo-1.golden +15/−0
- tests/global-bigram-example-example-1.golden +105/−0
- tests/global-bigram-order-order-1.golden +30/−0
- tests/global-unigram-PTSP800-PTpoSPpampa-1.golden +15/−0
- tests/global-unigram-PTSP800-PTsoloSPsuelo-1.golden +15/−0
- tests/global-unigram-example-example-1.golden +105/−0
- tests/order.bgdef +14/−0
- tests/order.bgms +2/−0
- tests/order.words +3/−0
- tests/properties.hs +122/−3
− Linguistics/Bigram.hs
@@ -1,131 +0,0 @@---- | Map between 'String's that represent characters and their 'Int'-based--- representation.------ NOTE filtering the scores list and creating a single bigram map takes about--- 70 seconds.------ NOTE A single bigram map costs around 160 MByte ram. This includes the--- overhead for actually storing the bigrams once (creating pointers instead of--- multiple copied 'Bigram' data structures.--module Linguistics.Bigram where--import Control.Applicative-import Control.Arrow-import Control.DeepSeq-import Control.Lens-import Data.Attoparsec.ByteString.Lazy ((<?>))-import Data.ByteString (ByteString)-import Data.Function-import Data.Hashable-import Data.Interned-import Data.List-import Data.Strict.Tuple-import GHC.Generics (Generic)-import qualified Data.Attoparsec.ByteString as AB-import qualified Data.Attoparsec.ByteString.Char8 as AB hiding (takeWhile1)-import qualified Data.Attoparsec.ByteString.Lazy as ABL-import qualified Data.ByteString as B-import qualified Data.ByteString.Lazy as BL hiding (unpack)-import qualified Data.ByteString.Lazy.Char8 as BL hiding (readFile)-import qualified Data.ByteString.Short as BS-import qualified Data.HashMap.Strict as HM-import qualified Data.Map.Strict as M-import qualified Data.Set as S-import qualified Data.Stringable as SA--import NLP.Text.BTI----data Bigram = Bigram- { peekChar :: {-# UNPACK #-} !BTI- , hitChar :: {-# UNPACK #-} !BTI- }- deriving (Show,Eq,Ord,Generic)--instance Hashable Bigram where- hashWithSalt s (Bigram p h) = hashWithSalt s (p,h) -- (uninternMultiChar p, uninternMultiChar h)- hash (Bigram p h) = hash (hash p , hash h)- {-# Inline hashWithSalt #-}- {-# Inline hash #-}--instance NFData Bigram where- rnf !(Bigram a b) = ()--instance Hashable (Pair Int Int) where- hashWithSalt s (a:!:b) = hashWithSalt s (a,b)- {-# Inline hashWithSalt #-}---- | Try to read the first line to figure out if there is a default score there--withDefault :: Double -> [BL.ByteString] -> (Double,[BL.ByteString])-withDefault d [] = (d,[])-withDefault d (x:xs)- | [(rd,"")] <- readsPrec 0 (BL.unpack x) = (rd,xs)- | otherwise = (d,(x:xs))--parseLine l = case ABL.eitherResult (ABL.parse go l) of- Left err -> error err- Right p -> force p- where- go = (,,,,) <$> lang <*> lang <*> bigram <*> bigram <*> score <?> "go"- lang = wrd <?> "lang"- bigram = Bigram <$> wrd <*> wrd <?> "bigram"- score = AB.double <?> "score"- wrd = SA.fromByteString <$> AB.takeWhile1 (not . AB.isHorizontalSpace) <* AB.space--type Lang = BTI-type Line = (Lang, Lang, Bigram, Bigram, Double)-type Scores = HM.HashMap (Bigram:!:Bigram) Double--data Mapping = Mapping- { bigrams :: !(M.Map Bigram Bigram)- , lliid :: !(M.Map (Lang:!:Lang) Scores)- }- deriving (Show)--instance Hashable (Pair Bigram Bigram) where- hashWithSalt s (a:!:b) = hashWithSalt s (a,b)--lines2mapping :: [Line] -> Mapping-lines2mapping = foldl' mkMapping emptyMapping . concatMap dupGroup . groupBy ((==) `on` ((^._1) &&& (^._2))) where- dupGroup ls@(l:_)- | l^._1 == l^._2 = [ls]- | otherwise = [ls,ls']- where ls' = map (\(l1,l2,b1,b2,d) -> (l2,l1,b2,b1,d)) . filter (\l -> l^._1 /= l^._2) $ ls--emptyMapping = let b = Bigram "" ""- in Mapping (M.singleton b b) M.empty--mkMapping :: Mapping -> [Line] -> Mapping-mkMapping !m [] = m-mkMapping !(Mapping bs ll) xs@(x:_)- | otherwise = Mapping bs' ll'- where- nom = filter (`M.notMember` bs) $ map (^._3) xs ++ map (^._4) xs- bs' = bs `M.union` (M.fromList $ map (\a -> (a,a)) nom)- ll' = M.insertWith HM.union (x^._1 :!: x^._2) ys ll- ys :: Scores- ys = HM.fromList- [ ((k1:!:k2),d)- | y <- xs- , let k1 = bs' M.! (y^._3)- , let k2 = bs' M.! (y^._4)- , let d = y ^._5- ]---- | Given a set of acceptable languages, a default score, and the lazy--- bytestring of scores, create the 'Mapping' of languages and scores.--generateLookups :: S.Set BTI -> Double -> BL.ByteString -> Mapping-generateLookups langs wd b = lines2mapping xs where- (d,ls) = withDefault wd $ BL.lines b- xs = filter inLangSet $ map parseLine ls- inLangSet l- | S.null langs = True- | (l^._1) `S.member` langs- && (l^._2) `S.member` langs = True- | otherwise = False-
− Linguistics/Common.hs
@@ -1,90 +0,0 @@---- | Some common functions and things that are not of immediate importance to--- understand the algorithms.--module Linguistics.Common where--import Control.Monad (forM_)-import Data.ByteString (ByteString)-import Data.Char-import Data.List (transpose,reverse)-import qualified Data.ByteString.Short as S-import qualified Data.Text as T-import qualified Data.Text.Lazy as TL-import qualified Data.Text.Lazy.IO as TL-import qualified Data.Text.Lazy.Builder as TL-import Data.Text (Text)-import qualified Data.Text.Format as TF-import qualified Data.Text.Encoding as T-import Text.Printf-import Data.Stringable (toString)-import GHC.IO.Handle--import NLP.Text.BTI----type IMCp = (BTI, BTI)---- | Actually align something prettily--alignPretty :: [[IMCp]] -> [String]-alignPretty xss = map concat . transpose . map (\xs -> map (f xs) xs) . transpose . map reverse $ xss where- f zs x = printAligned x zs---- | Prettyprint ``characters'', which are actually small bytestrings.--printAligned = printAlignedPad ' '---- | Print with special padding character--printAlignedPad :: Char -> IMCp -> [IMCp] -> String-printAlignedPad p (_,c) zs = printf " %s%s" (replicate pad p) (toUtf8String c) where- pad :: Int- pad = (1+) . maximum $ 0 : map (\(_,x) -> printLength x - printLength c) zs---- | Length in /printed characters/ of an UTF8 string wrapped as a 'ByteString'------ NOTE 'isMark' selects unicode symbols that modify a character, thereby not--- increasing the length of the /printed/ string.--printLength :: BTI -> Int-printLength = length . filter isAN . toUtf8String where- isAN c = not (isMark c) -- isAlphaNum c || c `elem` [ '\\', '\'', '^', '$', '-', '\'' ]---{-- } where prnt x z = let pad = max 0 (length (filter isAN $ pp z) - length (filter isAN $ pp x))- in printf " %s%s" (replicate pad ' ') (pp x)- ds x = ' ' : replicate (length $ filter isAN $ pp x) '-'- isAN c = isAlphaNum c || c `elem` [ '\\', '\'' ]--}---toUtf8String :: BTI -> String-toUtf8String = toString -- T.unpack . T.decodeUtf8 . conv-{-# INLINE toUtf8String #-}--buildLines :: [[Text]] -> TL.Builder-buildLines xss = s where- n = (1+) . maximum $ 1 : (map (T.length . T.filter (not . isMark)) . concat $ xss)- yss = transpose xss- fmt = "%" ++ show n ++ "s"- s = mconcat [ (mconcat $ map (TF.left n ' ') ys) `mappend` "\n"- | ys <- yss ]--printLines :: Handle -> [[Text]] -> IO ()-printLines hndl xss = do- let n = (1+) . maximum $ 1 : (map (T.length . T.filter (not . isMark)) . concat $ xss)- let yss = transpose xss- let fmt = "%" ++ show n ++ "s"- let s = mconcat [ (mconcat $ map (TF.left n ' ') ys) `mappend` "\n"- | ys <- yss ]- TL.hPutStrLn hndl $ TL.toLazyText s- --forM_ yss $ \ys -> do- -- forM_ ys $ \y -> hPrintf hndl fmt (T.unpack y)- -- hPrintf hndl "\n"----conv = S.fromShort . getMultiChar . uninternMultiChar---{-# INLINE conv #-}-
− Linguistics/TwoWay/Bigram.hs
@@ -1,167 +0,0 @@-{-# LANGUAGE MultiWayIf #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeOperators #-}-{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE NoMonomorphismRestriction #-}--{-# OPTIONS_GHC -fno-liberate-case #-}--module Linguistics.TwoWay.Bigram where--import Data.ByteString.Char8 (ByteString)-import Data.FMList (FMList)-import Data.Sequence (Seq)-import Data.Strict.Tuple (Pair (..))-import Data.Stringable (toString,toText)-import Data.Text (Text,pack)-import Data.Vector.Fusion.Util (Id(..))-import Data.Vector.Unboxed (Vector)-import qualified Data.ByteString.Char8 as B-import qualified Data.HashMap.Strict as HM-import qualified Data.List as L-import qualified Data.Vector.Fusion.Stream.Monadic as SM-import qualified Data.Vector.Unboxed as VU-import Text.Printf-import qualified Data.Text.Format as TF--import ADP.Fusion-import Data.PrimitiveArray-import DP.Alignment.Global.Tapes2-import NLP.Scoring.SimpleUnigram-import NLP.Text.BTI--import Linguistics.Common-import Linguistics.Bigram----type IMC = BTI-type SigT m x r = SigGlobal m x r IMCp IMCp----sScore :: Monad m => Double -> Double -> Scores -> SigT m Double Double-sScore dS gapopen s = SigGlobal- { delin = \ww (Z:.c :._ ) -> ww + gapopen- , indel = \ww (Z:._ :.c ) -> ww + gapopen- , align = \ww (Z:.(lp,l):.(up,u)) -> ww + lkup up u lp l- , done = const 0- , h = SM.foldl' max (-888888)- } where- lkup mc' c nd' d = {-# SCC "lkup" #-} HM.lookupDefault dS (Bigram mc' c :!: Bigram nd' d) s- {-# INLINE lkup #-}-{-# INLINE sScore #-}-{--sScore dS gapOpen s = SigGlobal- { indel = \ww (Z:.():.(mc,c)) -> if | c=="$" -> -500000- | otherwise -> ww + gapOpen- , delin = \ww (Z:.(mc,c):.()) -> if | c=="$" -> -500000- | otherwise -> ww + gapOpen- , align = \ww (Z:.(mc,c):.(nd,d)) -> if | c=="^" && d=="$" -> -500000- | c=="$" && d=="^" -> -500000- | otherwise -> case (mc,nd) of- (Nothing , Nothing ) -> 0- (Just mc' , Just nd') -> ww + lkup mc' c nd' d- _ -> -500000- , done = const 0- , h = S.foldl' max (-500000)- } where- lkup mc' c nd' d = maybe dS id . unsafePerformIO $ H.lookup s (Bigram mc' c :!: Bigram nd' d)- {-# INLINE lkup #-}-{-# INLINE sScore #-}--}--sBacktrack :: Monad m => SigT m (FMList (IMCp,IMCp)) [FMList (IMCp,IMCp)]-sBacktrack = backtrack ("-","-") ("-","-")-{-# Inline sBacktrack #-}--sBacktrackFun :: Monad m => Double -> Double -> Scores -> SigT m (FMList [Text]) [FMList [Text]]-sBacktrackFun defS go sco = backtrackFun f g ("-","-") ("-","-") where- f cc@(mc',c) dd@(nd',d) = let z = HM.lookupDefault defS (Bigram mc' c :!: Bigram nd' d) sco- in [toText c,toText d, pack $ printf "%3.1f" z]- g (_ ,c) ( _ ,"-") = [toText c,"-", pack $ printf "%3.1f" go]- g (_, "-") (_,d) = ["-", toText d, pack $ printf "%3.1f" go]-{-# Inline sBacktrackFun #-}--alignGlobal :: Double -> Double -> Scores -> Int -> Vector IMC -> Vector IMC -> (Double,[[[Text]]])-alignGlobal ds gapopen scoring k i1' i2' = (d, take k bs) where -- . L.map runPrettyF . S.toList . unId $ axiom b) where- i1 = VU.zip i1' (VU.tail i1') ; i2 = VU.zip i2' (VU.tail i2')- n1 = VU.length i1 ; n2 = VU.length i2- !(Z:.t) = alignGlobalForward ds gapopen scoring i1 i2- d = unId $ axiom t- bs = alignGlobalBacktrack ds gapopen scoring i1 i2 t-{-# NoInline alignGlobal #-}--alignGlobalForward :: Double -> Double -> Scores -> Vector IMCp -> Vector IMCp -> Z:.ITbl Id Unboxed (Z:.PointL I:.PointL I) Double-alignGlobalForward ds gapopen scoring i1 i2 = {-# SCC "ali_forw" #-} mutateTablesDefault $ - gGlobal (sScore ds gapopen scoring)- (ITbl 0 0 (Z:.EmptyOk:.EmptyOk) (fromAssocs (Z:.PointL 0:.PointL 0) (Z:.PointL n1:.PointL n2) (-999999) []))- (chr i1) (chr i2)- where n1 = VU.length i1- n2 = VU.length i2-{-# NoInline alignGlobalForward #-}--alignGlobalBacktrack :: Double -> Double -> Scores -> Vector IMCp -> Vector IMCp -> ITbl Id Unboxed (Z:.PointL I:.PointL I) Double -> [[[Text]]]-alignGlobalBacktrack ds gapopen scoring i1 i2 t = {-# SCC "ali_back" #-} L.map runBacktrack . unId $ axiom b- where (Z:.b) = gGlobal (sScore ds gapopen scoring <|| sBacktrackFun ds gapopen scoring) (toBacktrack t (undefined :: Id a -> Id a)) (chr i1) (chr i2)-{-# NoInline alignGlobalBacktrack #-}--{---- | Backtrack the alignment--sAlign :: Monad m => STwoWay m Aligned (S.Stream m Aligned) (Maybe InternedMultiChar,InternedMultiChar) ()-sAlign = STwoWay- { loop_step = \(w1,w2) (Z:.():.(_,c)) -> ( w1 ++ ["-"], w2 ++ [c] ) -- (w1++padd "" c, w2++prnt c "")- , step_loop = \(w1,w2) (Z:.(_,c):.()) -> ( w1 ++ [c] , w2 ++ ["-"] ) -- (w1++prnt c "", w2++padd "" c)- , step_step = \(w1,w2) (Z:.(_,a):.(_,b)) -> ( w1 ++ [a], w2 ++ [b] ) -- (w1++prnt a b,w2++prnt b a)- , nil_nil = const ([],[])- , h = return . id- } where prnt x z = printAligned x [z]- padd x z = printAlignedPad '-' x [z]-{-# INLINE sAlign #-}---- | Wrap calculations--twoWay dS gapOpen scores i1 i2 = (ws ! (Z:.pointL 0 n1:.pointL 0 n2), bt) where- ws = unsafePerformIO (twoWayFill dS gapOpen scores i1 i2)- n1 = V.length i1- n2 = V.length i2- bt = backtrack dS gapOpen scores i1 i2 ws-{-# NOINLINE twoWay #-}---- | Forward phase--twoWayFill- :: Double- -> Double- -> Scores- -> V.Vector InternedMultiChar- -> V.Vector InternedMultiChar- -> IO (PA.Unboxed (Z:.PointL:.PointL) Double)-twoWayFill dS gapOpen scores i1 i2 = do- let n1 = V.length i1- let n2 = V.length i2- !t' <- newWithM (Z:.pointL 0 0:.pointL 0 0) (Z:.pointL 0 n1:.pointL 0 n2) 0- let w = mTbl (Z:.EmptyT:.EmptyT) t'- fillTable2 $ gTwoWay (sScore dS gapOpen scores) w (chrLeft i1) (chrLeft i2) Empty Empty- freeze t'-{-# NOINLINE twoWayFill #-}--backtrack- :: Double- -> Double- -> Scores- -> V.Vector InternedMultiChar- -> V.Vector InternedMultiChar- -> PA.Unboxed (Z:.PointL:.PointL) Double- -> [Aligned]-backtrack dS gapOpen scores i1 i2 tbl = unId . S.toList . unId $ g $ Z:.pointL 0 n1 :.pointL 0 n2 where- n1 = V.length i1- n2 = V.length i2- w :: DefBtTbl Id (Z:.PointL:.PointL) Double Aligned- w = btTbl (Z:.EmptyT:.EmptyT) tbl (g :: (Z:.PointL:.PointL) -> Id (S.Stream Id Aligned))- (Z:.(_,g)) = gTwoWay (sScore dS gapOpen scores <** sAlign) w (chrLeft i1) (chrLeft i2) Empty Empty-{-# NOINLINE backtrack #-}---}-
− Linguistics/TwoWay/Simple.hs
@@ -1,150 +0,0 @@-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeOperators #-}-{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE MultiWayIf #-}-{-# LANGUAGE NoMonomorphismRestriction #-}--{-# OPTIONS_GHC -fno-liberate-case #-}--module Linguistics.TwoWay.Simple where--import Data.ByteString.Char8 (ByteString)-import Data.Vector.Fusion.Util (Id(..))-import GHC.Exts-import qualified Data.ByteString.Char8 as B-import qualified Data.List as L-import qualified Data.Text as T-import qualified Data.Text.Encoding as T-import qualified Data.Vector.Fusion.Stream.Monadic as SM---import qualified Data.Vector.Fusion.Stream as S-import qualified Data.Vector.Unboxed as VU-import Data.Vector.Unboxed (Vector)-import System.IO.Unsafe (unsafePerformIO)-import Data.Sequence (Seq)-import Data.FMList (FMList)-import Data.Text (Text)-import Data.Stringable (toText)--import ADP.Fusion-import Data.PrimitiveArray-import DP.Alignment.Global.Tapes2-import NLP.Scoring.SimpleUnigram-import NLP.Text.BTI--import Linguistics.Common---import Linguistics.TwoWay.Common-----type IMC = InternedMultiChar-type SigT m x r = SigGlobal m x r BTI BTI--sScore :: Monad m => SimpleScoring -> SigT m Double Double-sScore ss@SimpleScoring{..} = SigGlobal- { delin = \ww (Z:.c:._) -> ww + gapScore- , indel = \ww (Z:._:.c) -> ww + gapScore- , align = \ww (Z:.l:.u) -> ww + scoreUnigram ss l u- , done = const 0- , h = SM.foldl' max (-888888)- }-{-# Inline sScore #-}--sBacktrack :: Monad m => SigT m (FMList (BTI,BTI)) [FMList (BTI,BTI)]-sBacktrack = backtrack "-" "-"-{-# Inline sBacktrack #-}---- | Create a backtracking function------ TODO includes scores as well?--sBacktrackFun :: Monad m => SigT m (FMList [Text]) [FMList [Text]]-sBacktrackFun = backtrackFun f g "-" "-" where- f c d = [toText c, toText d]- g c d = [toText c, toText d]--alignGlobal :: SimpleScoring -> Int -> Vector BTI -> Vector BTI -> (Double,[[[Text]]])-alignGlobal scoring k i1 i2 = (d, take k bs) where- n1 = VU.length i1 ; n2 = VU.length i2- !(Z:.t) = alignGlobalForward scoring i1 i2- d = unId $ axiom t- bs = alignGlobalBacktrack scoring i1 i2 t-{-# NoInline alignGlobal #-}--alignGlobalForward :: SimpleScoring -> Vector BTI -> Vector BTI -> Z:.ITbl Id Unboxed (Z:.PointL I:.PointL I) Double-alignGlobalForward scoring i1 i2 = {-# SCC "alignGlobalForward" #-} mutateTablesDefault $- gGlobal (sScore scoring)- (ITbl 0 0 (Z:.EmptyOk:.EmptyOk) (fromAssocs (Z:.PointL 0:.PointL 0) (Z:.PointL n1:.PointL n2) (-999999) []))- (chr i1) (chr i2)- where n1 = VU.length i1- n2 = VU.length i2-{-# NoInline alignGlobalForward #-}--alignGlobalBacktrack :: SimpleScoring -> Vector BTI -> Vector BTI -> ITbl Id Unboxed (Z:.PointL I:.PointL I) Double -> [[[Text]]]-alignGlobalBacktrack scoring i1 i2 t = {-# SCC "alignGlobalBacktrack" #-} L.map runBacktrack . unId $ axiom b- where (Z:.b) = gGlobal (sScore scoring <|| sBacktrackFun) (toBacktrack t (undefined :: Id a -> Id a)) (chr i1) (chr i2)-{-# NoInline alignGlobalBacktrack #-}---- | Decoupling the forward phase for CORE observation.--{--alignGlobalForward :: Vector IMC -> Vector IMC -> Z:.(ITbl Id Unboxed (Z:.PointL:.PointL) Double)-alignGlobalForward i1 i2 = let n1 = VU.length i1; n2 = VU.length i2 in mutateTablesDefault $- gGlobal score- (ITbl 0 0 (Z:.EmptyOk:.EmptyOk) (PA.fromAssocs (Z:.PointL 0:.PointL 0) (Z:.PointL n1:.PointL n2) (-999999) []))- (chr i1) (chr i2)-{-# NoInline runNeedlemanWunschForward #-}--}----{--sAlign :: Monad m => STwoWay m Aligned (S.Stream m Aligned) InternedMultiChar ()-sAlign = STwoWay- { loop_step = \[w1,w2] (Z:.():.c) -> [ "-" : w1 , c : w2 ]- , step_loop = \[w1,w2] (Z:.c:.()) -> [ c : w1 , "-" : w2 ]- , step_step = \[w1,w2] (Z:.a:.b) -> [ a : w1 , b : w2 ]- , nil_nil = const [[],[]]- , h = return . id- }-{-# INLINE sAlign #-}----twoWay :: VU.Vector Char -> VU.Vector Char -> [Double] -> Double -> V.Vector ByteString -> V.Vector ByteString -> (Double-twoWay simpleScoring i1 i2 = (ws ! (Z:.pointL 0 n1:.pointL 0 n2), bt) where- ws = unsafePerformIO $ twoWayFill simpleScoring i1 i2- n1 = V.length i1- n2 = V.length i2- bt = backtrack simpleScoring i1 i2 ws-{-# NOINLINE twoWay #-}--twoWayFill- :: SimpleScoring- -> V.Vector InternedMultiChar- -> V.Vector InternedMultiChar- -> IO (PA.Unboxed (Z:.PointL:.PointL) Double)-twoWayFill simpleScoring i1 i2 = do- let n1 = V.length i1- let n2 = V.length i2- !t' <- newWithM (Z:.pointL 0 0:.pointL 0 0) (Z:.pointL 0 n1:.pointL 0 n2) 0- let w = mTblD (Z:.EmptyOk:.EmptyOk) t'- fillTable2 $ gTwoWay (sScore simpleScoring) w (chr i1) (chr i2) Empty Empty- freeze t'-{-# NOINLINE twoWayFill #-}--backtrack- :: SimpleScoring- -> V.Vector InternedMultiChar- -> V.Vector InternedMultiChar- -> PA.Unboxed (Z:.PointL:.PointL) Double- -> [Aligned]-backtrack simpleScoring i1 i2 tbl = unId . S.toList . unId $ g $ Z:.pointL 0 n1 :.pointL 0 n2 where- n1 = V.length i1- n2 = V.length i2- w = btTblD (Z:.EmptyOk:.EmptyOk) tbl (g :: (Z:.PointL:.PointL) -> Id (S.Stream Id Aligned))- (Z:.(_,g)) = gTwoWay (sScore simpleScoring <** sAlign) w (chr i1) (chr i2) Empty Empty-{-# NOINLINE backtrack #-}----test s = twoWay (VU.fromList "aeiou") (VU.fromList $ ['a' .. 'z'] L.\\ "aeiou") [3,1,1,0,0,-1] (-1) s' s' where--- s' = V.fromList $ L.map (B.pack . (:[])) $ s---}-
− Linguistics/Word.hs
@@ -1,77 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE BangPatterns #-}---- | A single word in a language. Uses a 'MultiChar' encoding for the actual--- characters. MultiChar encodings need to be decoded for printing on screen.--module Linguistics.Word where--import Control.Applicative-import Control.DeepSeq-import Data.ByteString (ByteString)-import Data.Interned-import Data.Interned.ByteString-import qualified Data.Attoparsec.ByteString as AB-import qualified Data.Attoparsec.ByteString.Char8 as AB hiding (takeWhile1)-import qualified Data.Attoparsec.ByteString.Lazy as ABL-import qualified Data.ByteString as B-import qualified Data.ByteString.Lazy as BL hiding (unpack)-import qualified Data.ByteString.Lazy.Char8 as BL hiding (readFile)-import qualified Data.ByteString.Short as S-import qualified Data.Vector.Unboxed as VU-import Prelude hiding (Word)-import Data.Stringable--import NLP.Text.BTI------ | A single word we want to align to another word. It comes with an id (here--- 9), the language name (which we intern), a word class (interned as well),--- the length of the word (so that we don't have to check wordWord length and--- check for word delims), and finally the word itself. Indivitual 'MultiChar'--- characters are interned to reduce memory cost (and we might want to do stuff--- with the Id's).------ 9 Albanian_Tosk 1.214 6 \' b a lʸ t ə-----data Word = Word- { wordID :: {-# UNPACK #-} !Int- , wordClass :: {-# UNPACK #-} !BTI -- InternedByteString- , wordLang :: {-# UNPACK #-} !BTI -- InternedByteString- , wordLength :: {-# UNPACK #-} !Int- , wordWord :: {-# UNPACK #-} !(VU.Vector BTI)- }- deriving (Show,Eq,Ord)--instance NFData Word where- rnf !(Word {}) = ()--parseWord :: BL.ByteString -> Word-parseWord w = case ABL.eitherResult (ABL.parse go w) of- Left err -> error err- Right p -> force p- where- go = Word <$> AB.decimal- <* AB.many1 AB.space- <*> (wW <$> wrd)- <*> (wW <$> wrd)- <*> AB.decimal- <* AB.many1 AB.space- <*> ((VU.fromList . map wW) <$> (AB.takeWhile1 (not . AB.isHorizontalSpace) `AB.sepBy` AB.space))- wrd = AB.takeWhile1 (not . AB.isHorizontalSpace) <* AB.space- wW = fromByteString--addWordDelims :: Word -> Word-addWordDelims w- | VU.length ww >= 2 && VU.head ww == "^" && VU.last ww == "$" = w- | otherwise = w { wordWord = "^" `VU.cons` wordWord w `VU.snoc` "$" }- where ww = wordWord w--removeWordDelims :: Word -> Word-removeWordDelims w- | VU.length ww >= 2 && VU.head ww == "^" && VU.last ww == "$" = w { wordWord = VU.init . VU.tail $ wordWord w }- | otherwise = w- where ww = wordWord w-
+ Linguistics/WordAlignment.hs view
@@ -0,0 +1,90 @@++-- | Functions of alignments of words.++module Linguistics.WordAlignment+ ( module Linguistics.WordAlignment+ , buildAlignmentBuilder+ ) where++import Control.Arrow (second)+import Control.DeepSeq+import Control.Lens hiding (each)+import Control.Monad.State.Strict+import Data.Default+import Data.Function (on)+import Data.List (groupBy)+import Data.Text.Lazy.Builder (Builder)+import Data.Vector (Vector)+import Pipes hiding ((<~))+import Prelude hiding (Word)+import qualified Data.Vector as V+import System.IO (stderr)+import Text.Printf+import qualified Data.Vector.Unboxed as VU++import Data.Paired.Vector+import NLP.Text.BTI++import Linguistics.WordAlignment.AlignmentBuilder+import Linguistics.WordAlignment.PipedPairs+import Linguistics.WordAlignment.FastLookups+import Linguistics.WordAlignment.Word (Word, wordLang, wordWord, FastChars, fastChars)+import qualified Linguistics.WordAlignment.TwoWay.Global.Bigram as GB2+import qualified Linguistics.WordAlignment.TwoWay.Global.Simple as GS2+--import qualified Linguistics.WordAlignment.TwoWay.Infix.Bigram as IB2+--import qualified Linguistics.WordAlignment.TwoWay.Infix.Simple as IS2++++alignGlobalSimple2 = GS2.alignGlobal+alignGlobalBigram2 = GB2.alignGlobal++--alignInfixSimple2 = IS2.alignInfix+--alignInfixBigram2 = IB2.alignInfix++-- | Wrap up the most common way to perform alignments. The function @f@+-- needs to have all scoring parts incorporated.++alignmentWrapper2 f fga fc fd x y = do+ w <- use aliWidth+ c <- use aliNumCoopts+ sf <- use aliFilterScore+ bf <- use aliFilterBackt+ fn <- use aliFilterNormalized+ let (d,bts) = f fga fc fd w c (wordWord x) (wordWord y)+ let l = VU.length (wordWord x) `max` VU.length (wordWord y)+ let ali = scoreFilter fn l sf d+ $ buildAlignmentBuilder 0 ([x,y],(d, btFilter fn l bf d bts))+ return ali+{-# Inline alignmentWrapper2 #-}++-- |++scoreFilter False _ (Just z) d _ | z > d = mempty+scoreFilter True l (Just z) d _ | z > d / fromIntegral l = mempty+scoreFilter _ _ _ _ blder = blder+{-# Inline scoreFilter #-}++-- |++btFilter False _ (Just z) d xs | z > d = []+btFilter True l (Just z) d xs | z > d / fromIntegral l = []+btFilter _ _ _ _ xs = xs+{-# Inline btFilter #-}++-- | Default system for printing out the status every 10k alignments.++eachGroupStatus len k x y = do+ let wLx :: String = btiToCS $ wordLang x+ wLy :: String = btiToCS $ wordLang y+ v <- use aliVerbose+ numGs <- use aliGroups+ curG <- use aliCurGroup+ lift . when (v && k `mod` 10000 == 9999)+ $ hPrintf stderr "%5d %5d %s %s %10d %10d\n" numGs curG wLx wLy len (k+1)+{-# Inline eachGroupStatus #-}++-- | In case we don't want to do anything for each group.++eachGroupNothing _ _ _ _ = return ()+
+ Linguistics/WordAlignment/AlignmentBuilder.hs view
@@ -0,0 +1,84 @@++module Linguistics.WordAlignment.AlignmentBuilder where++import Control.Lens+import Data.List (intersperse)+import Data.Monoid+--import Data.Text.Lazy.Builder (Builder)+import Prelude hiding (Word)+import qualified Data.Text.Format as TF+import qualified Data.Text.Lazy.Builder as TLB+import qualified Data.Vector.Unboxed as VU+import Data.ByteString.Builder (Builder)+import qualified Data.ByteString.Builder as BB+import Data.Text.Lazy.Encoding (encodeUtf8)++import NLP.Text.BTI++import Linguistics.WordAlignment.Common+import Linguistics.WordAlignment.Word (Word(..), wordUtf8Builder)++++-- | Build an actual k-way alignment.++buildAlignmentBuilder :: BuildAli t => Double -> ([Word],(Double,[t])) -> Builder+buildAlignmentBuilder k (ws,(s,xss)) = {-# SCC "buildAliBuilder" #-} hdr <> "\n" <> ls <> "\n"+ where+ hdr = "IDS: " <> wids <> " SCORE: " <> score <> " NSCORE: " <> nscore <> " " <> words+ wids = mconcat . map (mappend " " . BB.intDec . wordID) $ ws+ score = BB.lazyByteString . encodeUtf8 . TF.format "{}" . TF.Only . TF.left 6 ' ' $ TF.fixed 2 s+ nscore = BB.lazyByteString . encodeUtf8 . TF.format "{}" . TF.Only . TF.left 6 ' ' $ TF.fixed 2 ns+ words = mconcat . map (mappend " WORD: " . wordUtf8Builder) $ ws+ ls = mconcat $ map buildAli xss+ ns = s / (maximum $ 1 : map ((+k) . fromIntegral . VU.length . wordWord) ws)+ {-+ where hdr = TF.build "IDS:{} SCORE: {} NSCORE: {} {}"+ ( wids+ , TF.left 6 ' ' $ TF.fixed 2 s+ , TF.left 6 ' ' $ TF.fixed 2 normScore+ , words+ )+ wids = mconcat . map (TF.build " {}" . TF.Only . wordID) $ ws+ words = mconcat . map (TF.build " WORD: {}" . TF.Only . wordLazyTextWSB) $ ws+ normScore = s / (maximum $ 1 : map ((+k) . fromIntegral . VU.length . wordWord) ws)+ -}++-- | During backtracking, a list of tuples is created. Here, out of a list+-- of tuples, a tuple of lists is created and turned into lines of+-- builders.++class BuildAli t where+ buildAli ::t -> Builder++-- | Instance for three lines, here used for 2-way alignments with a third+-- line for scores.++instance BuildAli [(Builder, Builder, Builder)] where+ buildAli t =+ let l1 = t ^. traverse . _1+ l2 = t ^. traverse . _2+ l3 = t ^. traverse . _3+ in l1 <> "\n" <> l2 <> "\n" <> l3 <> "\n"++-- | Four lines: 3-way alignment + score.++instance BuildAli [(Builder, Builder, Builder, Builder)] where+ buildAli t =+ let l1 = t ^. traverse . _1+ l2 = t ^. traverse . _2+ l3 = t ^. traverse . _3+ l4 = t ^. traverse . _4+ in l1 <> "\n" <> l2 <> "\n" <> l3 <> "\n" <> l4 <> "\n"++-- | Five lines: 4-way alignment + score.++instance BuildAli [(Builder, Builder, Builder, Builder, Builder)] where+ buildAli t =+ let l1 = t ^. traverse . _1+ l2 = t ^. traverse . _2+ l3 = t ^. traverse . _3+ l4 = t ^. traverse . _4+ l5 = t ^. traverse . _5+ in l1 <> "\n" <> l2 <> "\n" <> l3 <> "\n" <> l4 <> "\n" <> l5 <> "\n"+
+ Linguistics/WordAlignment/Bigram.hs view
@@ -0,0 +1,152 @@++-- | Map between 'String's that represent characters and their 'Int'-based+-- representation.+--+-- NOTE filtering the scores list and creating a single bigram map takes about+-- 70 seconds.+--+-- NOTE A single bigram map costs around 160 MByte ram. This includes the+-- overhead for actually storing the bigrams once (creating pointers instead of+-- multiple copied 'Bigram' data structures.++module Linguistics.WordAlignment.Bigram where++import Control.Applicative+import Control.Arrow+import Control.DeepSeq+import Control.Lens+import Data.Attoparsec.ByteString.Lazy ((<?>))+import Data.ByteString (ByteString)+import Data.Function+import Data.Hashable+import Data.Interned+import Data.List+import Data.Strict.Tuple+import GHC.Generics (Generic)+import qualified Data.Attoparsec.ByteString as AB+import qualified Data.Attoparsec.ByteString.Char8 as AB hiding (takeWhile1)+import qualified Data.Attoparsec.ByteString.Lazy as ABL+import qualified Data.ByteString as B+import qualified Data.ByteString.Lazy as BL hiding (unpack)+import qualified Data.ByteString.Lazy.Char8 as BL hiding (readFile)+import qualified Data.ByteString.Short as BS+import qualified Data.HashMap.Strict as HM+import qualified Data.Map.Strict as M+import qualified Data.Set as S+import Debug.Trace+import Text.Printf++import NLP.Text.BTI++import Linguistics.WordAlignment.Common++++data Bigram = Bigram+ { peekChar :: {-# UNPACK #-} !BTI+ , hitChar :: {-# UNPACK #-} !BTI+ }+ deriving (Show,Eq,Ord,Generic)++instance Hashable Bigram where+ hashWithSalt s (Bigram p h) = hashWithSalt s (p,h) -- (uninternMultiChar p, uninternMultiChar h)+ hash (Bigram p h) = hash (hash p , hash h)+ {-# Inline hashWithSalt #-}+ {-# Inline hash #-}++instance NFData Bigram where+ rnf !(Bigram a b) = ()++instance Hashable (Pair Int Int) where+ hashWithSalt s (a:!:b) = hashWithSalt s (a,b)+ {-# Inline hashWithSalt #-}++-- | Try to read the first line to figure out if there is a default score there++withDefault :: Double -> [BL.ByteString] -> (Double,[BL.ByteString])+withDefault d [] = (d,[])+withDefault d (x:xs)+ | [(rd,"")] <- readsPrec 0 (BL.unpack x) = (rd,xs)+ | otherwise = (d,(x:xs))++parseLine l = case ABL.eitherResult (ABL.parse go l) of+ Left err -> error $ "Linguistics.WordAlignment.Bigram:parseLine of: " ++ show l ++ " failed with: " ++ show err+ Right p -> force p+ where+ go = (,,,,) <$> lang <*> lang <*> bigram <*> bigram <*> score <?> "go"+ lang = wrd <?> "lang"+ bigram = Bigram <$> wrd <*> wrd <?> "bigram"+ score = AB.double <?> "score"+ wrd = btiFromCS <$> AB.takeWhile1 (not . AB.isHorizontalSpace) <* AB.skipSpace <?> "word"+{-# NoInline parseLine #-}++type Lang = BTI+type Line = (Lang, Lang, Bigram, Bigram, Double)+type Scores = HM.HashMap (Bigram:!:Bigram) Double++data Mapping = Mapping+ { bigrams :: !(M.Map Bigram Bigram)+ , lliid :: !(M.Map (Lang:!:Lang) Scores)+ }+ deriving (Show)++instance Hashable (Pair Bigram Bigram) where+ hashWithSalt s (a:!:b) = hashWithSalt s (a,b)++-- |+--+-- This one is confusing. Given a bigram a-b :!: c-d from lang 1 to lang 2,+-- we create the bigram c-d :!: a-b from lang 2 to lang 1 as well.++lines2mapping :: [Line] -> Mapping+lines2mapping = foldl' mkMapping emptyMapping . concatMap dupGroup . groupBy ((==) `on` ((^._1) &&& (^._2))) where+ dupGroup ls@(l:_)+ | l^._1 == l^._2 = [ls]+ | otherwise = [ls,ls']+ where ls' = map (\(l1,l2,b1,b2,d) -> (l2,l1,b2,b1,d)) . filter (\l -> l^._1 /= l^._2) $ ls++emptyMapping = let b = Bigram "" ""+ in Mapping (M.singleton b b) M.empty++-- | Barf in case incompatible duplicated bigram have been found.++mkMapping :: Mapping -> [Line] -> Mapping+mkMapping !m [] = m+mkMapping !(Mapping bs ll) xs@(x:_)+ | otherwise = Mapping bs' ll'+ where+ nom = filter (`M.notMember` bs) $ map (^._3) xs ++ map (^._4) xs+ bs' = bs `M.union` (M.fromList $ map (\a -> (a,a)) nom)+ ll' = M.insertWith HM.union (x^._1 :!: x^._2) ys ll+ ys :: Scores+ ys = HM.fromListWith insertBarf+ [ ((k1:!:k2),d)+ | y <- xs+ , let k1 = bs' M.! (y^._3)+ , let k2 = bs' M.! (y^._4)+ , let d = y ^._5+ ]+ insertBarf dup1 dup2 = error $ "the following two elements are duplicated with language pair: " ++ show (x^._1,x^._2) ++ " " ++ show dup1 ++ " " ++ show dup2++-- | Given a set of acceptable languages, a default score, and the lazy+-- bytestring of scores, create the 'Mapping' of languages and scores.++mkBigramMap :: S.Set BTI -> Double -> BL.ByteString -> Mapping+mkBigramMap langs wd b = lines2mapping xs where+ (d,ls) = withDefault wd $ BL.lines b+ xs = filter inLangSet $ map parseLine ls+ inLangSet l+ | S.null langs = True+ | (l^._1) `S.member` langs+ && (l^._2) `S.member` langs = True+ | otherwise = False++-- | (write me)++getScores2 :: Bool -> Mapping -> Lang -> Lang -> Scores+getScores2 vrb ss a b+ | Just z <- M.lookup (a:!:b) (lliid ss) = z+ | vrb = trace (printf "Language pair %s %s not found in mapping! Returning empty hashmap\n" (toUtf8String a) (toUtf8String b))+ HM.empty+ | otherwise = HM.empty+
+ Linguistics/WordAlignment/Common.hs view
@@ -0,0 +1,79 @@++-- | Some common functions and things that are not of immediate importance to+-- understand the algorithms.++module Linguistics.WordAlignment.Common where++import Data.Char (isMark)+import Data.List (transpose,reverse)+import Data.Text (Text)+import GHC.IO.Handle (Handle)+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import qualified Data.Text.Format as TF+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Builder as TL+import qualified Data.Text.Lazy.IO as TL+import Text.Printf+import qualified Data.ByteString.Builder as BB++import NLP.Text.BTI++++type B3 = (BB.Builder, BB.Builder, BB.Builder)++data InfixScoring = InfixScoring+ { infixOpen :: !Double+ , infixExt :: !Double+ }++type IMCp = (BTI, BTI)++-- | Actually align something prettily++alignPretty :: [[IMCp]] -> [String]+alignPretty xss = map concat . transpose . map (\xs -> map (f xs) xs) . transpose . map reverse $ xss where+ f zs x = printAligned x zs++-- | Prettyprint ``characters'', which are actually small bytestrings.++printAligned = printAlignedPad ' '++-- | Print with special padding character++printAlignedPad :: Char -> IMCp -> [IMCp] -> String+printAlignedPad p (_,c) zs = printf " %s%s" (replicate pad p) (toUtf8String c) where+ pad :: Int+ pad = (1+) . maximum $ 0 : map (\(_,x) -> printLength x - printLength c) zs++-- | Length in /printed characters/ of an UTF8 string wrapped as a 'ByteString'+--+-- NOTE 'isMark' selects unicode symbols that modify a character, thereby not+-- increasing the length of the /printed/ string.++printLength :: BTI -> Int+printLength = length . filter isAN . toUtf8String where+ isAN c = not (isMark c)++toUtf8String :: BTI -> String+toUtf8String = btiToCS+{-# INLINE toUtf8String #-}++buildLines :: [[Text]] -> TL.Builder+buildLines xss = s where+ n = (1+) . maximum $ 1 : (map (T.length . T.filter (not . isMark)) . concat $ xss)+ yss = transpose xss+ fmt = "%" ++ show n ++ "s"+ s = mconcat [ (mconcat $ map (TF.left n ' ') ys) `mappend` "\n"+ | ys <- yss ]++printLines :: Handle -> [[Text]] -> IO ()+printLines hndl xss = do+ let n = (1+) . maximum $ 1 : (map (T.length . T.filter (not . isMark)) . concat $ xss)+ let yss = transpose xss+ let fmt = "%" ++ show n ++ "s"+ let s = mconcat [ (mconcat $ map (TF.left n ' ') ys) `mappend` "\n"+ | ys <- yss ]+ TL.hPutStrLn hndl $ TL.toLazyText s+
+ Linguistics/WordAlignment/FastLookups.hs view
@@ -0,0 +1,33 @@++module Linguistics.WordAlignment.FastLookups where++import qualified Data.HashMap.Strict as HM+import qualified Data.ByteString.Builder as BB+import qualified Data.Text as T+import qualified Data.Text.Format as TF+import qualified Data.Text.Lazy.Builder as TLB+import qualified Data.Text.Lazy as TL+import Data.Text.Encoding (encodeUtf8)+import Control.DeepSeq+import qualified Data.ByteString as B++++data FastDoubles = FastDoubles+ { fcTable :: !(HM.HashMap Double B.ByteString)+ , fcWidth :: !Int+ }++-- | Generate the fast lookup table++fastDoubles :: Int -> [Double] -> FastDoubles+fastDoubles width ds = {-# SCC "fastDoubles" #-} deepseq ds `seq` FastDoubles hm width+ where hm = HM.fromList . map fmt $ ds+ fmt k = (k , encodeUtf8 . TL.toStrict . TF.format "{}" . TF.Only . TF.left width ' ' $ TF.fixed 1 k)+{-# NoInline fastDoubles #-}++fastDouble :: FastDoubles -> Double -> BB.Builder -- TLB.Builder+fastDouble (FastDoubles hm width) k = {-# SCC "fastChar" #-} maybe encodek BB.byteString $ HM.lookup k hm+ where encodek = BB.byteString . encodeUtf8 . TL.toStrict . TF.format "{}" . TF.Only . TF.left width ' ' $ TF.fixed 1 k+{-# InlineAble fastDouble #-}+
+ Linguistics/WordAlignment/PipedPairs.hs view
@@ -0,0 +1,179 @@++-- | Creates producers for efficient enumeration of all language pairs and+-- for each language pair for all words. All words are either all @k * l@+-- words for two different languages, or the upper triangular matrix, if+-- a language is paired with itself.++module Linguistics.WordAlignment.PipedPairs where++import Control.Arrow (second)+import Control.DeepSeq+import Control.Lens hiding (each)+import Control.Monad.State.Strict+import Data.ByteString.Builder (Builder)+import Data.Default+import Data.Function (on)+import Data.List (groupBy,delete,partition)+import Data.Vector (Vector)+import Debug.Trace+import Pipes hiding ((<~))+import Prelude hiding (Word)+import qualified Data.Vector as V+import qualified Pipes.Prelude as PP+import System.Mem (performGC)++import Data.Paired.Vector+import NLP.Text.BTI++import Linguistics.WordAlignment.FastLookups+import Linguistics.WordAlignment.Word (Word, wordLang, FastChars, fastChars)++++-- | Alignment configuration for use in the Reader.++data AlignmentConfig t = AlignmentConfig+ { _aliWidth :: !Int -- ^ width of columns for the builder+ , _aliNumCoopts :: !Int -- ^ how many co-optimals to take+ , _aliGroups :: !Int -- ^ how many groups will be processed+ , _aliCurGroup :: !Int -- ^ current group+ , _aliFilterScore :: !(Maybe Double) -- ^ if @Just s@ then keep only scores @>= s@+ , _aliFilterBackt :: !(Maybe Double) -- ^ if @Just s@ then backtrack only for scores @>= s@+ , _aliFilterNormalized :: !Bool -- ^ apply filter on length-normalized scores+ , _aliFilterLanguages :: ![Either Int BTI] -- ^ if @not null@ then we want only the languages indicated here. @Left Int@ is the language index, @Right BTI@ the language name+ , _aliVerbose :: !Bool -- ^ be verbose (what that means depends on the functions)+ , _aliGroupLanguages :: ![BTI] -- ^ the languages used in the current group+ , _aliCustom :: !t -- ^ custom data+ }++instance Default t => Default (AlignmentConfig t) where+ def = AlignmentConfig+ { _aliWidth = 8+ , _aliNumCoopts = 1+ , _aliGroups = 0+ , _aliCurGroup = 0+ , _aliFilterScore = Nothing+ , _aliFilterBackt = Nothing+ , _aliFilterNormalized = False+ , _aliFilterLanguages = []+ , _aliVerbose = False+ , _aliGroupLanguages = []+ , _aliCustom = def+ }++makeLenses ''AlignmentConfig++++-- | Generic system for twoway alignments.+--+-- TODO this will almost surely force the @ps@ vector, which maybe is not+-- what we want.++runTwowayAlignments+ :: (Monad m)+ -- | Perform an action before each group is sent downstream+ => (Int -> Vector (Word,Word) -> AlignmentT t m t)+ -- | Perform the actual alignment+ -> (t -> FastChars -> FastDoubles -> Word -> Word -> AlignmentT t m Builder)+ -- | An action that might do something with each total length, pair+ -- counter, pairX, pairY tripel.+ -> (Int -> Int -> Word -> Word -> AlignmentT t m ())+ -- | Fast Doubles+ -> [Double]+ -- | The input words+ -> Vector Word+ -- | A producer of 'Builder's+ -> Producer Builder (AlignmentT t m) ()+runTwowayAlignments groupAction alignXY eachXY ds ws = do+ width <- use aliWidth+ let !fc = fastChars width ws+ let !fd = fastDoubles width ds+ -- Generate the pairs of languages+ for (languagePairProducer ws) $ \(lenPs,ps) -> do+ !t <- lift $ groupAction lenPs ps+ for (each $ V.indexed ps) $ \(k,(x,y)) -> do+ lift $ eachXY lenPs k x y+ lift (alignXY t fc fd x y) >>= yield+{-# Inline runTwowayAlignments #-}++-- | Given the full list of words in the vector @ws@, produce all pairs of+-- words to be aligned.+--+-- This function will produce an upper-triangular list if the two languages+-- are the same, since @(x,y)@ and @(y,x)@ are then the same in enumeration+-- due to @x,y ε S@. For different languages we have @x ε X@ and @y ε Y@+-- and need the full square set. This is implemented below.+--+-- @ws@ is evaluated to NF.+--+-- TODO re-add language filter!++languagePairProducer+ :: (Monad m)+ => Vector Word+ -> Producer (Int, Vector (Word,Word)) (AlignmentT t m) ()+languagePairProducer ws = do+ -- produces a vector with (wordLanguage, language index from 1, first index in @ws@, length of+ -- group)+ let gs = deepseq ws . V.fromList . map (\(k,g@((i,w):_)) -> (wordLang w,k,i,length g))+ . zip [1..] . groupBy ((==) `on` (wordLang . snd)) . V.toList . V.indexed $ ws+ -- from @gs@ we generate the upper-triangular pairs+ let (lenWgs,wgs) = second (V.toList . V.map mkGroup) . upperTriVG OnDiag $ gs+ aliGroups .= lenWgs+ -- each wgs+ for (each $ zip [0..] wgs) $+ \(k,w) -> do+ aliCurGroup .= k+ aliGroupLanguages .= w ^. _1+ afl <- use aliFilterLanguages+ let cls = zip (w^._1) (w^._2)+ when (null afl || accept afl cls) $+ yield $ w ^. _3+ where+ -- This builds up a group+ -- TODO the lang and index lists should be @nub@'bed, so that @accept@+ -- can actually drop acceptance criteria after testing?!+ mkGroup ( (langX,lixX,startX,lengthX) , (langY,lixY,startY,lengthY) )+ -- same language, perform upper-triangular number of comparisons.+ -- Test for starting position as well, in case we have non-contiguous+ -- data.+ | langX == langY && startX == startY+ = ([langX], [lixX], upperTriVG OnDiag (V.slice startX lengthX ws))+ | otherwise+ = ([langX,langY], [lixX,lixY], rectangularVG (V.slice startX lengthX ws) (V.slice startY lengthY ws))+ -- here, @accept []@ should actually be false now, since we test+ -- a non-empty list of acceptance conditions+ accept [] [] = True -- all languages found in constraints+ accept (Left k : as) cs | ([_], cs') <- partition ((k==) . snd) cs = accept as cs'+ accept (Right l : as) cs | ([_], cs') <- partition ((l==) . fst) cs = accept as cs'+ accept _ _ = False+{-# Inline languagePairProducer #-}++-- | All important information for running the alignment producer.++newtype AlignmentT t (m :: * -> *) a = AlignmentT+ { runAlignmentT :: StateT (AlignmentConfig t) m a+ }+ deriving+ ( Applicative+ , Functor+ , Monad+ , MonadState (AlignmentConfig t)+ , MonadTrans+ )++-- | Wrap up the full call to the monad transformer++runAlignment+ :: Monad m+ => Effect (AlignmentT t m) a -> AlignmentConfig t -> m a+runAlignment = evalStateT . runAlignmentT . runEffect+{-# Inline runAlignment #-}++-- | Useful default group action: perform garbage collection after a group+-- has been processed.++groupActionGC _ _ = lift performGC+{-# Inline groupActionGC #-}+
+ Linguistics/WordAlignment/TwoWay/Global/Bigram.hs view
@@ -0,0 +1,136 @@++module Linguistics.WordAlignment.TwoWay.Global.Bigram where++import Data.ByteString.Char8 (ByteString)+import Data.FMList (FMList)+import qualified Data.FMList as FM+import Data.Sequence (Seq)+import Data.Strict.Tuple (Pair (..))+import Data.Text (Text,pack)+import Data.Vector.Fusion.Util (Id(..))+import Data.Vector.Unboxed (Vector)+import qualified Data.ByteString.Char8 as B+import qualified Data.HashMap.Strict as HM+import qualified Data.List as L+import qualified Data.Vector.Fusion.Stream.Monadic as SM+import qualified Data.Vector.Unboxed as VU+import Text.Printf+import qualified Data.Text.Format as TF+import qualified Data.Text.Lazy.Builder as TLB+import qualified Data.ByteString.Builder as BB++import ADP.Fusion.Point+import Data.PrimitiveArray+import DP.Seq.Align.Global.Linear2+import NLP.Scoring.SimpleUnigram+import NLP.Text.BTI++import Linguistics.WordAlignment.Common+import Linguistics.WordAlignment.FastLookups+import Linguistics.WordAlignment.Bigram+import Linguistics.WordAlignment.Word (FastChars, fastChar, wordToBigramVector)++++type IMC = BTI+type SigT m x r = SigGlobal m x r IMCp IMCp++-- |+--+-- TODO for general amusement: which of the two tapes @l@ or @u@ belongs to+-- the first bigram in this?++sScore :: Monad m => SimpleScoring -> Scores -> SigT m Double Double+sScore !ss@SimpleScoring{..} !bgm = SigGlobal+ { delin = \ww (Z:.c :._ ) -> ww + gapScore+ , indel = \ww (Z:._ :.c ) -> ww + gapScore+ , align = \ww (Z:.(lp,l):.(up,u)) -> ww + HM.lookupDefault defMismatch (Bigram lp l :!: Bigram up u) bgm+ , done = const 0+ , h = SM.foldl' max (-888888)+ }+{-# INLINE sScore #-}++-- |++sBacktrackBuilder+ :: Monad m+ => FastChars+ -> FastDoubles+ -> Int+ -> SimpleScoring+ -> Scores+ -> SigT m (FMList B3) [FMList B3]+sBacktrackBuilder !fc !fd !k !ss@SimpleScoring{..} !bgm = SigGlobal+ { delin = \ww (Z:.(_ ,b):._ ) -> ww `FM.snoc` ( fastChar fc "-"+ , fastChar fc b+ , fastDouble fd gapScore+ )+ , indel = \ww (Z:._ :.(_ ,u)) -> ww `FM.snoc` ( fastChar fc u+ , fastChar fc "-"+ , fastDouble fd gapScore+ )+ , align = \ww (Z:.(lb,b):.(lu,u)) -> let z = HM.lookupDefault defMismatch (Bigram lb b :!: Bigram lu u) bgm+ in ww `FM.snoc` ( fastChar fc u+ , fastChar fc b+ , fastDouble fd z+ )+ , done = const $ FM.singleton ( fastChar fc "^"+ , fastChar fc "^"+ , fastChar fc "-"+ )+ , h = SM.toList+ }+{-# Inline sBacktrackBuilder #-}++-- |++alignGlobalForward+ :: SimpleScoring+ -> Scores+ -> Vector IMCp+ -> Vector IMCp+ -> Z:.TwITbl Id Unboxed (Z:.EmptyOk:.EmptyOk) (Z:.PointL I:.PointL I) Double+alignGlobalForward !simpleS !bgm !i1 !i2 = {-# SCC "ali_forw" #-} mutateTablesDefault $+ gGlobal (sScore simpleS bgm)+ (ITbl 0 0 (Z:.EmptyOk:.EmptyOk) (fromAssocs (Z:.PointL 0:.PointL 0) (Z:.PointL n1:.PointL n2) (-999999) []) )+ (chr i1) (chr i2)+ where !n1 = VU.length i1+ !n2 = VU.length i2+{-# NoInline alignGlobalForward #-}++-- |++alignGlobalBacktrackBuilder+ :: FastChars+ -> FastDoubles+ -> Int+ -> SimpleScoring+ -> Scores+ -> Vector IMCp+ -> Vector IMCp+ -> TwITbl Id Unboxed (Z:.EmptyOk:.EmptyOk) (Z:.PointL I:.PointL I) Double+ -> [[B3]]+alignGlobalBacktrackBuilder !fc !fd !width !simpleS !bgm !i1 !i2 !t = {-# SCC "AliBtBuilder" #-} L.map FM.toList . unId $ axiom b+ where (Z:.b) = gGlobal (sScore simpleS bgm <|| sBacktrackBuilder fc fd width simpleS bgm) (toBacktrack t (undefined :: Id a -> Id a)) (chr i1) (chr i2)+{-# NoInline alignGlobalBacktrackBuilder #-}++-- |++alignGlobal+ :: SimpleScoring+ -> Scores+ -> FastChars+ -> FastDoubles+ -> Int+ -> Int+ -> Vector BTI+ -> Vector BTI+ -> (Double,[[B3]])+alignGlobal simpleS bgm fc fd width k i1' i2' = {-# SCC "alignGlobal" #-} (d, take k bs)+ where i1 = wordToBigramVector i1' ; i2 = wordToBigramVector i2'+ n1 = VU.length i1 ; n2 = VU.length i2+ !(Z:.t) = alignGlobalForward simpleS bgm i1 i2+ d = unId $ axiom t+ bs = alignGlobalBacktrackBuilder fc fd width simpleS bgm i1 i2 t+{-# NoInline alignGlobal #-}+
+ Linguistics/WordAlignment/TwoWay/Global/Simple.hs view
@@ -0,0 +1,104 @@++module Linguistics.WordAlignment.TwoWay.Global.Simple where++import Data.ByteString.Char8 (ByteString)+import Data.FMList (FMList)+import Data.Sequence (Seq)+import Data.Text (Text)+import Data.Vector.Fusion.Util (Id(..))+import Data.Vector.Unboxed (Vector)+import GHC.Exts+import qualified Data.ByteString.Char8 as B+import qualified Data.FMList as FM+import qualified Data.List as L+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import qualified Data.Vector.Fusion.Stream.Monadic as SM+import qualified Data.Vector.Unboxed as VU+import System.IO.Unsafe (unsafePerformIO)+import qualified Data.Text.Format as TF+import qualified Data.ByteString.Builder as BB++import ADP.Fusion.Point+import Data.PrimitiveArray+import DP.Seq.Align.Global.Linear2+import NLP.Scoring.SimpleUnigram+import NLP.Text.BTI++import Linguistics.WordAlignment.FastLookups+import Linguistics.WordAlignment.Common+import Linguistics.WordAlignment.Word (FastChars, fastChar)++++type SigT m x r = SigGlobal m x r BTI BTI++-- |++sScore :: Monad m => SimpleScoring -> SigT m Double Double+sScore ss@SimpleScoring{..} = SigGlobal+ { delin = \ww (Z:.c:._) -> ww + gapScore+ , indel = \ww (Z:._:.c) -> ww + gapScore+ , align = \ww (Z:.l:.u) -> ww + scoreUnigram ss l u+ , done = const 0+ , h = SM.foldl' max (-888888)+ }+{-# Inline sScore #-}++-- |++sBacktrackBuilder :: Monad m => FastChars -> FastDoubles -> Int -> SimpleScoring -> SigT m (FMList B3) [FMList B3]+sBacktrackBuilder !fc !fd !k !ss@SimpleScoring{..} = SigGlobal+ { delin = \ww (Z:.b:._) -> ww `FM.snoc` ( fastChar fc "-"+ , fastChar fc b+ , fastDouble fd gapScore+ )+ , indel = \ww (Z:._:.u) -> ww `FM.snoc` ( fastChar fc u+ , fastChar fc "-"+ , fastDouble fd gapScore+ )+ , align = \ww (Z:.b:.u) -> ww `FM.snoc` ( fastChar fc u+ , fastChar fc b+ , fastDouble fd (scoreUnigram ss b u)+ )+ , done = const mempty+ , h = SM.toList+ }+{-# Inline sBacktrackBuilder #-}++-- |++alignGlobalForward :: SimpleScoring -> Vector BTI -> Vector BTI -> Z:.TwITbl Id Unboxed (Z:.EmptyOk:.EmptyOk) (Z:.PointL I:.PointL I) Double+alignGlobalForward !scoring !i1 !i2 = {-# SCC "alignGlobalForward" #-} mutateTablesST $+ gGlobal (sScore scoring)+ (ITbl 0 0 (Z:.EmptyOk:.EmptyOk) (fromAssocs (Z:.PointL 0:.PointL 0) (Z:.PointL n1:.PointL n2) (-999999) []))+ (chr i1) (chr i2)+ where n1 = VU.length i1+ n2 = VU.length i2+{-# NoInline alignGlobalForward #-}++-- |++alignGlobalBacktrack :: FastChars -> FastDoubles -> Int -> SimpleScoring -> Vector BTI -> Vector BTI -> TwITbl Id Unboxed (Z:.EmptyOk:.EmptyOk) (Z:.PointL I:.PointL I) Double -> [[B3]]+alignGlobalBacktrack !fc !fd !width !scoring !i1 !i2 !t = {-# SCC "alignGlobalBacktrack" #-} L.map FM.toList . unId $ axiom b+ where (Z:.b) = gGlobal (sScore scoring <|| sBacktrackBuilder fc fd width scoring) (toBacktrack t (undefined :: Id a -> Id a)) (chr i1) (chr i2)+{-# NoInline alignGlobalBacktrack #-}++-- |++alignGlobal+ :: SimpleScoring+ -> FastChars+ -> FastDoubles+ -> Int+ -> Int+ -> Vector BTI+ -> Vector BTI+ -> (Double,[[B3]])+alignGlobal scoring fc fd width k i1 i2 = (d, take k bs) where+ n1 = VU.length i1 ; n2 = VU.length i2+ !(Z:.t) = alignGlobalForward scoring i1 i2+ d = unId $ axiom t+ bs = alignGlobalBacktrack fc fd width scoring i1 i2 t+{-# NoInline alignGlobal #-}+
+ Linguistics/WordAlignment/Word.hs view
@@ -0,0 +1,147 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE BangPatterns #-}++-- | A single word in a language. Uses a 'MultiChar' encoding for the actual+-- characters. MultiChar encodings need to be decoded for printing on screen.++module Linguistics.WordAlignment.Word where++import Control.Applicative+import Control.DeepSeq+import Data.Aeson+import Data.Attoparsec.ByteString.Lazy ((<?>))+import Data.ByteString (ByteString)+import Data.Interned+import Data.Interned.ByteString+import Data.List (intersperse)+import GHC.Generics+import Prelude hiding (Word)+import qualified Data.Attoparsec.ByteString as AB+import qualified Data.Attoparsec.ByteString.Char8 as AB hiding (takeWhile1)+import qualified Data.Attoparsec.ByteString.Lazy as ABL+import qualified Data.ByteString as B+import qualified Data.ByteString.Builder as BB+import qualified Data.ByteString.Lazy as BL hiding (unpack)+import qualified Data.ByteString.Lazy.Char8 as BL hiding (readFile)+import qualified Data.ByteString.Short as S+import qualified Data.HashMap.Strict as HM+import qualified Data.Text as T+import qualified Data.Text.Format as TF+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Builder as TLB+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import Data.Text.Encoding (encodeUtf8)++import NLP.Text.BTI++++-- | A single word we want to align to another word. It comes with an id (here+-- 9), the language name (which we intern), a word class (interned as well),+-- the length of the word (so that we don't have to check wordWord length and+-- check for word delims), and finally the word itself. Indivitual 'MultiChar'+-- characters are interned to reduce memory cost (and we might want to do stuff+-- with the Id's).+--+-- 9 Albanian_Tosk 1.214 6 \' b a lʸ t ə+--++data Word = Word+ { wordID :: {-# UNPACK #-} !Int+ , wordClass :: {-# UNPACK #-} !BTI -- InternedByteString+ , wordLang :: {-# UNPACK #-} !BTI -- InternedByteString+ , wordLength :: {-# UNPACK #-} !Int+ , wordWord :: {-# UNPACK #-} !(VU.Vector BTI)+ }+ deriving (Show,Eq,Ord,Generic)++instance ToJSON Word where+ toJSON (Word i c lang len w) = object+ [ "id" .= i+ , "class" .= c+ , "lang" .= lang+ , "word" .= w+ ]++instance NFData Word where+ rnf !(Word {}) = ()++parseWord :: BL.ByteString -> Word+parseWord w = case ABL.eitherResult (ABL.parse go w) of+ Left err -> error $ "failed to parse line: " ++ show w ++ " with error: " ++ show err+ Right p -> force p+ where+ go = Word <$> (AB.decimal <?> "number")+ <* (AB.skipSpace <?> "1+ ws")+ <*> ((wW <$> wrd) <?> "class")+ <*> ((wW <$> wrd) <?> "language")+ <*> (AB.decimal <?> "length")+ <* (AB.skipSpace <?> "1+ ws")+ <*> ((VU.fromList . map wW) <$> (AB.takeWhile1 (not . AB.isHorizontalSpace) `AB.sepBy` AB.space) <?> "word")+ wrd = AB.takeWhile1 (not . AB.isHorizontalSpace) <* AB.skipSpace+ wW = btiFromCS++addWordDelims :: Word -> Word+addWordDelims w+ | VU.length ww >= 2 && VU.head ww == "^" && VU.last ww == "$" = w+ | otherwise = w { wordWord = "^" `VU.cons` wordWord w `VU.snoc` "$" }+ where ww = wordWord w++removeWordDelims :: Word -> Word+removeWordDelims w+ | VU.length ww >= 2 && VU.head ww == "^" && VU.last ww == "$" = w { wordWord = VU.init . VU.tail $ wordWord w }+ | otherwise = w+ where ww = wordWord w++-- | Format a 'Word' as a lazy text with interspersed white space.++wordLazyTextWS :: Word -> TL.Text+wordLazyTextWS = TLB.toLazyText . wordLazyTextWSB+{-# Inline wordLazyTextWS #-}++-- | A Builder for the actual word in a 'Word'. With intersperse white+-- space.++wordLazyTextWSB :: Word -> TLB.Builder+wordLazyTextWSB = mconcat . intersperse " " . map (TLB.fromText . btiToCS) . VU.toList . wordWord+{-# Inline wordLazyTextWSB #-}++-- | A builder for an utf8 bytestring.++wordUtf8Builder :: Word -> BB.Builder+wordUtf8Builder = mconcat . intersperse " " . map (BB.byteString . btiToUtf8) . VU.toList . wordWord+{-# Inline wordUtf8Builder #-}++-- |+--+-- TODO @Builder@ or @Text@ or @Lazy.Text@ ?++data FastChars = FastChars+ { fcTable :: !(HM.HashMap BTI B.ByteString)+ , fcWidth :: !Int+ }++-- | Generate the fast lookup table++fastChars :: Int -> V.Vector Word -> FastChars+fastChars width ws = {-# SCC "fastChars" #-} deepseq ws `seq` FastChars hm width+ where hm = HM.fromList . map fmt . addDefaultChars . concatMap (VU.toList . wordWord) . V.toList $ ws+ fmt k = (k , encodeUtf8 . TL.toStrict . TLB.toLazyText . TF.left width ' ' . btiToText $ k)+ addDefaultChars xs = xs ++ map btiFromCS ["-", "$", "^" :: T.Text]+{-# NoInline fastChars #-}++fastChar :: FastChars -> BTI -> BB.Builder -- TLB.Builder+--fastChar (FastChars hm width) k = {-# SCC "fastChar" #-} maybe (BB.byteString . encodeUtf8 . TF.format "{}" . TF.left width ' ' . TF.Only $ btiToCS k) BB.byteString $ HM.lookup k hm+fastChar (FastChars hm width) k = {-# SCC "fastChar" #-} maybe encodek BB.byteString $ HM.lookup k hm+ where encodek = BB.byteString . encodeUtf8 . TL.toStrict . TF.format "{}" . TF.Only . TF.left width ' ' $ (btiToCS k :: T.Text)+{-# InlineAble fastChar #-}++-- | Word to bigram vector++wordToBigramVector :: VU.Vector BTI -> VU.Vector (BTI,BTI)+wordToBigramVector m' =+ let m = "^" `VU.cons` m' `VU.snoc` "$"+ in VU.zip m (VU.tail m)+{-# Inline wordToBigramVector #-}+
README.md view
@@ -1,12 +1,51 @@ [](https://travis-ci.org/choener/WordAlignment) +++This manual can be displayed by calling just ```WordAlign``` or alternatively+```WordAlign manual```.+++ # Word alignments in natural languages This library and program are designed for the alignment of *words* in human-languages. Each word is encoded as a list of *characters*. A single-*character*, however, is encoded as a *list of unicode symbols*, not just a-single symbol.+languages. +Implemented with ideas described in:++1. Christian Hoener zu Siederdissen + *Sneaking Around ConcatMap: Efficient Combinators for Dynamic Programming* + 2012, Proceedings of the 17th ACM SIGPLAN international conference on Functional programming + [paper](http://doi.acm.org/10.1145/2364527.2364559) [preprint](http://www.tbi.univie.ac.at/newpapers/pdfs/TBI-p-2012-2.pdf) +1. Andrew Farmer, Christian Höner zu Siederdissen, and Andy Gill. + *The HERMIT in the stream: fusing stream fusion’s concatMap* + 2014, Proceedings of the ACM SIGPLAN 2014 workshop on Partial evaluation and program manipulation. + [paper](http://dl.acm.org/citation.cfm?doid=2543728.2543736) +1. Christian Höner zu Siederdissen, Ivo L. Hofacker, and Peter F. Stadler. + *Product Grammars for Alignment and Folding* + 2014, IEEE/ACM Transactions on Computational Biology and Bioinformatics. 99 + [paper](http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=6819790) +1. Christian Höner zu Siederdissen, Sonja J. Prohaska, and Peter F. Stadler + *Algebraic Dynamic Programming over General Data Structures* + 2015, BMC Bioinformatics + [preprint](http://www.bioinf.uni-leipzig.de/Software/gADP/preprints/hoe-pro-2015.pdf) ++++# Usage++Word alignments require three pieces of information: (i) the words to be+aligned, (ii) the selection of an alignment algorithm, and (iii) a scoring+scheme (either simple or bigram-based). Each of these is described below.++++## Words to be Aligned++Each word is encoded as a list of *characters*. A single *character*, however,+is encoded as a *list of unicode symbols*, not just a single symbol.+ This particular encoding is necessary because there is no general one-to-one mapping of atomic alphabet symbols to unicode characters. Just consider character decorations, such as *Umlaute* (even though Umlaute are available in@@ -29,51 +68,100 @@ -The WordAlignment program comes with two modes, *twowaysimple* and *twoway*.+## Selection of Alignment Algorithm -## TwoWay - Simple+The WordAlignment program comes with these modes: -```WordAlign twowaysimple``` aligns words based on a simple scoring model. Gaps-are scored with linear costs. Matches are scored based on a simple scoring file-handling unigrams of characters. Even the simple model allows for the concept-of equivalence classes. This makes it possible to not only score exact matches,-but to give somewhat high scores to, say, two vowels that should be matched.+| Grammar | Simple Scoring | Bigram Scoring | Number of Input Tapes |+|-----------|-----------------|-----------------|-----------------------|+| Global | global2simple | global2bigram | 2 |+| Overhang | global2infix | global2infix | 2 | -The following command-line options are provided:+Mode names can be shortened to unique prefixes. - --scorefile=ITEM- --lpblock=ITEM,ITEM- --showmanual- --prettystupid- --outfile=ITEM -```--scorefile``` is the simple score file to be used by the mode. the-*defaultSimpleScoring* file can be used as a template. +### Global Alignments++Global alignments use a NeedlemanWunsch style algorithm with linear gap costs.++++### Overhang alignments++Overhang alignments use affine gap scoring. Overhang alignments separate the+alignment into three phases, the prefix, infix, and suffix phase. Typically,+prefix and suffix phases have very low costs.++++### Options++#### Common default options++ -? --help Display help message+ -V --version Print version information+ --numeric-version Print just the version number+ -v --verbose Loud verbosity+ -q --quiet Quiet verbosity++```--verbose``` prints status information every 10.000 alignments++++#### Common options for all alignments variants++ --simplescorefile=ITEM the file to read the simple scores from+ -l --lpblock=ITEM,ITEM compare ONLY the given pair of languages: i.e+ 'Breton','Breton' or 2,3 (with the latter+ notation '2' being the 2nd language in the input+ file)+ --showmanual show the manual and quit+ --filterscore=NUM only print results with this score or higher+ --filterbacktrack=NUM only provide backtracking results for results+ with this score or higher++```--simplescorefile``` expects a score file for simple unigram based scoring.+An example file is provided under ```scores/defaultSimpleScoring```. For bigram+score files, use a file like ```scores/defaultBigramScoring```.+ ```--lpblock``` expects a pair of language names (Breton,Breton) or a pair of integers (3,3 or 4,6) and will then align only the given language pairs with each other. This option should be very helpful in case you want to parallelize the program. -```--showmanual``` will show this manual in plain text.+```--filterscore``` is used to limit printing results to only the alignments+with score not lower than this option. Given that printing requires a+significant amount of CPU time due to unicode conversion, this option improves+performance substantially. -```--prettystupid``` will show a progress bar of the current language pair.-It's pretty and helpful with smaller tasks but should not be used when you-parallelize on a grid engine.+```--filterbacktrack``` is used to limit printing of backtracking for a given+alignment to the best results. Works like ```--filterscore``` but will always+print the forward result. -```--outfile``` writes to the given output file, not stdout. Actually required-when ```--prettystupid``` is active.+```--filternormalized``` applies filter on the length-normalized scores instead+of the absolute ones. -```--nobacktrack``` disables backtracking. Sometimes just the alignment score-is enough. +#### Options for bigram alignment variants -## TwoWay (Complex Scoring)+ -b --bigramscorefile=ITEM the file to read the bigram scores from -The complex scoring model uses linear gap costs. In contrast to the simple-model above, however, character matching is now performed in a bigram context.+```--bigramscorefile``` is used to point toward a file with a list of bigram+scores for all language pairs. +++## Simple score file description++++## Bigram score file description++In contrast to the simple model above, however, character matching is now+performed in a bigram context.+ The required score file is currently using an in-house format with the following columns all required (with whitespace, not tab between the entries): @@ -91,80 +179,108 @@ Albanian_Tosk \' a Albanian_Tosk \' b 0.402228 Albanian_Tosk \' a Albanian_Tosk \' g 1.07432 -In addition to the scoring file, set via ```--scorefile```, the default score-constant and the gap cost constant need to be set. -```--bigramdef=NUM``` is the score given to unknown matches. -```--gapopen=NUM``` is the score given to each gap character (and not just the-opening score)--```--lpblock``` as above--```--showmanual``` as above--```--prettystupid``` as above--```--outfile``` as above--```--nobacktrack``` as above--An example output is given below.+# Example output The output are four lines for each alignment. An info line with the word ids (IDS), the alignment score (SCORE), the normalized scores (NSCORE) and the-actual words, started by (WORDS) and interleaved by (WORD). The next two lines-are the alignment, with deletions showning up as minus symbols (---) in the+actual words, started by (WORD) and interleaved by (WORD). The next two lines+are the alignment, with deletions showning up as minus symbols ```-``` in the deletion field. Note that a deletion does not delete a character from the input, it merely aligns an existing character in one alignment with the symbol-for deletion (--) in the other. The final line provides the per-column score+for deletion ```-``` in the other. The final line provides the per-column score for the alignment. After the alignment follows one empty line. -The normalized score is defined as SCORE / maximum of input word lengths+Words are written left-to-right in the information line, and bottom to top in+the alignment. -```cat albanian.input | head -n 3 | ./dist/build/WordAlign/WordAlign twoway -s albanian.scores```+ IDS: 2 3 SCORE: 93.40 NSCORE: 10.38 WORD: ^ h a u s b a u $ WORD: ^ b a u m h a u s $+ b a u m h a u s $ - - -+ - - - - h a u s b a u $+ 0.0 0.0 0.0 -1.0 -3.0 33.9 33.8 33.7 -3.0 -1.0 0.0 0.0 -yields: - IDS: 1 2 SCORE: 32.85 NSCORE: 5.48 WORDS: ^ d o u a r $ WORD ^ d o u a r $- ^ d o u a r $- ^ d o u a r $- 0.0 4.8 5.9 4.1 6.6 4.6 6.8- - IDS: 1 3 SCORE: -12.36 NSCORE: -1.77 WORDS: ^ d o u a r $ WORD ^ p o u l t r $- ^ d - o u a r - - $- ^ p o - u - l t r $- 0.0 0.3 -5.0 -5.0 4.1 -5.0 -20.0 -5.0 -5.0 6.8 --## Performance Notes+# Performance Notes -Measured on a core i5-3570K @ 3.40 GHz; single-threaded.+Measured on a core i5-3570K @ 3.40 GHz; single-threaded, based on 2001000 alignments. The program is not compiled for multi-threading, if you need this consider the-```--lpblock``` option first. Otherwise, send a mail.+```--lpblock``` option first and parallelize on the language pair level.+Otherwise, send a mail. -The running time for calculating 100 000 alignments is:+When aligning short words, as occur in human language, the approximate number+of alignments per second is: - Mode Seconds Alignments per Second- ==== ======= =====================- TwoWay Simple: 6.7 s ~ 15 000- TwoWay Complex: 15.8 s ~ 6 300+| Mode | Simple/Bigram | Tapes | Alignments per Second |+| :--- | :--- | :---: | ---: |+| Global | Simple | 2 | 34 076+| | Bigram | 2 | 26 600 -Disabling backtracking (to get just the alignment scores) improves performance-by roughly a factor of ```x 2```. Words are very short, backtracking overhead-is quite large!+and when printing alignments via ```--filterscore``` is restricted to scores+```>=10``` to return about 0.6% of alignments: -As in: the number of actual /unicode/ symbols is approximately the same as the-number of forward DP cells to be filled, since each /character/ can be a-combination of multiple unicdoe symbols.+| Mode | Simple/Bigram | Tapes | Alignments per Second |+| :--- | :--- | :---: | ---: |+| Global | Simple | 2 | 155 635+| | Bigram | 2 | 192 500 -## Three- and Four-way Alignments+# Three- and Four-way Alignments These are currently disabled. If you need them, consider contacting me.++++# Writing unit tests for WordAlign++First, prepare a version of WordAlign that includes the properties test+program. This is done as follows with a sufficiently new version of cabal:++ cabal new-configure --enable-tests+ cabal new-build++Once done, take a look at the ```tests``` folder. The files with suffix+```.golden``` are named in a special manner:+1. grammar type ```global``` or ```infix```+1. score system ```unigram``` or ```bigram```+1. the score files used; for unigrams a single ```.ugdef``` file is required,+ for bigrams both a ```.bgdef``` and a ```.bgms``` file are required.+ ```.ugdef``` files hold a unigram scoring system. ```.bgdef``` files bigram+ default scores. Last, ```.bgms``` files hold scores for known bigrams.+1. the input words file with the words to be aligned to each other. From the+ file, all pairs are created where fst <= snd. I.e. the upper triangular+ matrix of words.+1. a number with the the co-optimals to backtrack. Normally, this should be+ ```1``` but a small number of tests should have a big number here -- say+ ```99``` and all possible co-optimals should be checked for correctness.+ This may be fewer than the given number but at least one backtrack should+ always be there.++Running the properties program will then automatically test each golden file vs+the WordAlign algorithm together with the score files as inputs.++Automatic property are done with a simple ```./properties``` and should be all+green.++## How to create new test files:++1. Create a new *empty* file named like this example here:+ ```global-bigram-PTSP800-PTpoSPpampa-1.golden```.+2. This will test against the global Needleman-Wunsch algorithm with bigram+ scoring. The score file used is the ```PTSP800.bgms``` bigram score file and+ the ```PTSP800.bgdef``` file. Words are taken from the+ ```PTpoSPpampa.words``` file. Exactly one backtrack is generated.+3. Fill the necessary scoring and words files.+4. Run the ```properties``` program as follows: ```properties --accept -i```.+ This should fill the previously empty golden file created in step 1.+5. Carefully check if the contents of this file are correct.+6. If so a run of ```./properties``` should now only show green results. In+ particular, the file name created under 1. should show up as a new property+ that was tested.
WordAlignment.cabal view
@@ -1,7 +1,7 @@ name: WordAlignment-version: 0.1.0.0-author: Christian Hoener zu Siederdissen, 2013-2015-copyright: Christian Hoener zu Siederdissen, 2013-2015+version: 0.2.0.0+author: Christian Hoener zu Siederdissen, 2013-2017+copyright: Christian Hoener zu Siederdissen, 2013-2017 homepage: https://github.com/choener/WordAlignment bug-reports: https://github.com/choener/WordAlignment/issues maintainer: choener@bioinf.uni-leipzig.de@@ -11,77 +11,107 @@ build-type: Simple stability: experimental cabal-version: >= 1.10.0-tested-with: GHC == 7.8.4, GHC == 7.10.2+tested-with: GHC == 7.10.3, GHC == 8.0.1 synopsis: Bigram word pair alignments. description: The library provides fast dynamic programming algorithms to align word pairs using either a simple or a bigram scoring- scheme.+ scheme. Simple schemes are unigram schemes in nature but use an+ ad-hoc scoring scheme. Bigram schemes use actual training data+ for bigram frequences.+ .+ The WordAlign executable provides a wrapper around the provided+ alignment algorithms. Call WordAlign without any arguments (or+ just *WordAlign manual* to display the README.md). Extra-Source-Files:- README.md changelog.md+ README.md+ scores/defaultBigramScores+ scores/defaultSimpleScores+ stack.yaml+ -- we have lots of test files now that are being checked+ tests/*.bgdef+ tests/*.bgms+ tests/*.golden+ tests/*.ugdef+ tests/*.words +flag debug+ description: dump intermediate Core files+ default: False+ manual: True++flag llvm+ description: use llvm+ default: False+ manual: True+++ library- build-depends: base >= 4.7 && < 4.9- , ADPfusion >= 0.4.0.2 && < 0.5.1- , AlignmentAlgorithms >= 0.0.2 && < 0.0.3- , attoparsec >= 0.10 && < 0.14+ build-depends: base >= 4.7 && < 5.0+ , aeson >= 0.9+ , attoparsec >= 0.10 , bytestring , containers- , control-monad-omega >= 0.3 && < 0.4- , deepseq >= 1.3 && < 1.5- , file-embed >= 0.0.6 && < 0.1- , fmlist >= 0.9 && < 0.10- , FormalGrammars >= 0.2.1 && < 0.2.2+ , data-default >= 0.5+ , deepseq >= 1.3+ , file-embed >= 0.0.6+ , fmlist >= 0.9 , ghc-prim- , GrammarProducts >= 0.1.1 && < 0.1.2- , hashable >= 1.1 && < 1.3- , intern >= 0.9 && < 0.10- , lens >= 4.0 && < 4.14- , LinguisticsTypes >= 0.0.0 && < 0.0.1- , NaturalLanguageAlphabets >= 0.1.0 && < 0.2.0- , primitive >= 0.5 && < 0.7- , PrimitiveArray >= 0.6.0 && < 0.7.1- , strict >= 0.3.2 && < 0.3.3- , stringable >= 0.1.2 && < 0.1.4+ , hashable >= 1.1+ , intern >= 0.9+ , lens >= 4.0+ , mtl+ , pipes >= 4.0+ , primitive >= 0.5+ , strict >= 0.3.2 , template-haskell- , text >= 1.0 && < 1.3- , text-format >= 0.3 && < 0.4+ , text >= 1.0+ , text-format >= 0.3 , transformers- , tuple-th >= 0.2.4 && < 0.2.6- , unordered-containers >= 0.2.3 && < 0.2.6- , vector >= 0.10 && < 0.12+ , tuple-th >= 0.2.4+ , unordered-containers >= 0.2.3+ , vector >= 0.10+ --+ , ADPfusion == 0.5.2.*+ , AlignmentAlgorithms == 0.1.0.*+ , bimaps == 0.1.0.*+ , DPutils == 0.0.1.*+ , FormalGrammars == 0.3.1.*+ , GrammarProducts == 0.1.1.*+ , LinguisticsTypes == 0.0.0.*+ , NaturalLanguageAlphabets == 0.1.1.*+ , PrimitiveArray == 0.8.0.* exposed-modules:- Linguistics.Bigram- Linguistics.Common--- Linguistics.FourWay--- Linguistics.FourWay.Backward--- Linguistics.FourWay.Common--- Linguistics.FourWay.Forward--- Linguistics.ThreeWay--- Linguistics.ThreeWay.Bigram--- Linguistics.ThreeWay.Common--- Linguistics.ThreeWay.Simple--- Linguistics.TwoWay--- Linguistics.TwoWay.AdvancedBigram- Linguistics.TwoWay.Bigram--- Linguistics.TwoWay.Common- Linguistics.TwoWay.Simple- Linguistics.Word+ Linguistics.WordAlignment+ Linguistics.WordAlignment.AlignmentBuilder+ Linguistics.WordAlignment.Bigram+ Linguistics.WordAlignment.Common+ Linguistics.WordAlignment.FastLookups+ Linguistics.WordAlignment.PipedPairs+ Linguistics.WordAlignment.TwoWay.Global.Bigram+ Linguistics.WordAlignment.TwoWay.Global.Simple+-- Linguistics.WordAlignment.TwoWay.Infix.Bigram+-- Linguistics.WordAlignment.TwoWay.Infix.Simple+ Linguistics.WordAlignment.Word default-language: Haskell2010 default-extensions: BangPatterns , DeriveGeneric , FlexibleContexts , FlexibleInstances+ , GeneralizedNewtypeDeriving , OverloadedStrings , PatternGuards , QuasiQuotes+ , RecordWildCards+ , ScopedTypeVariables , StandaloneDeriving , TemplateHaskell , TypeFamilies@@ -89,24 +119,42 @@ ghc-options: -fsimpl-tick-factor=900 -O2+ -fexcess-precision+ if flag(debug)+ ghc-options:+ -ddump-to-file+ -ddump-simpl+ -dsuppress-all+ if flag(llvm)+ ghc-options:+ -fllvm+ -optlo-O3 executable WordAlign build-depends: base- , ascii-progress >= 0.3.2 && < 0.4+ , aeson , bytestring- , cmdargs == 0.10.*+ , cmdargs >= 0.10 , containers+ , data-default , file-embed , intern- , LinguisticsTypes- , NaturalLanguageAlphabets- , parallel >= 3.2 && < 3.3+ , lens+ , mtl+ , parallel >= 3.2+ , pipes , strict , text+ , text-format >= 0.3+ , transformers , unordered-containers , vector+ --+ , DPutils+ , LinguisticsTypes+ , NaturalLanguageAlphabets , WordAlignment main-is: WordAlign.hs@@ -116,6 +164,8 @@ Haskell2010 default-extensions: BangPatterns , DeriveDataTypeable+ , FlexibleContexts+ , NondecreasingIndentation , OverloadedStrings , PatternGuards , RecordWildCards@@ -127,6 +177,11 @@ -O2 -funfolding-use-threshold100 -funfolding-keeness-factor100+ -fexcess-precision+ if flag(llvm)+ ghc-options:+ -fllvm+ -optlo-O3 @@ -136,19 +191,28 @@ main-is: properties.hs ghc-options:- -threaded -rtsopts -with-rtsopts=-N+ -O2 -threaded -rtsopts -with-rtsopts=-N hs-source-dirs: tests default-language: Haskell2010 default-extensions: BangPatterns+ , OverloadedStrings , ScopedTypeVariables , TemplateHaskell build-depends: base- , QuickCheck- , test-framework >= 0.8 && < 0.9- , test-framework-quickcheck2 >= 0.3 && < 0.4- , test-framework-th >= 0.2 && < 0.3+ , bytestring+ , containers+ , filepath+ , split >= 0.2.3+ , tasty >= 0.11+ , tasty-quickcheck >= 0.8+ , tasty-silver >= 3.1.9+ , tasty-th >= 0.1+ , text+ --+ , DPutils+ , NaturalLanguageAlphabets , WordAlignment
changelog.md view
@@ -1,3 +1,15 @@+0.2.0.0+-------++- removed bigram scoring bug [this one deprecates all earlier versions]+- removed upper bounds+- flexible framework for unit tests, including a set of unit tests++0.1.x.y+-------++- updated README+ 0.1.0.0 -------
+ scores/defaultBigramScores view
@@ -0,0 +1,14 @@+-- a gap in linear scoring global alignments+Gap -4+-- gap opening in affine languages+GapOpen -4+-- gap extension in affine languages+GapExtend -2+-- opening the prefix or suffix in overhang languages+PreSufOpen -1+-- extending the prefix or suffix in overhang languages+PreSufExtend 0+-- score to return for bigram mismatches+Mismatch -3+-- needs to be set, but is not used (need better simple scoring parser)+Match -999999
+ scores/defaultSimpleScores view
@@ -0,0 +1,22 @@+Set Conso b c d f g h j k m n p q s t v w x y z+Set Liquid l r+Set Vowel a e i o u a\' e\' i\' o\' u\' ə ɐ æ œ æh ɔh ɒ ɔ+Eq Conso 4+Eq Liquid 3+Eq Vowel 2+InSet Conso Conso 0+InSet Conso Liquid -1+InSet Conso Vowel -999999+InSet Liquid Conso -1+InSet Liquid Liquid -1+InSet Liquid Vowel -1+InSet Vowel Conso -999999+InSet Vowel Liquid -1+InSet Vowel Vowel 0+Gap -4+GapOpen -4+GapExtend -2+PreSufOpen -1+PreSufExtend 0+Match 1+Mismatch -999999
src/WordAlign.hs view
@@ -3,102 +3,129 @@ module Main where -import Control.Arrow ((***))-import Control.Concurrent (threadDelay)+import Control.Arrow ((***),(&&&))+import Control.Lens import Control.Monad (forM_,when)-import Control.Parallel.Strategies (rdeepseq,parMap,parBuffer,using,evalTuple2,r0,rseq,evalBuffer,parList,evalList,evalTuple3,evalTuple5)+import Control.Monad.Trans.Class (lift)+import Data.Default import Data.FileEmbed-import Data.Function (on)-import Data.List (sortBy,groupBy,intersperse,genericLength)-import Data.Maybe (isJust)-import Data.Sequence (Seq)-import Data.Strict.Tuple-import Data.Text (Text)+import Data.List (nub) import Data.Version (showVersion)-import Debug.Trace-import Debug.Trace (trace)-import GHC.Conc (numCapabilities)-import GHC.IO.Handle+import Pipes (for) import Prelude hiding (Word)+import qualified Data.ByteString.Builder as BB import qualified Data.ByteString.Char8 as BS import qualified Data.ByteString.Lazy.Char8 as BL-import qualified Data.HashMap.Strict as HM-import qualified Data.Map.Strict as M import qualified Data.Set as S-import qualified Data.Text as T import qualified Data.Text.Lazy.Builder as TL import qualified Data.Text.Lazy.IO as TL import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU-import qualified System.Console.AsciiProgress as CAP-import System.Console.CmdArgs+import System.Console.CmdArgs hiding (def) import System.Exit-import System.IO+import System.IO (stderr, stdout, stdin)+import System.Mem (performGC) import Text.Printf-import Text.Read (readMaybe) import NLP.Scoring.SimpleUnigram import NLP.Scoring.SimpleUnigram.Import import NLP.Text.BTI -import Linguistics.Bigram-import Linguistics.Common-import Linguistics.TwoWay.Simple-import Linguistics.Word (parseWord,Word(..),addWordDelims)-import qualified Linguistics.TwoWay.Bigram as BI+import Linguistics.WordAlignment+import Linguistics.WordAlignment.FastLookups+import Linguistics.WordAlignment.AlignmentBuilder (BuildAli)+import Linguistics.WordAlignment.Bigram+import Linguistics.WordAlignment.Common+import Linguistics.WordAlignment.PipedPairs+import Linguistics.WordAlignment.TwoWay.Global.Simple+import Linguistics.WordAlignment.Word (parseWord,Word(..),addWordDelims,wordLazyTextWS,wordLazyTextWSB, fastChars, fastChar, FastChars)+import qualified Linguistics.WordAlignment.TwoWay.Global.Bigram as BI+--import qualified Linguistics.WordAlignment.TwoWay.Infix.Simple as IS import Paths_WordAlignment (version) data Config- = TwoWaySimple- { scoreFile :: String- , lpblock :: Maybe (String,String)- , showManual :: Bool- , prettystupid :: Bool- , outfile :: String- , nobacktrack :: Bool+ = Global2Simple+ { simpleScoreFile :: String+ , lpblock :: Maybe (String,String)+ , filterScore :: Maybe Double+ , filterBacktrack :: Maybe Double+ , filterNormalized :: Bool }- | TwoWay- { scoreFile :: String- , bigramDef :: Double--- , unibiDef :: Double- , gapOpen :: Double--- , gapExtend :: Double- , lpblock :: Maybe (String,String)- , showManual :: Bool- , prettystupid :: Bool- , outfile :: String- , nobacktrack :: Bool+ | Global2Bigram+ { simpleScoreFile :: String+ , bigramScoreFile :: String+ , lpblock :: Maybe (String,String)+ , filterScore :: Maybe Double+ , filterBacktrack :: Maybe Double+ , filterNormalized :: Bool }- deriving (Show,Data,Typeable)+ | Infix2Simple+ { simpleScoreFile :: String+ , lpblock :: Maybe (String,String)+ , filterScore :: Maybe Double+ , filterBacktrack :: Maybe Double+ , filterNormalized :: Bool+ }+ | Infix2Bigram+ { simpleScoreFile :: String+ , bigramScoreFile :: String+ , lpblock :: Maybe (String,String)+ , filterScore :: Maybe Double+ , filterBacktrack :: Maybe Double+ , filterNormalized :: Bool+ }+ | Manual+ {+ }+ deriving (Show,Eq,Data,Typeable) -twowaySimple = TwoWaySimple- { scoreFile = def &= help "the file to read the simple scores from"- , lpblock = Nothing- , showManual = False &= help "show the manual and quit"- , prettystupid = False- , outfile = ""- , nobacktrack = False+oGlobal2Simple = Global2Simple+ { simpleScoreFile = def &= help "the file to read the simple scores from"+ , lpblock = def &= help "compare ONLY the given pair of languages: i.e 'Breton','Breton' or 2,3 (with the latter notation '2' being the 2nd language in the input file)"+ , filterScore = def &= help "only print results with this score or higher"+ , filterBacktrack = def &= help "only provide backtracking results for results with this score or higher"+ , filterNormalized = False &= help "apply filters to length-normalized scores" } &= help "Align words based on a simple, linear scoring model for gaps, and an unigram model for matches." -twoway = TwoWay- { scoreFile = "" &= help "the file to read the scores from"- , bigramDef = (-20) &= help "score to use for unknown bigram matches"--- , unibiDef = (-5) &= help "score to close a gap if the closing characters are unknown"- , gapOpen = (-5) &= help "cost to open a gap"--- , gapExtend = (-1) &= help "cost to extend a gap" -- we currently do not use the affine cost model. Should come later once a generic affine system is written in AlignmentAlgorithms- , lpblock = Nothing &= help "compare ONLY the given pair of languages: i.e 'Breton','Breton' or 2,3 (with the latter notation '2' being the 2nd language in the input file)"- , prettystupid = False &= help "a pretty stupid developer option"- , outfile = "" &= help "write output to this file"- , showManual = False- , nobacktrack = False+oGlobal2Bigram = Global2Bigram+ { simpleScoreFile = def+ , lpblock = def+ , filterScore = def+ , filterBacktrack = def+ , filterNormalized = False &= help "apply filters to length-normalized scores"+ , bigramScoreFile = def &= help "the file to read the bigram scores from" } &= help "Align words based on a linear scoring model for gaps, but with bigram-based scoring for matches." -config = [twowaySimple, twoway]+oInfix2Simple = Infix2Simple+ { simpleScoreFile = def+ , lpblock = def+ , filterScore = def+ , filterBacktrack = def+ , filterNormalized = False &= help "apply filters to length-normalized scores"+ } &= help "Infix-Affine grammar with simple scoring. (VERY EXPERIMENTAL, YOU HAVE BEEN WARNED)"++oInfix2Bigram = Infix2Bigram+ { simpleScoreFile = def+ , lpblock = def+ , filterScore = def+ , filterBacktrack = def+ , filterNormalized = False &= help "apply filters to length-normalized scores"+ , bigramScoreFile = def+ } &= help "Infix-Affine grammar with simple scoring. (VERY EXPERIMENTAL, YOU HAVE BEEN WARNED)"++oManual = Manual+ {+ }++config = [ oGlobal2Simple, oGlobal2Bigram+-- , oInfix2Simple, oInfix2Bigram+ , oManual &= auto] &= program "WordAlign"- &= summary ("WordAlign " ++ showVersion version ++ " (c) Christian Höner zu Siederdissen 2014--2015, choener@bioinf.uni-leipzig.de")+ &= summary ("WordAlign " ++ showVersion version ++ " (c) Christian Höner zu Siederdissen 2014--2016, choener@bioinf.uni-leipzig.de")+ &= verbosity embeddedManual = $(embedFile "README.md") @@ -106,479 +133,135 @@ main = do o <- cmdArgs $ modes config- when (showManual o) $ do- BS.putStrLn embeddedManual- exitSuccess- when (prettystupid o && null (outfile o)) $ do- putStrLn "The --prettystupid mode requires giving and --outfile"- exitFailure- (if prettystupid o then CAP.displayConsoleRegions else id) $ do- ws <- BL.getContents >>= return . map parseWord . BL.lines+ if (o == Manual)+ then runManual o+ else do+ ws <- BL.getContents >>= return . V.fromList . map parseWord . BL.lines case o of- TwoWaySimple{..} -> run2Simple o (blockSelection2 lpblock ws)- TwoWay{..} -> run2 o (blockSelection2 lpblock $ map addWordDelims ws)------ | Given a @Config@ and a @List of Lists of Word-Pairs@ align everything.--run2Simple :: Config -> [[(Word,Word)]] -> IO ()-run2Simple TwoWaySimple{..} wss = do- hndl <- if null outfile then return stdout else openFile outfile AppendMode- scoring <- simpleScoreFromFile scoreFile- let wsslen = length wss- -- for each language pairing- forM_ (zip [1 :: Int ..] wss) $ \(k,ws) -> do- pg <- if prettystupid- then do- let len = genericLength ws- let (x,y) = head ws- printf "[%4d / %4d] Language pair: %s / %s with %d alignments:\n" k wsslen (show $ wordLang x) (show $ wordLang y) len- Just <$> CAP.newProgressBar CAP.def { CAP.pgWidth = 100, CAP.pgTotal = len }- else return Nothing- let alis = [ (x,y,d,bts,sss)- | (x,y) <- ws- , let (d,bts') = alignGlobal scoring 1 (wordWord x) (wordWord y) -- calculate score and all co-optimal backtraces- , let bts = if nobacktrack then [] else bts'- , let sss = TL.toLazyText $ buildAlignmentSimple 0 ([x,y],(d,bts)) -- make nice strings- ]- -- print the actual alignments- forM_ alis $ \(x,y,d,bts,sss) -> do- TL.hPutStr hndl sss- when (isJust pg) $ let Just pg' = pg in CAP.tick pg'---- | Given a @Config@ and a @List of List of Word-Pairs@ align everything.--run2 :: Config -> [[(Word,Word)]] -> IO ()-run2 TwoWay{..} wss = do- hndl <- if null outfile then return stdout else openFile outfile AppendMode- let wsslen = length wss- -- build up scoring system- let chkLs = S.fromList . map wordLang . concat . map (\(x,y) -> [x,y]) . map head $ wss- scoring <- BL.readFile scoreFile >>= return . generateLookups chkLs (-999999)- -- for each language pairing- forM_ (zip [1 :: Int ..] wss) $ \(k,ws) -> do- pg <- if prettystupid- then do- let len = genericLength ws- let (x,y) = head ws- printf "[%4d / %4d] Language pair: %s / %s with %d alignments:\n" k wsslen (show $ wordLang x) (show $ wordLang y) len- Just <$> CAP.newProgressBar CAP.def { CAP.pgWidth = 100, CAP.pgTotal = len }- else return Nothing- -- get score pairing- let (hx,hy) = head ws- let sco = getScores2 scoring (wordLang hx) (wordLang hy)- -- align the words the in @ws@ pairing- let as = map (\(x,y) -> ( let (d,bts) = BI.alignGlobal bigramDef gapOpen sco 1 (wordWord x) (wordWord y)- in TL.toLazyText $ buildAlignment (-1) ([x,y],(d,if nobacktrack then [] else bts))- )- ) ws- forM_ (as) {- `using` parBuffer 100 rseq)-} $ \ali -> do- when (isJust pg) $ let Just pg' = pg in CAP.tick pg'- TL.hPutStr hndl ali---- | Given a set of words from different languages, we want to do two--- things:------ (i) We want to block alignments into groups. This will make alignments--- faster as we do not have to reload the scoring system every time.------ (ii) We want to decide on alignment maybe only a subset of words. We--- make this selection rather coarse-grained by giving just the name or--- running id of the language. I.e you can type @Breton@ or @1@ (if Breton--- happens to be the first language). We allow numeric identification as--- that is easier for scripts to handle.--blockSelection2 :: Maybe (String,String) -> [Word] -> [[(Word,Word)]]-blockSelection2 s ws = go (mkCmp s)- -- grouping words by their languages- where gs = zip [1..] $ groupBy ((==) `on` wordLang) ws- ls = M.fromList $ map (\(k,(v:_)) -> (show $ wordLang v,k)) gs- go f = [ [ (x,y)- | (kk,x) <- zip [1..] xs, (ll,y) <- zip [1..] ys- , k/=l || kk < ll- ]- | (k,xs) <- gs, (l,ys) <- gs -- @k@ and @l@ are word groups, i.e. language identifiers- , f k l- ]- -- Create comparator function for group selection- mkCmp :: Maybe (String,String) -> (Int -> Int -> Bool)- -- nothing selected, produce all upper triangular groups- mkCmp Nothing = \k l -> k <= l- mkCmp (Just (a,b))- -- we have two strings that actually are numbers- | Just a' <- readMaybe a- , Just b' <- readMaybe b = \k l -> a'==k && b'==l- -- we have two language names- | Just a' <- M.lookup a ls- , Just b' <- M.lookup b ls = \k l -> a'==k && b'==l- -- the user did provide crappy input- mkCmp (Just (a,b)) = \k l -> traceShow ("Unknown languages or ID's: " ++ a ++ " , " ++ b) $ False---- | (write me)--getScores2 :: Mapping -> Lang -> Lang -> Scores-getScores2 ss a b- | Just z <- M.lookup (a:!:b) (lliid ss) = z- | otherwise = trace (printf "Language pair %s %s not found in mapping! Returning empty hashmap\n" (toUtf8String a) (toUtf8String b))- HM.empty---- | (write me)--prettyAli2 :: Double -> [(BTI,BTI)] -> IO ()-prettyAli2 d s = do- print d- forM_ s $ \(x,xs) -> do- putStr $ show x- putStr $ show xs- putStrLn ""- forM_ s $ \(_,y) -> do- putStr $ show y- putStrLn ""--buildAlignmentSimple :: Double -> ([Linguistics.Word.Word],(Double,[[[Text]]])) -> TL.Builder-buildAlignmentSimple k (ws,(s,(xs))) = TL.fromText hdr `mappend` ls `mappend` "\n" where- ids = concat . intersperse " " . map (show . wordID) $ ws- wds = concat . intersperse " WORD " . map (concat . intersperse " " . map toUtf8String . VU.toList . wordWord) $ ws- ns = s / (maximum $ 1 : map ((+k) . fromIntegral . VU.length . wordWord) ws)- hdr = T.pack $ printf "IDS: %s SCORE: %.2f NSCORE: %.2f WORDS: %s\n" ids s ns wds- ls = case xs of [] -> "" ; [xs'] -> buildLines xs'--buildAlignment :: Double -> ([Linguistics.Word.Word],(Double,[[[Text]]])) -> TL.Builder-buildAlignment k (ws,(s,(xss))) = TL.fromText hdr `mappend` ls `mappend` "\n" where- ids = concat . intersperse " " . map (show . wordID) $ ws- wds = concat . intersperse " WORD " . map (concat . intersperse " " . map toUtf8String . VU.toList . wordWord) $ ws- ns = s / (maximum $ 1 : map ((+k) . fromIntegral . VU.length . wordWord) ws)- hdr = T.pack $ printf "IDS: %s SCORE: %.2f NSCORE: %.2f WORDS: %s\n" ids s ns wds- ls = case xss of [] -> "" ; [xs'] -> buildLines $ ["^","^","0.0"] : xs'----printAlignment :: Double -> ([Linguistics.Word.Word],(Double,[[(IMCp,IMCp)]])) -> IO ()---printAlignment k (ws,(s,(xs))) = do--- let ids = concat . intersperse " " . map (show . wordID) $ ws--- let wds = concat . intersperse " WORD " . map (concat . intersperse " " . map toUtf8String . VU.toList . wordWord) $ ws--- --let ns = s / (maximum $ 1 : map ((+k) . fromIntegral . VU.length . wordWord) ws)--- let ns = s / (maximum $ 1 : map ((+k) . fromIntegral . VU.length . wordWord) ws)--- printf "IDS: %s SCORE: %.2f NSCORE: %.2f WORDS: %s\n" ids s ns wds--- let xs1 = map ({- tail . -} reverse . map Prelude.fst) $ xs--- let xs2 = map ({- tail . -} reverse . map Prelude.snd) $ xs--- mapM_ (putStrLn) (alignPretty $ xs1 ++ xs2)--- putStrLn ""-----------------+ Global2Simple{..} -> runGlobal2Simple o ws+ Global2Bigram{..} -> runGlobal2Bigram o ws+-- Infix2Simple{..} -> runInfix2Simple o ws+-- Infix2Bigram{..} -> runInfix2Bigram o ws +-- ** Show manual +runManual :: Config -> IO ()+runManual Manual{} = do+ BS.putStrLn embeddedManual+ exitSuccess -{--import Control.Arrow-import Control.Monad (unless)-import Control.Monad (when)-import Control.Parallel.Strategies-import Data.ByteString (ByteString)-import Data.ByteString.Char8 (unpack)-import Data.Function-import Data.List (intersperse)-import Data.List (tails,genericLength,genericTake,genericDrop,group)-import Data.Ord-import Data.Strict.Tuple-import Data.Vector (fromList)-import Debug.Trace (trace)-import qualified Data.ByteString as BS-import qualified Data.ByteString.Char8 as BC-import qualified Data.ByteString.Lazy.Char8 as BL-import qualified Data.HashTable.IO as H-import qualified Data.List as L-import qualified Data.Map.Strict as M-import qualified Data.Set as S-import qualified Data.Vector as V-import qualified Data.Vector.Unboxed as VU-import System.Console.CmdArgs-import System.IO.Unsafe (unsafePerformIO)-import Text.Printf--import NLP.Alphabet.MultiChar-import NLP.Scoring.SimpleUnigram-import NLP.Scoring.SimpleUnigram.Default-import NLP.Scoring.SimpleUnigram.Import--import Linguistics.TwoWay.Simple-{- we test interning with TwoWay alignments only-import Linguistics.ThreeWay-import Linguistics.FourWay--}-import Linguistics.Bigram-import Linguistics.Common-import Linguistics.Word-import Linguistics.TwoWay.AdvancedBigram--data Config- = TwoWay- { scoreFile :: String- , bigramDef :: Double- , unibiDef :: Double- , gapOpen :: Double- , gapExtend :: Double- , block :: Maybe (Integer,Integer)- , selfAlign :: Bool- }- | TwoWaySimple- { scoreFile :: String- , block :: Maybe (Integer,Integer)- , selfAlign :: Bool- }- {- NLA- | ThreeWay- { scoreFile :: String- , defaultScore :: Double- , gapOpen :: Double- , block :: Maybe (Integer,Integer)- }- | ThreeWaySimple- { scoreFile :: String- , gapOpen :: Double- , block :: Maybe (Integer,Integer)- }- | FourWay- { scoreFile :: String- , defaultScore :: Double- , gapOpen :: Double- , block :: Maybe (Integer,Integer)- }- | FourWaySimple- { scoreFile :: String- , gapOpen :: Double- , block :: Maybe (Integer,Integer)- }- -}- | Info- {- }- deriving (Show,Data,Typeable)--twoway = TwoWay- { scoreFile = "" &= help "the file to read the scores from"- , bigramDef = (-20) &= help "score to use for unknown bigram matches"- , unibiDef = (-5) &= help "score to close a gap if the closing characters are unknown"- , gapOpen = (-5) &= help "cost to open a gap"- , gapExtend = (-1) &= help "cost to extend a gap"- , block = Nothing &= help "when using --block N,k calculate only the k'th block (starting at 1) with length N. For parallelized computations."- , selfAlign = False &= help "align each word with itself as well"- } &= help "Align two words at a time for all ordered word combinations"--twowaySimple = TwoWaySimple- { scoreFile = def &= help ""- }--{- NLA-threeway = ThreeWay- {- }--threewaySimple = ThreeWaySimple- {- }--fourway = FourWay- {- }--fourwaySimple = FourWaySimple- {- }--}--info = Info- {- }+-- ** Global grammars. -config = [twoway,twowaySimple, {- threeway,threewaySimple,fourway,fourwaySimple, -} info]- &= program "WordAlign"- &= summary "WordAlign v.0.0.1"+-- | Simple global alignment. -main = do- o <- cmdArgs $ modes config- ws' <- BL.getContents >>= return . map parseWord . BL.lines- case o of- Info{} -> do- let l :: Integer = genericLength ws'- let c2 = l * (l-1) `div` 2 -- number of alignments- let t2 :: Double = fromIntegral c2 / 5000 / 60 / 60 -- approximate time in hours- let c3 = l * (l-1) * (l-2) `div` 3- let t3 :: Double = fromIntegral c3 / 5000 / 60 / 60 -- TODO fix time constant- let c4 = l * (l-1) * (l-2) * (l-3) `div` 4- let t4 :: Double = fromIntegral c4 / 5000 / 60 / 60 -- TODO fix time constant- printf "%d %.1f %d %.1f %d %.1f\n" c2 t2 c3 t3 c4 t4- TwoWay{..} -> do- let ws = map addWordDelims ws'- let bs = blockWith block $ [ (a,b) | (a:as) <- tails ws, b <- if selfAlign then (a:as) else as ]- let chkLs = S.fromList . map wordLang $ ws- {-- let chkLs = if block==Nothing- then S.fromList . map wordLang $ ws- else S.fromLIst . map wordLang $ ws -- S.fromList . map head . group . map wordLang . concatMap (\(a,b) -> [a,b]) $ bs- -}- ss <- BL.readFile scoreFile >>= return . generateLookups chkLs (-999999)- let ts = map (\(a,b) -> ( [a,b], alignTwo (BigramScores {gapOpen = gapOpen, gapExtend = gapExtend, bigramDef = bigramDef, unigramDef = -999999, biUniDef = unibiDef, scores = getScores2 ss (wordLang a) (wordLang b)})- (wordWord a) (wordWord b)- )- ) bs- mapM_ (printAlignment (-2)) ts- TwoWaySimple{..} -> do- simpleScoring <- if null scoreFile then return $ error "scorefile missing"- else simpleScoreFromFile scoreFile- let ws = ws'- let bs = blockWith block $ [ (a,b) | (a:as) <- tails ws, b <- if selfAlign then (a:as) else as ]- let ts = map (\(a,b) -> ( [a,b], alignTwoSimple simpleScoring (wordWord a) (wordWord b)- )- ) bs- mapM_ (printAlignment 0) ts- {-- ThreeWay{..} -> do- let ws = map addWordDelims ws'- let bs = blockWith block $ [ (a,b,c) | (a:as) <- tails ws, (b:bs) <- tails as, c <- bs ]- let chkLs = S.fromList . map wordLang $ ws- {-- let chkLs = if block==Nothing- then S.fromList . map wordLang $ ws- --else S.fromList . map head . group . map wordLang . concatMap (\(a,b,c) -> [a,b,c]) $ bs- else S.fromLIst . map wordLang $ ws -- S.fromList . map head . group . map wordLang . concatMap (\(a,b) -> [a,b]) $ bs- -}- ss <- BL.readFile scoreFile >>= return . generateLookups chkLs defaultScore- let ts = ss `seq` map (\(a,b,c) -> ( [a,b,c], alignThree defaultScore gapOpen (getScores3 ss (wordLang a) (wordLang b) (wordLang c))- (wordWord a) (wordWord b) (wordWord c)- )- ) bs- mapM_ (printAlignment (-2)) ts- ThreeWaySimple{..} -> do- [vwl,cns] <- readFile vowelConsonantFile >>= return . map VU.fromList . lines- scs <- if null scoreFile then return [3,1,1,0,0,-1] else (readFile scoreFile >>= return . map read . words)- let ws = ws'- let bs = blockWith block $ [ (a,b,c) | (a:as) <- tails ws, (b:bs) <- tails as, c <- bs ]- let ts = map (\(a,b,c) -> ( [a,b,c], alignThreeSimple vwl cns scs gapOpen (wordWord a) (wordWord b) (wordWord c)- )- ) bs- mapM_ (printAlignment 0) ts- FourWay{..} -> do- let ws = map addWordDelims ws'- let bs = blockWith block $ [ (a,b,c,d) | (a:as) <- tails ws, (b:bs) <- tails as, (c:cs) <- tails bs, d <- cs ]- let chkLs = S.fromList . map wordLang $ ws- {-- let chkLs = if block==Nothing- then S.fromList . map wordLang $ ws- --else S.fromList . map head . group . map wordLang . concatMap (\(a,b,c,d) -> [a,b,c,d]) $ bs- else S.fromLIst . map wordLang $ ws -- S.fromList . map head . group . map wordLang . concatMap (\(a,b) -> [a,b]) $ bs- -}- ss <- BL.readFile scoreFile >>= return . generateLookups chkLs defaultScore- let ts = map (\(a,b,c,d) -> ( [a,b,c,d], alignFour defaultScore gapOpen (getScores4 ss (wordLang a) (wordLang b) (wordLang c) (wordLang d))- (wordWord a) (wordWord b) (wordWord c) (wordWord d)- )- ) bs- mapM_ (printAlignment (-2)) ts- FourWaySimple{..} -> do- [vwl,cns] <- readFile vowelConsonantFile >>= return . map VU.fromList . lines- scs <- if null scoreFile then return [3,1,1,0,0,-1] else (readFile scoreFile >>= return . map read . words)- let ws = ws'- let bs = blockWith block $ [ (a,b,c,d) | (a:as) <- tails ws, (b:bs) <- tails as, (c:cs) <- tails bs, d <- cs ]- let ts = map (\(a,b,c,d) -> ( [a,b,c,d], alignFourSimple vwl cns scs gapOpen (wordWord a) (wordWord b) (wordWord c) (wordWord d)- )- ) bs- mapM_ (printAlignment 0) ts- -}+runGlobal2Simple :: Config -> V.Vector Word -> IO ()+runGlobal2Simple = wrapSimple2IO (alignGlobalSimple2) -alignTwo :: BigramScores -> V.Vector InternedMultiChar -> V.Vector InternedMultiChar -> (Double, [[String]])-alignTwo scores x y- = second (map alignPretty) -- . filter (any (\c -> c/= "$" && c/= "^")))- $ runBigram scores 1 x y+---- | Affine infix simple grammar+--+--runInfix2Simple :: Config -> V.Vector Word -> IO ()+--runInfix2Simple = wrapSimple2IO alignInfixSimple2 -alignTwoSimple- :: SimpleScoring- -> V.Vector InternedMultiChar- -> V.Vector InternedMultiChar- -> (Double, [[String]])-alignTwoSimple simpleScoring x y = second (map alignPretty) $ twoWaySimple simpleScoring x y+-- | Wrap simple alignments on two tapes with IO. -{--alignThree :: Double -> Double -> (Scores,Scores,Scores) -> V.Vector ByteString -> V.Vector ByteString -> V.Vector ByteString -> (Double, [[String]])-alignThree sDef sGapOpen scores x y z = second (map (alignPretty . map (filter (\c -> c/= "$" && c/="^")) . tup3List)) $ threeWayBigram sDef sGapOpen scores x y z+wrapSimple2IO+ :: BuildAli t2+ => ( SimpleScoring+ -> FastChars+ -> FastDoubles+ -> Int+ -> Int+ -> VU.Vector BTI+ -> VU.Vector BTI+ -> (Double, [t2])+ )+ -> Config+ -> V.Vector Word+ -> IO ()+wrapSimple2IO f cfg ws = do+ !v <- getVerbosity+ !scoring <- simpleScoreFromFile $ simpleScoreFile cfg+ let align = alignmentWrapper2 (const $ f scoring)+ -- 4 arguments, @const@ takes care of the @()@ group action result+ {-# Inline align #-}+ runAlignment+ (for (runTwowayAlignments groupActionGC align eachGroupStatus (collectSimpleScores scoring) ws)+ (lift . lift . (BB.hPutBuilder stdout))+ )+ ( set aliFilterScore (filterScore cfg) .+ set aliFilterBackt (filterBacktrack cfg) .+ set aliFilterNormalized (filterNormalized cfg) .+ set aliFilterLanguages (lpBlockConstraints $ lpblock cfg) .+ set aliVerbose (v==Loud) $+ def+ ) -alignThreeSimple- :: VU.Vector Char- -> VU.Vector Char- -> [Double]- -> Double- -> V.Vector ByteString- -> V.Vector ByteString- -> V.Vector ByteString- -> (Double, [[String]])-alignThreeSimple v c scores sGapOpen x y z = second (map (alignPretty . tup3List)) $ threeWaySimple v c scores sGapOpen x y z+-- | Helper function to collect all scores. Used in conjunction with fast+-- printing of numbers. -alignFour :: Double -> Double -> (Scores,Scores,Scores,Scores,Scores,Scores) -> V.Vector ByteString -> V.Vector ByteString -> V.Vector ByteString -> V.Vector ByteString -> (Double, [[String]])-alignFour sDef sGapOpen scores w x y z = second (map (alignPretty . map (filter (\c -> c/= "$" && c/="^")) . tup4List)) $ fourWayBigram sDef sGapOpen scores w x y z+collectSimpleScores :: SimpleScoring -> [Double]+collectSimpleScores SimpleScoring{..} =+ simpleScore ^.. traverse+ +++ [gapScore, gapOpen, gapExt, defMatch, defMismatch, preSufOpen, preSufExt] -alignFourSimple- :: VU.Vector Char- -> VU.Vector Char- -> [Double]- -> Double- -> V.Vector ByteString- -> V.Vector ByteString- -> V.Vector ByteString- -> V.Vector ByteString- -> (Double, [[String]])-alignFourSimple v c scores sGapOpen w x y z = second (map (alignPretty . tup4List)) $ fourWaySimple v c scores sGapOpen w x y z--}+-- | Collect bigram scores -blockWith Nothing xs = xs-blockWith (Just (l,k)) xs = genericTake l . genericDrop (l * (k-1)) $ xs+collectBigramScores :: Mapping -> [Double]+collectBigramScores Mapping{..} = lliid ^.. traverse . traverse -getScores2 :: Mapping -> Lang -> Lang -> Scores-getScores2 ss a b- | Just z <- M.lookup (a:!:b) (lliid ss) = z- | otherwise = trace (printf "Language pair %s %s not found in mapping! Returning empty hashmap\n" (toUtf8String a) (toUtf8String b))- (unsafePerformIO H.new)+-- ** Affine grammars -{--getScores3 :: Mapping -> Lang -> Lang -> Lang -> (Scores,Scores,Scores)-getScores3 ss a b c = (getScores2 ss a b, getScores2 ss a c, getScores2 ss b c) -- (lliid ss M.! (a:!:b), lliid ss M.! (a:!:c), lliid ss M.! (b:!:c))+-- | Global bigram grammar. -getScores4 :: Mapping -> Lang -> Lang -> Lang -> Lang -> (Scores,Scores,Scores,Scores,Scores,Scores)-getScores4 ss a b c d = (getScores2 ss a b, getScores2 ss a c, getScores2 ss a d, getScores2 ss b c, getScores2 ss b d, getScores2 ss c d)--- (lliid ss M.! (a:!:b), lliid ss M.! (a:!:c), lliid ss M.! (a:!:d), lliid ss M.! (b:!:c), lliid ss M.! (b:!:d), lliid ss M.! (c:!:d))--}+runGlobal2Bigram :: Config -> V.Vector Word -> IO ()+runGlobal2Bigram = wrapBigram2IO alignGlobalBigram2 -printAlignment :: Double -> ([Word], (Double, [[String]])) -> IO ()-printAlignment k (ws,(s,[])) = do- printf "DEBUG!\nScore: %f\nDEBUG!\n\n" s+---- | Affine infix bigram grammar.+--+--runInfix2Bigram :: Config -> V.Vector Word -> IO ()+--runInfix2Bigram = wrapBigram2IO alignInfixBigram2 -printAlignment k (ws,(s,(x:xs))) = do- let ids = concat . intersperse " " . map (show . wordID) $ ws- let wds = concat . intersperse " WORD " . map (concat . intersperse " " . map toUtf8String . V.toList . wordWord) $ ws- let ns = s / (maximum $ 1 : map ((+k) . fromIntegral . V.length . wordWord) ws)- printf "IDS: %s SCORE: %.2f NSCORE: %.2f WORDS: %s\n" ids s ns wds- mapM_ putStrLn x- putStrLn ""+-- | Wrap two-tape bigram alignments in IO. +wrapBigram2IO f cfg ws = do+ !v <- getVerbosity+ !simpleScoring <- simpleScoreFromFile $ simpleScoreFile cfg+ let chkLs = S.fromList . map wordLang . V.toList $ ws+ !bigramScoring <- BL.readFile (bigramScoreFile cfg) >>= return . mkBigramMap chkLs (-999999)+ let perGroup _ _ = do+ groupActionGC () ()+ xy <- use aliGroupLanguages+ let [x,y] = case xy of+ [z] -> [z,z]+ [x,y] -> [x,y]+ return $ getScores2 True bigramScoring x y+ let align = alignmentWrapper2 (f simpleScoring)+ -- 5 arguments, receives the group action result (the bigram scores)+ {-# Inline align #-}+ runAlignment+ (for (runTwowayAlignments perGroup align eachGroupStatus (collectSimpleScores simpleScoring ++ collectBigramScores bigramScoring) ws)+ --(lift . lift . (TL.hPutStr stdout . TL.toLazyText))+ (lift . lift . (BB.hPutBuilder stdout))+ )+ ( set aliFilterScore (filterScore cfg) .+ set aliFilterBackt (filterBacktrack cfg) .+ set aliFilterNormalized (filterNormalized cfg) .+ set aliFilterLanguages (lpBlockConstraints $ lpblock cfg) .+ set aliVerbose (v==Loud) .+ set aliCustom (mempty :: Scores) $+ (def :: AlignmentConfig ())+ ) -tup2List (a,b) = [a,b]-tup3List (a,b,c) = [a,b,c]-tup4List (a,b,c,d) = [a,b,c,d]+-- | Transform the lpblock input to constraints --}+lpBlockConstraints :: Maybe (String,String) -> [Either Int BTI]+lpBlockConstraints Nothing = []+lpBlockConstraints (Just (x,y)) = nub $ map go [x,y]+ where go s | [(k,"")] <- reads s = Left k+ | otherwise = Right $ btiFromCS s -- [(k,"")] <- reads s = Right k+-- | otherwise = error $ "lpBlockConstraints, can't parse " ++ show s
+ stack.yaml view
@@ -0,0 +1,20 @@+resolver: lts-8.4++packages:+- '.'++extra-deps:+- intern-0.9.1.4+- storable-tuple-0.0.3.2++- ADPfusion-0.5.2.2+- AlignmentAlgorithms-0.1.0.0+- bimaps-0.1.0.2+- DPutils-0.0.1.0+- FormalGrammars-0.3.1.1+- GrammarProducts-0.1.1.3+- LinguisticsTypes-0.0.0.3+- NaturalLanguageAlphabets-0.1.1.0+- OrderedBits-0.0.1.2+- PrimitiveArray-0.8.0.1+
+ tests/PTSP800.bgdef view
@@ -0,0 +1,14 @@+-- a gap in linear scoring global alignments+Gap -4+-- gap opening in affine languages+GapOpen -4+-- gap extension in affine languages+GapExtend -2+-- opening the prefix or suffix in overhang languages+PreSufOpen -1+-- extending the prefix or suffix in overhang languages+PreSufExtend 0+-- score to return for bigram mismatches+Mismatch -3+-- needs to be set, but is not used (need better simple scoring parser)+Match -999999
+ tests/PTSP800.bgms view
@@ -0,0 +1,779 @@+Portuguese Spanish - $ a $ 1.943104048013861+Portuguese Spanish - $ d $ 5.489483488753676+Portuguese Spanish - $ e $ 3.5410255538342787+Portuguese Spanish - $ l $ 4.79633630819373+Portuguese Spanish - $ n $ 5.532043095295166+Portuguese Spanish - $ o $ 2.5276526802250006+Portuguese Spanish - $ r $ 2.63255965875795+Portuguese Spanish - $ s $ 4.167727688157884+Portuguese Spanish - $ z $ 4.496231703139703+Portuguese Spanish - a b a 2.790432042862439+Portuguese Spanish - a c a 2.7466294481684788+Portuguese Spanish - a c o 2.554829133437764+Portuguese Spanish - a d e 2.526920332037582+Portuguese Spanish - a e a 4.787849751605905+Portuguese Spanish - a i a 3.8170708319054594+Portuguese Spanish - a i o 3.2815526392435235+Portuguese Spanish - a l a 2.9697729016071697+Portuguese Spanish - a n a 3.706722774736594+Portuguese Spanish - a p a 2.4842650980599092+Portuguese Spanish - a r a 2.944582622317351+Portuguese Spanish - a r i 2.664391283515977+Portuguese Spanish - a t a 2.143094370369354+Portuguese Spanish - a u a 4.309547326324731+Portuguese Spanish - a u e 3.515595494803787+Portuguese Spanish - b n t 3.9479230191544223+Portuguese Spanish - b s t 4.388790057513658+Portuguese Spanish - c n c 4.779294787197738+Portuguese Spanish - c n t 3.691493489074383+Portuguese Spanish - c s c 4.779294787197738+Portuguese Spanish - d a d 3.9459183314985107+Portuguese Spanish - d n f 6.762449199856891+Portuguese Spanish - d n t 3.8738151092806445+Portuguese Spanish - e d e 2.8943841763584177+Portuguese Spanish - e i e 5.154892661359672+Portuguese Spanish - e i o 3.689838485825715+Portuguese Spanish - e l a 2.143314370335569+Portuguese Spanish - e n a 2.9755743490010973+Portuguese Spanish - e n e 3.491387637730277+Portuguese Spanish - e r a 2.4365778621697456+Portuguese Spanish - e r e 3.279213037509737+Portuguese Spanish - e r i 3.0318552260504052+Portuguese Spanish - e r o 2.583129007839233+Portuguese Spanish - e u e 2.8898075533864818+Portuguese Spanish - i h e 4.654635214809804+Portuguese Spanish - i l e 3.6556528102820836+Portuguese Spanish - i l i 3.701115271993862+Portuguese Spanish - i r a 2.159010344341782+Portuguese Spanish - i r e 2.5496604844243684+Portuguese Spanish - i r i 3.313903538468783+Portuguese Spanish - i t e 2.5871179954736947+Portuguese Spanish - l l l 3.637099265764308+Portuguese Spanish - m n t 3.9479230191544223+Portuguese Spanish - n c h 5.321852619011424+Portuguese Spanish - o i e 3.1438389475762634+Portuguese Spanish - o i o 4.294719783072238+Portuguese Spanish - o l o 3.642013440082184+Portuguese Spanish - o n o 3.986517843831824+Portuguese Spanish - o r a 2.11720014825336+Portuguese Spanish - o r e 2.507850199131615+Portuguese Spanish - o r o 3.388680855952017+Portuguese Spanish - o t e 2.322164124115504+Portuguese Spanish - o u a 4.106319076768513+Portuguese Spanish - o u e 5.075955713146285+Portuguese Spanish - p a n 4.1777780039242565+Portuguese Spanish - p n t 4.114977177421158+Portuguese Spanish - r a r 2.1258581704234274+Portuguese Spanish - r e l 3.4695928626972616+Portuguese Spanish - r e r 2.548867599067953+Portuguese Spanish - r i r 3.3518098864160715+Portuguese Spanish - r l i 3.9850588709676873+Portuguese Spanish - r r r 2.352631501333392+Portuguese Spanish - s a s 4.297456767459031+Portuguese Spanish - s n t 3.7720323725071045+Portuguese Spanish - t c h 4.965177675072692+Portuguese Spanish - t c t 6.7804677456913724+Portuguese Spanish - t e r 2.7462270988739417+Portuguese Spanish - t m b 4.67025448902205+Portuguese Spanish - t n t 3.644973573087399+Portuguese Spanish - u i e 4.306989777941711+Portuguese Spanish - u l o 4.33516062064213+Portuguese Spanish - v n t 4.161497142099734+Portuguese Spanish - x n d 6.000309137727044+Portuguese Spanish ^ - ^ a 3.8142294436785535+Portuguese Spanish ^ - ^ b 2.500020839536907+Portuguese Spanish ^ - ^ c 2.7788532956679624+Portuguese Spanish ^ - ^ d 3.465101806405908+Portuguese Spanish ^ - ^ e 4.011018665661078+Portuguese Spanish ^ - ^ f 3.688245345791627+Portuguese Spanish ^ - ^ h 3.821776726487659+Portuguese Spanish ^ - ^ i 4.291780334858536+Portuguese Spanish ^ - ^ j 3.821776726487659+Portuguese Spanish ^ - ^ l 3.6424357113502817+Portuguese Spanish ^ - ^ m 2.5339223703377285+Portuguese Spanish ^ - ^ o 3.3362688376439507+Portuguese Spanish ^ - ^ p 3.1649971632594847+Portuguese Spanish ^ - ^ r 3.1496829342506873+Portuguese Spanish ^ - ^ s 3.084177806129453+Portuguese Spanish ^ - ^ t 2.374857670489326+Portuguese Spanish ^ - ^ u 4.697245477758131+Portuguese Spanish ^ - ^ v 2.605381355194733+Portuguese Spanish ^ - ^ y 4.920389042988914+Portuguese Spanish ^ a ^ - 4.040490538607975+Portuguese Spanish ^ a ^ a 5.276625117822049+Portuguese Spanish ^ a ^ e 3.2277204154872625+Portuguese Spanish ^ b ^ - 3.245896746778494+Portuguese Spanish ^ b ^ b 5.708172644705329+Portuguese Spanish ^ b ^ g 4.455409790149559+Portuguese Spanish ^ b ^ h 3.356797457171607+Portuguese Spanish ^ b ^ m 3.167555465841174+Portuguese Spanish ^ b ^ p 2.951332415528205+Portuguese Spanish ^ b ^ s 3.3348185859007056+Portuguese Spanish ^ b ^ t 3.1908122855418415+Portuguese Spanish ^ c ^ - 3.356987860593835+Portuguese Spanish ^ c ^ b 2.2920866803003195+Portuguese Spanish ^ c ^ c 4.5310139479460325+Portuguese Spanish ^ c ^ d 2.0044046720978113+Portuguese Spanish ^ c ^ f 2.9206954168218204+Portuguese Spanish ^ c ^ g 2.871905206392913+Portuguese Spanish ^ c ^ h 2.3329086902404+Portuguese Spanish ^ c ^ l 3.193339518455573+Portuguese Spanish ^ c ^ p 2.8029123426158744+Portuguese Spanish ^ c ^ r 3.0217913814718305+Portuguese Spanish ^ c ^ s 2.876243663596335+Portuguese Spanish ^ c ^ t 2.9553808860288973+Portuguese Spanish ^ c ^ v 2.3974472055955367+Portuguese Spanish ^ d ^ - 3.272925426097348+Portuguese Spanish ^ d ^ d 5.76337226664369+Portuguese Spanish ^ d ^ p 2.5728958462137834+Portuguese Spanish ^ d ^ r 3.117197418943278+Portuguese Spanish ^ d ^ t 3.4409845391654232+Portuguese Spanish ^ e ^ - 3.7779997228723903+Portuguese Spanish ^ e ^ a 3.706658039172684+Portuguese Spanish ^ e ^ e 5.545247060281424+Portuguese Spanish ^ f ^ - 2.869419094918419+Portuguese Spanish ^ f ^ b 3.637099366199963+Portuguese Spanish ^ f ^ c 3.262005335687822+Portuguese Spanish ^ f ^ d 3.5317388951802022+Portuguese Spanish ^ f ^ f 5.489483488753676+Portuguese Spanish ^ f ^ h 5.212635671545608+Portuguese Spanish ^ f ^ p 2.6438476559619586+Portuguese Spanish ^ g ^ - 3.043772568629161+Portuguese Spanish ^ g ^ c 2.999641078527668+Portuguese Spanish ^ g ^ g 6.073215881949226+Portuguese Spanish ^ g ^ m 3.5532179562905495+Portuguese Spanish ^ g ^ p 2.9315297962077964+Portuguese Spanish ^ g ^ s 3.720481031382976+Portuguese Spanish ^ h ^ h 5.9695374341596485+Portuguese Spanish ^ i ^ - 4.38533099135165+Portuguese Spanish ^ i ^ i 7.3866034232248+Portuguese Spanish ^ j ^ c 3.88694430146839+Portuguese Spanish ^ j ^ j 7.098921375980398+Portuguese Spanish ^ l ^ - 3.616960349453706+Portuguese Spanish ^ l ^ l 5.863527826640945+Portuguese Spanish ^ m ^ - 2.881253665864991+Portuguese Spanish ^ m ^ c 2.760161079953945+Portuguese Spanish ^ m ^ h 2.9921542744603404+Portuguese Spanish ^ m ^ l 3.0641277249014993+Portuguese Spanish ^ m ^ m 5.575500949206958+Portuguese Spanish ^ m ^ p 2.1812240821713282+Portuguese Spanish ^ m ^ s 3.4401789996458914+Portuguese Spanish ^ n ^ n 6.613413547595008+Portuguese Spanish ^ o ^ a 3.882548684481238+Portuguese Spanish ^ o ^ o 7.018878601087183+Portuguese Spanish ^ p ^ - 2.8941570295722756+Portuguese Spanish ^ p ^ b 3.0427979801023897+Portuguese Spanish ^ p ^ c 2.9359680024026398+Portuguese Spanish ^ p ^ h 2.8604764597015886+Portuguese Spanish ^ p ^ m 2.4480909491225042+Portuguese Spanish ^ p ^ p 4.983403115644131+Portuguese Spanish ^ p ^ r 2.3707042333814745+Portuguese Spanish ^ p ^ t 2.6944913502803827+Portuguese Spanish ^ q ^ c 3.8179514076295833+Portuguese Spanish ^ q ^ q 7.296991342520439+Portuguese Spanish ^ r ^ - 3.2499047598589113+Portuguese Spanish ^ r ^ c 2.666776804836229+Portuguese Spanish ^ r ^ r 5.627873565382054+Portuguese Spanish ^ s ^ - 3.494358156573803+Portuguese Spanish ^ s ^ p 2.91211167844761+Portuguese Spanish ^ s ^ s 5.536307496394534+Portuguese Spanish ^ s ^ z 5.397018259112255+Portuguese Spanish ^ t ^ - 3.0493905660254996+Portuguese Spanish ^ t ^ c 2.263321776394188+Portuguese Spanish ^ t ^ t 5.810569858815712+Portuguese Spanish ^ u ^ u 8.07975067940688+Portuguese Spanish ^ v ^ l 4.077582314083823+Portuguese Spanish ^ v ^ v 6.2392010252133066+Portuguese Spanish a $ - $ 3.202426965288919+Portuguese Spanish a $ a $ 4.316306690833602+Portuguese Spanish a $ e $ 2.5512520318981355+Portuguese Spanish a $ o $ 2.5744189674167255+Portuguese Spanish a $ r $ 0.5107258883009363+Portuguese Spanish a - a d 3.1837782162362354+Portuguese Spanish a - a i 5.322910308227259+Portuguese Spanish a - a l 4.061117918390248+Portuguese Spanish a - a n 3.644479496391005+Portuguese Spanish a - a r 2.1184231978808135+Portuguese Spanish a - a s 3.2316342832007106+Portuguese Spanish a - e n 3.560396310966622+Portuguese Spanish a - e r 2.4360721149366333+Portuguese Spanish a - e s 2.266553297706621+Portuguese Spanish a - i n 2.902542151661628+Portuguese Spanish a - o n 2.96090167474603+Portuguese Spanish a - o s 3.1336539224735884+Portuguese Spanish a - u n 3.749839982139686+Portuguese Spanish a b a b 6.255902910833949+Portuguese Spanish a c - c 4.1167177562612975+Portuguese Spanish a c a - 2.731948917553975+Portuguese Spanish a c a c 5.394173315251195+Portuguese Spanish a c a s 3.387638530840328+Portuguese Spanish a c a z 5.373554020827596+Portuguese Spanish a d a - 3.3119736009480865+Portuguese Spanish a d a d 5.28740931938796+Portuguese Spanish a g a g 6.044228332669217+Portuguese Spanish a i a - 4.807070773903504+Portuguese Spanish a i a y 6.745376433307276+Portuguese Spanish a l a - 3.681991909180767+Portuguese Spanish a l a l 5.367835785538958+Portuguese Spanish a m - m 4.826473360765783+Portuguese Spanish a m a - 3.309913794173627+Portuguese Spanish a m a m 5.804967751699694+Portuguese Spanish a n a - 3.4586187560443977+Portuguese Spanish a n a d 2.482332232861595+Portuguese Spanish a n a m 3.5100843107586286+Portuguese Spanish a n a n 4.734792983352166+Portuguese Spanish a n a ɲ 5.075719644787911+Portuguese Spanish a n e n 2.804883134687458+Portuguese Spanish a o - o 5.440693236358417+Portuguese Spanish a o a - 3.7153788790634317+Portuguese Spanish a o i o 4.877004263881585+Portuguese Spanish a p a p 6.603844079248784+Portuguese Spanish a r - r 2.2005117821129283+Portuguese Spanish a r a - 1.9803323576272431+Portuguese Spanish a r a m 1.6392361415949157+Portuguese Spanish a r a r 4.158616197754217+Portuguese Spanish a r a s 1.624421098639547+Portuguese Spanish a r e - 1.5489135406049799+Portuguese Spanish a r e m 2.1062591468473304+Portuguese Spanish a r e r 2.6206183560896172+Portuguese Spanish a r i r 2.953557073551599+Portuguese Spanish a r o - 2.3058722449627234+Portuguese Spanish a s a - 4.067599495871144+Portuguese Spanish a s a s 5.351897834466149+Portuguese Spanish a s e s 3.0005225321066296+Portuguese Spanish a s o - 3.1021551995838146+Portuguese Spanish a s o s 3.644479496391005+Portuguese Spanish a t - t 3.9688768116887814+Portuguese Spanish a t a d 3.167248949843676+Portuguese Spanish a t a m 3.6353851686686443+Portuguese Spanish a t a t 5.865996782159193+Portuguese Spanish a t e t 4.3435702701398595+Portuguese Spanish a v - v 5.57933981590621+Portuguese Spanish a v a b 4.70387107855231+Portuguese Spanish a v a v 6.247169229837719+Portuguese Spanish a z a c 5.663836840608123+Portuguese Spanish b a b a 5.634027790498081+Portuguese Spanish b a g o 4.422619934832684+Portuguese Spanish b a t a 2.9717871075194253+Portuguese Spanish b e b e 6.438764417963484+Portuguese Spanish b e n e 4.706724149504043+Portuguese Spanish b e p e 4.38827045284872+Portuguese Spanish b i b i 6.726400466654192+Portuguese Spanish b o b o 6.160651734586647+Portuguese Spanish b r b r 5.97742479908034+Portuguese Spanish b u b u 6.91214553005498+Portuguese Spanish c - c a 3.3823052017519393+Portuguese Spanish c - c h 4.339603908760342+Portuguese Spanish c - c i 4.892897308999788+Portuguese Spanish c - c u 5.15053415814341+Portuguese Spanish c - t r 3.8103445995032232+Portuguese Spanish c a - a 2.1894117300965488+Portuguese Spanish c a c - 2.9952455905067765+Portuguese Spanish c a c a 4.732699678323564+Portuguese Spanish c a c e 2.576793194731945+Portuguese Spanish c a c i 2.889884947630467+Portuguese Spanish c a c o 2.04048843187774+Portuguese Spanish c a d a 2.737416385790816+Portuguese Spanish c a g - 4.506703004779393+Portuguese Spanish c a h a 2.8972651554868185+Portuguese Spanish c a l a 1.8901184739549577+Portuguese Spanish c a n e 3.0150481092930743+Portuguese Spanish c a r a 1.8060877248696081+Portuguese Spanish c a t a 2.075040918152566+Portuguese Spanish c a t o 2.2612763478417586+Portuguese Spanish c a z a 3.9566566959230216+Portuguese Spanish c e c e 5.596739704414973+Portuguese Spanish c e t e 3.1439286663736823+Portuguese Spanish c h c - 4.212463573875803+Portuguese Spanish c h c h 5.07623631134537+Portuguese Spanish c h l l 2.826169212755913+Portuguese Spanish c i c i 5.356538193935521+Portuguese Spanish c i p - 5.277400900474729+Portuguese Spanish c i s - 4.526189910915324+Portuguese Spanish c o - a 2.9074270162052898+Portuguese Spanish c o - e 3.065233985184673+Portuguese Spanish c o c - 3.4901172202103945+Portuguese Spanish c o c o 4.917987947638173+Portuguese Spanish c o c u 3.2552776154461753+Portuguese Spanish c o n o 3.1124868467676263+Portuguese Spanish c o p i 3.7431136949825117+Portuguese Spanish c o t e 2.14128040708316+Portuguese Spanish c o z o 4.588729547657822+Portuguese Spanish c r c r 7.168233281651585+Portuguese Spanish c u c u 6.0362020593333785+Portuguese Spanish d - d e 4.487015052761163+Portuguese Spanish d - d i 5.440693302527787+Portuguese Spanish d a - r 3.598004414676304+Portuguese Spanish d a d a 5.403141979663575+Portuguese Spanish d a l a 2.6099339831025627+Portuguese Spanish d a m a 3.061919113221314+Portuguese Spanish d a r a 2.343581602315004+Portuguese Spanish d e - l 3.2257691256418455+Portuguese Spanish d e d - 4.817815216669787+Portuguese Spanish d e d e 5.123003758872241+Portuguese Spanish d e t - 3.7424599220320482+Portuguese Spanish d i d e 4.05801945337585+Portuguese Spanish d i d i 5.838376268126105+Portuguese Spanish d o - o 3.397619350665415+Portuguese Spanish d o d o 5.4044901590312255+Portuguese Spanish d o s o 3.5799409287280963+Portuguese Spanish d o t e 2.393623062299472+Portuguese Spanish d o t o 3.2316342832007106+Portuguese Spanish d r d r 7.2614403206026+Portuguese Spanish d u d - 6.098749144370929+Portuguese Spanish d u d u 7.029928511695554+Portuguese Spanish e $ - $ 3.5262247570562986+Portuguese Spanish e $ a $ 2.792255516518972+Portuguese Spanish e $ e $ 4.969995434782811+Portuguese Spanish e $ l $ 3.4482631488658098+Portuguese Spanish e $ o $ 2.8843276850815665+Portuguese Spanish e $ r $ 0.8249541702953255+Portuguese Spanish e $ y $ 5.1222396127651075+Portuguese Spanish e - a d 2.39532094279311+Portuguese Spanish e - a l 2.8026569576998+Portuguese Spanish e - a n 2.568340149979147+Portuguese Spanish e - a r 1.6018995759305292+Portuguese Spanish e - e a 3.84927397919427+Portuguese Spanish e - e c 4.583243151025082+Portuguese Spanish e - e d 3.7359452516452167+Portuguese Spanish e - e l 4.238402563811167+Portuguese Spanish e - e n 3.582869275261196+Portuguese Spanish e - e r 2.7462270988739417+Portuguese Spanish e - e s 2.2890262743953214+Portuguese Spanish e - o n 2.6469023282093977+Portuguese Spanish e - o s 3.379270333701592+Portuguese Spanish e a e a 6.429490766938411+Portuguese Spanish e b e b 6.898250719729724+Portuguese Spanish e c e c 5.462760529945502+Portuguese Spanish e c e j 6.386431319621547+Portuguese Spanish e d e d 6.133840483087733+Portuguese Spanish e e - e 1.2821451124288163+Portuguese Spanish e f e f 8.39560360361798+Portuguese Spanish e g e g 6.650817876196546+Portuguese Spanish e i - a 3.3420913299675066+Portuguese Spanish e i - i 4.598510562894484+Portuguese Spanish e i - o 3.271639610429436+Portuguese Spanish e i e - 5.049668873949677+Portuguese Spanish e j e - 5.171029781267057+Portuguese Spanish e l a - 3.12451056439272+Portuguese Spanish e l a l 3.7342150810805204+Portuguese Spanish e l e - 3.522371155679676+Portuguese Spanish e l e l 5.332479710325341+Portuguese Spanish e l i e 3.9461853492054506+Portuguese Spanish e m a - 3.276526777065966+Portuguese Spanish e m a r 1.72039303386465+Portuguese Spanish e m e - 3.60539446982876+Portuguese Spanish e m e m 5.618027299614127+Portuguese Spanish e m e n 3.3959809979363245+Portuguese Spanish e m o - 3.2919116904077423+Portuguese Spanish e n a - 2.6937276934470753+Portuguese Spanish e n a n 2.8178009374489874+Portuguese Spanish e n a r 1.137593983086327+Portuguese Spanish e n e - 3.0225953247750894+Portuguese Spanish e n e n 4.583887959681108+Portuguese Spanish e n e r 1.742924934376688+Portuguese Spanish e n i - 3.5258738078934675+Portuguese Spanish e n o - 2.932256115155333+Portuguese Spanish e p e p 7.674285533487649+Portuguese Spanish e r - r 2.8219081037142284+Portuguese Spanish e r a - 2.5833795965060204+Portuguese Spanish e r a r 2.6366837486684367+Portuguese Spanish e r e - 2.7580966094415564+Portuguese Spanish e r e r 4.405165645345216+Portuguese Spanish e r i r 3.4695929814476445+Portuguese Spanish e s a - 2.5467454778377148+Portuguese Spanish e s a s 3.356585877444995+Portuguese Spanish e s e - 3.126927632509029+Portuguese Spanish e s e n 2.196196093678901+Portuguese Spanish e s e r 1.5959427968798376+Portuguese Spanish e s e s 5.025447033413921+Portuguese Spanish e s e x 4.64935417403653+Portuguese Spanish e s i - 2.973426470148597+Portuguese Spanish e s o - 2.9675955429943945+Portuguese Spanish e t e t 6.167363251475645+Portuguese Spanish e v e b 6.685157538158934+Portuguese Spanish e v e v 7.185932853799537+Portuguese Spanish e x e x 7.232452793812299+Portuguese Spanish e z e c 5.4696808942270865+Portuguese Spanish e z e z 6.492785580812542+Portuguese Spanish e z o - 4.724015686000631+Portuguese Spanish f - d e 4.620546509782657+Portuguese Spanish f a f a 6.885828150436738+Portuguese Spanish f a h a 5.6128625459247194+Portuguese Spanish f e f e 6.18388143309747+Portuguese Spanish f e h e 5.544438371860675+Portuguese Spanish f i f i 6.452294220221665+Portuguese Spanish f i h i 5.909337352319209+Portuguese Spanish f o h o 5.984804924014375+Portuguese Spanish f r f r 7.3866034988469345+Portuguese Spanish f u h u 6.885828150436738+Portuguese Spanish g a - a 2.576623467593107+Portuguese Spanish g a c a 2.9604270796257675+Portuguese Spanish g a g a 5.529280061011166+Portuguese Spanish g a l a 2.4596517497255985+Portuguese Spanish g a t a 2.6445741443207957+Portuguese Spanish g i c i 5.652002386742228+Portuguese Spanish g o g o 6.4096880884397445+Portuguese Spanish g r g r 6.765962947293154+Portuguese Spanish g u g u 6.252899941682571+Portuguese Spanish h $ - $ 5.855127097161055+Portuguese Spanish h a - a 3.312532527725962+Portuguese Spanish h a c a 2.3745802990706277+Portuguese Spanish h a d a 3.1265679086592906+Portuguese Spanish h a h a 3.915025238597466+Portuguese Spanish h a j a 4.751273303366211+Portuguese Spanish h a l a 3.5320330167674547+Portuguese Spanish h a n a 3.51699514447393+Portuguese Spanish h a r a 2.706064970727356+Portuguese Spanish h a t a 2.687335998932494+Portuguese Spanish h a ɲ a 4.949098989224619+Portuguese Spanish h o - o 4.321461836249953+Portuguese Spanish h o d o 3.453961275782504+Portuguese Spanish h o h o 5.217549800840603+Portuguese Spanish h o j o 4.657933971549326+Portuguese Spanish h o l o 3.81063619559699+Portuguese Spanish h o m o 4.230490053219377+Portuguese Spanish h o r o 3.4619933659398128+Portuguese Spanish i - a c 4.090766531225361+Portuguese Spanish i - a n 3.0566928015797408+Portuguese Spanish i - i l 4.804967142251865+Portuguese Spanish i - i n 4.041976418100871+Portuguese Spanish i - i o 4.248395509990624+Portuguese Spanish i - o n 3.3175765791856966+Portuguese Spanish i a - a 4.197757419790399+Portuguese Spanish i a i - 3.976074724251175+Portuguese Spanish i a i a 5.993751663548807+Portuguese Spanish i a y a 6.0266959433149925+Portuguese Spanish i b i b 7.137142578717644+Portuguese Spanish i c i c 6.083830166008745+Portuguese Spanish i d a d 3.7170767422122295+Portuguese Spanish i d i d 6.185760492805324+Portuguese Spanish i d i v 5.031267834559381+Portuguese Spanish i f i f 7.925599974372244+Portuguese Spanish i g i g 6.834955842224469+Portuguese Spanish i l i - 4.763748197570564+Portuguese Spanish i l i l 5.426655395853003+Portuguese Spanish i m i m 6.498483710633993+Portuguese Spanish i m i n 4.252468899627535+Portuguese Spanish i n e n 2.987535317969683+Portuguese Spanish i n e r 2.3872818881795452+Portuguese Spanish i n i - 4.170230717644955+Portuguese Spanish i n i n 5.536484399936025+Portuguese Spanish i n o n 3.163426003043482+Portuguese Spanish i o - a 3.698766273906239+Portuguese Spanish i o - e 4.038894794159994+Portuguese Spanish i o - o 4.524402620280658+Portuguese Spanish i o i o 5.693632463630644+Portuguese Spanish i o y o 6.7861656911838795+Portuguese Spanish i r - r 4.084935673228204+Portuguese Spanish i r - s 3.0691153404099825+Portuguese Spanish i r a r 2.277028202069728+Portuguese Spanish i r e r 3.2188314971035132+Portuguese Spanish i r i r 4.984584482654758+Portuguese Spanish i s e - 3.816484161786926+Portuguese Spanish i s e s 4.143306070334523+Portuguese Spanish i s i s 6.388732802194357+Portuguese Spanish i t - h 5.910696981400548+Portuguese Spanish i t - t 4.593395508309951+Portuguese Spanish i t i t 5.6511858196128255+Portuguese Spanish i v i - 4.540604646256354+Portuguese Spanish i v i v 6.546773939237813+Portuguese Spanish i x - j 6.951285400090131+Portuguese Spanish i z i z 7.52970424669957+Portuguese Spanish j a - a 4.700214758234032+Portuguese Spanish j o j o 6.575673244819539+Portuguese Spanish j u j u 8.213282024767569+Portuguese Spanish l $ - $ 4.58216137762912+Portuguese Spanish l $ a $ 2.8185728081380983+Portuguese Spanish l $ l $ 6.519102905934834+Portuguese Spanish l - l o 5.286542688869896+Portuguese Spanish l a - a 3.4110841699780585+Portuguese Spanish l a l a 4.834557449330444+Portuguese Spanish l a n - 3.5817944808350766+Portuguese Spanish l a r a 2.2168298348266973+Portuguese Spanish l d l d 8.030960454739741+Portuguese Spanish l e l - 6.239200906108439+Portuguese Spanish l e l e 5.792913863032455+Portuguese Spanish l f l f 8.262072210993457+Portuguese Spanish l h - j 5.757362870260645+Portuguese Spanish l h - m 4.826473360765783+Portuguese Spanish l h - n 4.917445088458283+Portuguese Spanish l h a n 3.0566928015797408+Portuguese Spanish l h l l 2.420704019905171+Portuguese Spanish l i l i 6.110310047120522+Portuguese Spanish l m l m 8.010757737699375+Portuguese Spanish l o l o 5.756546356714426+Portuguese Spanish l p l p 7.663235639324093+Portuguese Spanish l t l t 6.994041824660113+Portuguese Spanish l u l u 6.844140744899521+Portuguese Spanish l v l v 7.124239105821811+Portuguese Spanish m $ - $ 5.056619332935875+Portuguese Spanish m $ n $ 5.9375081832724375+Portuguese Spanish m - m a 3.559321719067589+Portuguese Spanish m - m b 5.340928872218928+Portuguese Spanish m - m i 5.282088440876246+Portuguese Spanish m - m o 4.315647736999791+Portuguese Spanish m - n a 4.16274016200759+Portuguese Spanish m - n o 4.017154803831101+Portuguese Spanish m - s u 5.436238998851443+Portuguese Spanish m a m - 4.253285581954871+Portuguese Spanish m a m a 5.3800687013892805+Portuguese Spanish m b m b 6.3305507915129375+Portuguese Spanish m e m e 5.7379448430107+Portuguese Spanish m e m i 4.667503480463672+Portuguese Spanish m i m e 4.874297815003297+Portuguese Spanish m i m i 6.037448683101039+Portuguese Spanish m o m o 6.0836907527158655+Portuguese Spanish m p - p 5.601812697359512+Portuguese Spanish m p m p 6.143410016658045+Portuguese Spanish m u m u 6.815153263505006+Portuguese Spanish n - l o 3.7706307713789418+Portuguese Spanish n - n i 5.151227147038849+Portuguese Spanish n - n t 3.497337433868369+Portuguese Spanish n - n u 5.939684459194009+Portuguese Spanish n - q u 4.8002502136690115+Portuguese Spanish n - t r 3.728666581068029+Portuguese Spanish n a n a 5.31851073696894+Portuguese Spanish n a p a 3.508266489335212+Portuguese Spanish n a t a 3.167095817302744+Portuguese Spanish n a ɲ a 4.841072150438769+Portuguese Spanish n c n c 5.991235791697411+Portuguese Spanish n d n d 6.213883165311512+Portuguese Spanish n e n e 5.839686018443403+Portuguese Spanish n g n g 7.16345998534379+Portuguese Spanish n h - m 4.704583555612735+Portuguese Spanish n h - ɲ 6.074293919241768+Portuguese Spanish n h n - 4.943191406291814+Portuguese Spanish n h n d 3.9688768116887814+Portuguese Spanish n i n i 6.451963559334396+Portuguese Spanish n o n o 5.639015194716441+Portuguese Spanish n o ɲ o 6.113637752813494+Portuguese Spanish n s n s 7.306560810078931+Portuguese Spanish n t - d 4.463217991995234+Portuguese Spanish n t - l 3.3646056153967043+Portuguese Spanish n t n - 4.075452382311885+Portuguese Spanish n t n t 5.208256021753779+Portuguese Spanish n u n u 7.4511419396359875+Portuguese Spanish n v n v 8.367432714047593+Portuguese Spanish o $ - $ 3.088808014166296+Portuguese Spanish o $ a $ 2.688330364568234+Portuguese Spanish o $ e $ 2.780039964274269+Portuguese Spanish o $ l $ 2.540842881025595+Portuguese Spanish o $ o $ 4.1094586736902805+Portuguese Spanish o $ r $ -0.08246617161463834+Portuguese Spanish o - a l 3.2330513797648783+Portuguese Spanish o - a r 1.2903566873193713+Portuguese Spanish o - e l 3.2627000951543077+Portuguese Spanish o - e n 2.6782626135614627+Portuguese Spanish o - e r 2.118831274736059+Portuguese Spanish o - i l 3.689218618545523+Portuguese Spanish o - o l 4.521161028337739+Portuguese Spanish o - o n 4.652833107799401+Portuguese Spanish o - o r 3.1326469729913287+Portuguese Spanish o / o / 7.897429062115218+Portuguese Spanish o b o b 6.984229502784873+Portuguese Spanish o c o c 5.879444331552816+Portuguese Spanish o d o d 7.3866034232248+Portuguese Spanish o g e g 7.060602514829183+Portuguese Spanish o l e l 4.072417384285992+Portuguese Spanish o l o - 3.7124147065773236+Portuguese Spanish o l o l 5.641033367187612+Portuguese Spanish o m a - 3.2915646877531515+Portuguese Spanish o m a m 4.185213023137442+Portuguese Spanish o m o m 6.036027241556388+Portuguese Spanish o n a - 2.916871193164931+Portuguese Spanish o n e - 3.091588152936858+Portuguese Spanish o n e n 2.748643409687334+Portuguese Spanish o n i n 3.8720773713800685+Portuguese Spanish o n o n 4.896877463985417+Portuguese Spanish o n u n 4.496231618064798+Portuguese Spanish o r a r 1.9177524833100419+Portuguese Spanish o r e r 3.5346843479712713+Portuguese Spanish o r o - 3.6070542116193063+Portuguese Spanish o r o r 5.218657771817695+Portuguese Spanish o s a - 3.255846638688701+Portuguese Spanish o s e - 3.207420076774933+Portuguese Spanish o s e s 3.534242075337154+Portuguese Spanish o s o - 4.146700251103954+Portuguese Spanish o s o s 5.499954883805076+Portuguese Spanish o t o t 6.767564274235968+Portuguese Spanish o u o - 5.640306431423752+Portuguese Spanish o v e v 6.087320565131427+Portuguese Spanish o v o b 5.9154702778771435+Portuguese Spanish o v o v 7.250471322946893+Portuguese Spanish o z o c 6.836557066139624+Portuguese Spanish p - p i 5.933169758216973+Portuguese Spanish p - p u 6.042868755296762+Portuguese Spanish p a - a 2.6871653386204595+Portuguese Spanish p a p a 5.320910251003666+Portuguese Spanish p e p a 3.671169946831412+Portuguese Spanish p e p e 5.601812697359512+Portuguese Spanish p e r a 2.4732594911836454+Portuguese Spanish p i p i 6.337262172292865+Portuguese Spanish p l p l 6.646936273843664+Portuguese Spanish p o p a 3.7512126769060368+Portuguese Spanish p o p o 6.131337408093672+Portuguese Spanish p r c - 3.958683100806634+Portuguese Spanish p r p - 4.755014426225068+Portuguese Spanish p r p l 5.02260982120697+Portuguese Spanish p r p r 6.05222925274733+Portuguese Spanish p r t r 3.600049240983152+Portuguese Spanish p u p u 6.655973185908298+Portuguese Spanish q u q u 6.342479398900264+Portuguese Spanish r $ - $ 2.344730369305473+Portuguese Spanish r $ a $ 0.5217183402462306+Portuguese Spanish r $ o $ -0.2642789763026767+Portuguese Spanish r $ r $ 4.411533603835967+Portuguese Spanish r - r c 4.860874750558242+Portuguese Spanish r - r d 4.743091751826731+Portuguese Spanish r - r e 3.819420939733192+Portuguese Spanish r - r i 3.7081952898530197+Portuguese Spanish r - r o 3.6241122284400595+Portuguese Spanish r - r s 5.959487117999411+Portuguese Spanish r a - a 2.3951636900853814+Portuguese Spanish r a - e 2.958435771456216+Portuguese Spanish r a - o 2.766544702728284+Portuguese Spanish r a b a 2.7049868259101486+Portuguese Spanish r a d a 2.3145596389512124+Portuguese Spanish r a g a 2.426676854261747+Portuguese Spanish r a g e 3.8651570713510406+Portuguese Spanish r a i e 2.4506917670163184+Portuguese Spanish r a l a 2.789017515937862+Portuguese Spanish r a r - 3.8186369833762295+Portuguese Spanish r a r a 4.3254745289784395+Portuguese Spanish r a r e 1.8147030185716695+Portuguese Spanish r a s - 2.8406527605476986+Portuguese Spanish r a t a 2.05764924896221+Portuguese Spanish r a t e 2.0344820516637228+Portuguese Spanish r a u e 2.6192199879714257+Portuguese Spanish r c r c 7.291293341729249+Portuguese Spanish r d - g 5.419491050449179+Portuguese Spanish r d r d 6.411370275564772+Portuguese Spanish r e - a 2.5019619601830727+Portuguese Spanish r e - e 3.065233985184673+Portuguese Spanish r e - o 2.808804482241952+Portuguese Spanish r e c o 2.3530386250517044+Portuguese Spanish r e l a 2.2026686690908632+Portuguese Spanish r e l e 3.4329587995270368+Portuguese Spanish r e m e 3.1452766860148+Portuguese Spanish r e r - 3.2322881314385223+Portuguese Spanish r e r e 4.7605796862055065+Portuguese Spanish r e r i 2.868065911149178+Portuguese Spanish r e r o 2.419339666207681+Portuguese Spanish r g r g 7.337813274179795+Portuguese Spanish r i - a 2.648944081958133+Portuguese Spanish r i l a 2.531972386664027+Portuguese Spanish r i r e 2.6562701801453343+Portuguese Spanish r i r i 5.342325709926795+Portuguese Spanish r m r m 6.64466609361985+Portuguese Spanish r n r n 6.78876641559444+Portuguese Spanish r o - e 2.8668550810992097+Portuguese Spanish r o - i 3.5600022616591547+Portuguese Spanish r o - o 2.792747025495283+Portuguese Spanish r o h a 3.4169013679483595+Portuguese Spanish r o i o 3.50999182161278+Portuguese Spanish r o r a 1.9202589256676692+Portuguese Spanish r o r o 5.04233967736237+Portuguese Spanish r p r p 7.80781688357472+Portuguese Spanish r r - g 2.6198105004517775+Portuguese Spanish r r - m 2.742412902671082+Portuguese Spanish r r - r 1.646703388617968+Portuguese Spanish r r l l -0.3565036548604597+Portuguese Spanish r r r r 1.7370425361645545+Portuguese Spanish r s r s 7.568925030433512+Portuguese Spanish r t r - 4.405561887137269+Portuguese Spanish r t r t 6.16437721490804+Portuguese Spanish r u r - 4.988708207143031+Portuguese Spanish r u r u 6.565622950214474+Portuguese Spanish r v r v 7.211250564735424+Portuguese Spanish s $ - $ 4.340999320812232+Portuguese Spanish s $ s $ 6.404172850388687+Portuguese Spanish s - s u 5.875605661589279+Portuguese Spanish s a d a 3.443808789851393+Portuguese Spanish s a r a 2.330158579508897+Portuguese Spanish s a s a 5.811851142467204+Portuguese Spanish s a z a 3.9699018746707107+Portuguese Spanish s c - c 4.740872025366812+Portuguese Spanish s c s c 6.578601570021722+Portuguese Spanish s e - a 3.447451919819699+Portuguese Spanish s e s e 5.878745343997715+Portuguese Spanish s e t e 3.269091886722216+Portuguese Spanish s i s i 6.557324142386948+Portuguese Spanish s o d o 3.4956338454076885+Portuguese Spanish s o s o 5.79821900782049+Portuguese Spanish s o t e 3.63107177002362+Portuguese Spanish s p s p 6.37678673565212+Portuguese Spanish s q s q 8.07975067940688+Portuguese Spanish s s - n 2.6241624776878276+Portuguese Spanish s s - s 3.0784177708177185+Portuguese Spanish s t - t 3.648933800215048+Portuguese Spanish s t n t 2.9983463085144377+Portuguese Spanish s t s t 5.543347398487936+Portuguese Spanish s t x t 5.217549718128893+Portuguese Spanish s u s u 7.163459890816124+Portuguese Spanish t - b r 4.849824965574147+Portuguese Spanish t - t e 3.9258713747397076+Portuguese Spanish t - t i 4.5583041719563395+Portuguese Spanish t - t r 4.664759977975581+Portuguese Spanish t a - a 2.425800564368638+Portuguese Spanish t a - e 2.5836075135508203+Portuguese Spanish t a c a 1.9986740819467708+Portuguese Spanish t a h a 3.826801103033534+Portuguese Spanish t a p a 3.122604146289913+Portuguese Spanish t a r a 2.330158579508897+Portuguese Spanish t a t - 3.9735716315170753+Portuguese Spanish t a t a 4.860874868717829+Portuguese Spanish t a t e 2.065119106582069+Portuguese Spanish t a t o 3.1908122855418415+Portuguese Spanish t e p e 3.2614871843166076+Portuguese Spanish t e r a 1.9655154584990882+Portuguese Spanish t e t e 5.014661804092411+Portuguese Spanish t e t o 3.049312741472103+Portuguese Spanish t i t i 5.553237989492562+Portuguese Spanish t o - a 2.415850237731059+Portuguese Spanish t o d o 3.1361260950027865+Portuguese Spanish t o h o 4.74556397428656+Portuguese Spanish t o r a 2.255669705747253+Portuguese Spanish t o t a 2.7069444722114238+Portuguese Spanish t o t o 5.2445551261982635+Portuguese Spanish t r d - 4.6959255247037825+Portuguese Spanish t r t r 5.5973111806370985+Portuguese Spanish t u - r 4.200767516831774+Portuguese Spanish t u c - 4.723289197641795+Portuguese Spanish t u t u 6.632831639753954+Portuguese Spanish u $ o $ 3.972787567475468+Portuguese Spanish u - o n 4.197499736524366+Portuguese Spanish u - u c 5.623014881378198+Portuguese Spanish u a u a 6.831760960360593+Portuguese Spanish u c - c 5.934794483746449+Portuguese Spanish u c u c 6.652634298559357+Portuguese Spanish u d u d 7.055246276467194+Portuguese Spanish u e - a 4.25838196741246+Portuguese Spanish u e - e 4.598510562894484+Portuguese Spanish u e u e 5.581050652345405+Portuguese Spanish u g u g 7.16345998534379+Portuguese Spanish u i - e 4.226947121667589+Portuguese Spanish u i u i 6.6712684004030836+Portuguese Spanish u l u l 6.370229306425741+Portuguese Spanish u m u m 6.988107058479402+Portuguese Spanish u n u n 6.62696654771458+Portuguese Spanish u r u r 6.167126532010815+Portuguese Spanish u s u s 6.74474960133122+Portuguese Spanish u t u t 7.232452793812299+Portuguese Spanish v - v i 5.871476284789608+Portuguese Spanish v a b a 4.298541650388705+Portuguese Spanish v a c a 3.0019760917288294+Portuguese Spanish v a l a 2.906665855060205+Portuguese Spanish v a t a 3.091588239587222+Portuguese Spanish v a v a 5.864176875176639+Portuguese Spanish v e v e 5.852427916124317+Portuguese Spanish v e v i 4.070147321121756+Portuguese Spanish v i v i 6.490515400558718+Portuguese Spanish v o t e 3.362182264641421+Portuguese Spanish v o v o 6.283101154266904+Portuguese Spanish v r b r 6.276941278204202+Portuguese Spanish x o j o 6.78876641559444+Portuguese Spanish z $ z $ 7.451142010531739+Portuguese Spanish z a z a 6.207948398524851+Portuguese Spanish z e c e 5.795669016700312+Portuguese Spanish z i c i 6.1909989410405935
+ tests/PTSP800.ugdef view
@@ -0,0 +1,22 @@+Set Conso b c d f g h j k m n p q s t v w x y z+Set Liquid l r+Set Vowel a e i o u a\' e\' i\' o\' u\' ə ɐ æ œ æh ɔh ɒ ɔ+Eq Conso 4+Eq Liquid 3+Eq Vowel 2+InSet Conso Conso 0+InSet Conso Liquid -1+InSet Conso Vowel -999999+InSet Liquid Conso -1+InSet Liquid Liquid -1+InSet Liquid Vowel -1+InSet Vowel Conso -999999+InSet Vowel Liquid -1+InSet Vowel Vowel 0+Gap -4+GapOpen -4+GapExtend -2+PreSufOpen -1+PreSufExtend 0+Match 1+Mismatch -999999
+ tests/PTpoSPpampa.words view
@@ -0,0 +1,2 @@+49205 1.213 Portuguese 2 p o +57671 1.23 Spanish 5 p a m p a
+ tests/PTsoloSPsuelo.words view
@@ -0,0 +1,2 @@+3 1.212 Portuguese 3 s o l o+1683 1.212 Spanish 1683 s u e l o
+ tests/example.bgdef view
@@ -0,0 +1,14 @@+-- a gap in linear scoring global alignments+Gap -4+-- gap opening in affine languages+GapOpen -4+-- gap extension in affine languages+GapExtend -2+-- opening the prefix or suffix in overhang languages+PreSufOpen -1+-- extending the prefix or suffix in overhang languages+PreSufExtend 0+-- score to return for bigram mismatches+Mismatch -3+-- needs to be set, but is not used (need better simple scoring parser)+Match -999999
+ tests/example.bgms view
@@ -0,0 +1,8 @@+German German h h h h 42.0+German German h a h a 33.9+German German a u a u 33.8+German German u s u s 33.7+German English m h t h 22.9+German English h a h o 22.9+German English a u o u 22.8+German English u s u s 22.7
+ tests/example.ugdef view
@@ -0,0 +1,22 @@+Set Conso b c d f g h j k m n p q s t v w x y z+Set Liquid l r+Set Vowel a e i o u a\' e\' i\' o\' u\' ə ɐ æ œ æh ɔh ɒ ɔ+Eq Conso 4+Eq Liquid 3+Eq Vowel 2+InSet Conso Conso 0+InSet Conso Liquid -1+InSet Conso Vowel -999999+InSet Liquid Conso -1+InSet Liquid Liquid -1+InSet Liquid Vowel -1+InSet Vowel Conso -999999+InSet Vowel Liquid -1+InSet Vowel Vowel 0+Gap -4+GapOpen -4+GapExtend -2+PreSufOpen -1+PreSufExtend 0+Match 1+Mismatch -999999
+ tests/example.words view
@@ -0,0 +1,6 @@+1 1.000 German 6 a a a a h h+2 1.100 German 6 h h a a a a+3 1.100 German 11 s c h e i s s h a u s+4 1.100 German 7 h a u s b a u+5 1.100 German 8 b a u m h a u s+6 1.100 English 9 s h i t h o u s e
+ tests/global-bigram-PTSP800-PTpoSPpampa-1.golden view
@@ -0,0 +1,15 @@+IDS: 49205 49205 SCORE: -9.00 NSCORE: -4.50 WORD: p o WORD: p o+ ^ p o $+ ^ p o $+ - -3.0 -3.0 -3.0++IDS: 49205 57671 SCORE: -0.58 NSCORE: -0.12 WORD: p o WORD: p a m p a+ ^ p a m p a $+ ^ p - - - o $+ - 5.0 -4.0 -4.0 -4.0 3.8 2.7++IDS: 57671 57671 SCORE: -18.00 NSCORE: -3.60 WORD: p a m p a WORD: p a m p a+ ^ p a m p a $+ ^ p a m p a $+ - -3.0 -3.0 -3.0 -3.0 -3.0 -3.0+
+ tests/global-bigram-PTSP800-PTsoloSPsuelo-1.golden view
@@ -0,0 +1,15 @@+IDS: 3 3 SCORE: -15.00 NSCORE: -3.75 WORD: s o l o WORD: s o l o+ ^ s o l o $+ ^ s o l o $+ - -3.0 -3.0 -3.0 -3.0 -3.0++IDS: 3 1683 SCORE: 12.47 NSCORE: 2.49 WORD: s o l o WORD: s u e l o+ ^ s u e l o $+ ^ s - o l o $+ - 5.5 -4.0 -3.0 4.1 5.8 4.1++IDS: 1683 1683 SCORE: -18.00 NSCORE: -3.60 WORD: s u e l o WORD: s u e l o+ ^ s u e l o $+ ^ s u e l o $+ - -3.0 -3.0 -3.0 -3.0 -3.0 -3.0+
+ tests/global-bigram-example-example-1.golden view
@@ -0,0 +1,105 @@+IDS: 1 1 SCORE: 24.00 NSCORE: 4.00 WORD: a a a a h h WORD: a a a a h h+ ^ a a a a h h $+ ^ a a a a h h $+ - -3.0 -3.0 -3.0 -3.0 -3.0 42.0 -3.0++IDS: 1 2 SCORE: 4.00 NSCORE: 0.67 WORD: a a a a h h WORD: h h a a a a+ ^ - - - - h h a a a a $+ ^ a a a a h h - - - - $+ - -4.0 -4.0 -4.0 -4.0 -3.0 42.0 -4.0 -4.0 -4.0 -4.0 -3.0++IDS: 1 3 SCORE: -41.00 NSCORE: -3.73 WORD: a a a a h h WORD: s c h e i s s h a u s+ ^ s c h e i s s h a u s $+ ^ - - - - - a a a a h h $+ - -4.0 -4.0 -4.0 -4.0 -4.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0++IDS: 1 4 SCORE: -25.00 NSCORE: -3.57 WORD: a a a a h h WORD: h a u s b a u+ ^ h a u s b a u $+ ^ - a a a a h h $+ - -4.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0++IDS: 1 5 SCORE: -29.00 NSCORE: -3.63 WORD: a a a a h h WORD: b a u m h a u s+ ^ b a u m h a u s $+ ^ - - a a a a h h $+ - -4.0 -4.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0++IDS: 1 6 SCORE: -33.00 NSCORE: -3.67 WORD: a a a a h h WORD: s h i t h o u s e+ ^ s h i t h o u s e $+ ^ - - - a a a a h h $+ - -4.0 -4.0 -4.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0++IDS: 2 2 SCORE: 60.90 NSCORE: 10.15 WORD: h h a a a a WORD: h h a a a a+ ^ h h a a a a $+ ^ h h a a a a $+ - -3.0 42.0 33.9 -3.0 -3.0 -3.0 -3.0++IDS: 2 3 SCORE: -9.10 NSCORE: -0.83 WORD: h h a a a a WORD: s c h e i s s h a u s+ ^ s c h e i s s h a - u s $+ ^ - - - - - - h h a a a a $+ - -4.0 -4.0 -4.0 -4.0 -4.0 -4.0 -3.0 -3.0 33.9 -4.0 -3.0 -3.0 -3.0++IDS: 2 4 SCORE: 6.90 NSCORE: 0.99 WORD: h h a a a a WORD: h a u s b a u+ ^ - h a u s b a u $+ ^ h h a - - a a a $+ - -4.0 -3.0 33.9 -4.0 -4.0 -3.0 -3.0 -3.0 -3.0++IDS: 2 5 SCORE: 2.90 NSCORE: 0.36 WORD: h h a a a a WORD: b a u m h a u s+ ^ b a u m h a - u s $+ ^ - - - h h a a a a $+ - -4.0 -4.0 -4.0 -3.0 -3.0 33.9 -4.0 -3.0 -3.0 -3.0++IDS: 2 6 SCORE: -7.10 NSCORE: -0.79 WORD: h h a a a a WORD: s h i t h o u s e+ ^ s h i t h o u s e $+ ^ - - - h h a a a a $+ - -4.0 -4.0 -4.0 -3.0 -3.0 22.9 -3.0 -3.0 -3.0 -3.0++IDS: 3 3 SCORE: 74.40 NSCORE: 6.76 WORD: s c h e i s s h a u s WORD: s c h e i s s h a u s+ ^ s c h e i s s h a u s $+ ^ s c h e i s s h a u s $+ - -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 33.9 33.8 33.7 -3.0++IDS: 3 4 SCORE: 55.40 NSCORE: 5.04 WORD: s c h e i s s h a u s WORD: h a u s b a u+ ^ - - - - - - - h a u s b a u $+ ^ s c h e i s s h a u s - - - $+ - -4.0 -4.0 -4.0 -4.0 -4.0 -4.0 -4.0 -3.0 33.9 33.8 33.7 -4.0 -4.0 -4.0 -3.0++IDS: 3 5 SCORE: 71.40 NSCORE: 6.49 WORD: s c h e i s s h a u s WORD: b a u m h a u s+ ^ - - - b a u m h a u s $+ ^ s c h e i s s h a u s $+ - -4.0 -4.0 -4.0 -3.0 -3.0 -3.0 -3.0 -3.0 33.9 33.8 33.7 -3.0++IDS: 3 6 SCORE: 34.40 NSCORE: 3.13 WORD: s c h e i s s h a u s WORD: s h i t h o u s e+ ^ - - - s h i t h o u s e $+ ^ s c h e i s s h a u s - $+ - -4.0 -4.0 -4.0 -3.0 -3.0 -3.0 -3.0 -3.0 22.9 22.8 22.7 -4.0 -3.0++IDS: 4 4 SCORE: 123.20 NSCORE: 17.60 WORD: h a u s b a u WORD: h a u s b a u+ ^ h a u s b a u $+ ^ h a u s b a u $+ - -3.0 33.9 33.8 33.7 -3.0 -3.0 33.8 -3.0++IDS: 4 5 SCORE: 67.40 NSCORE: 8.43 WORD: h a u s b a u WORD: b a u m h a u s+ ^ b a u m h a u s - - - $+ ^ - - - - h a u s b a u $+ - -4.0 -4.0 -4.0 -4.0 -3.0 33.9 33.8 33.7 -4.0 -4.0 -4.0 -3.0++IDS: 4 6 SCORE: 35.40 NSCORE: 3.93 WORD: h a u s b a u WORD: s h i t h o u s e+ ^ s h i t h o u s - - e $+ ^ - - - - h a u s b a u $+ - -4.0 -4.0 -4.0 -4.0 -3.0 22.9 22.8 22.7 -4.0 -4.0 -3.0 -3.0++IDS: 5 5 SCORE: 120.20 NSCORE: 15.03 WORD: b a u m h a u s WORD: b a u m h a u s+ ^ b a u m h a u s $+ ^ b a u m h a u s $+ - -3.0 -3.0 33.8 -3.0 -3.0 33.9 33.8 33.7 -3.0++IDS: 5 6 SCORE: 72.30 NSCORE: 8.03 WORD: b a u m h a u s WORD: s h i t h o u s e+ ^ s h i t h o u s e $+ ^ b a u m h a u s - $+ - -3.0 -3.0 -3.0 -3.0 22.9 22.9 22.8 22.7 -4.0 -3.0++IDS: 6 6 SCORE: -30.00 NSCORE: -3.33 WORD: s h i t h o u s e WORD: s h i t h o u s e+ ^ s h i t h o u s e $+ ^ s h i t h o u s e $+ - -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0 -3.0+
+ tests/global-bigram-order-order-1.golden view
@@ -0,0 +1,30 @@+IDS: 1 1 SCORE: -15.00 NSCORE: -3.75 WORD: a b D E WORD: a b D E+ ^ a b D E $+ ^ a b D E $+ - -3.0 -3.0 -3.0 -3.0 -3.0++IDS: 1 2 SCORE: 87.00 NSCORE: 21.75 WORD: a b D E WORD: a c E N+ ^ a c E N $+ ^ a b D E $+ - -3.0 99.0 -3.0 -3.0 -3.0++IDS: 1 3 SCORE: -15.00 NSCORE: -3.75 WORD: a b D E WORD: a b D E+ ^ a b D E $+ ^ a b D E $+ - -3.0 -3.0 -3.0 -3.0 -3.0++IDS: 2 2 SCORE: -15.00 NSCORE: -3.75 WORD: a c E N WORD: a c E N+ ^ a c E N $+ ^ a c E N $+ - -3.0 -3.0 -3.0 -3.0 -3.0++IDS: 2 3 SCORE: 87.00 NSCORE: 21.75 WORD: a c E N WORD: a b D E+ ^ a b D E $+ ^ a c E N $+ - -3.0 99.0 -3.0 -3.0 -3.0++IDS: 3 3 SCORE: -15.00 NSCORE: -3.75 WORD: a b D E WORD: a b D E+ ^ a b D E $+ ^ a b D E $+ - -3.0 -3.0 -3.0 -3.0 -3.0+
+ tests/global-unigram-PTSP800-PTpoSPpampa-1.golden view
@@ -0,0 +1,15 @@+IDS: 49205 49205 SCORE: 6.00 NSCORE: 3.00 WORD: p o WORD: p o+ p o+ p o+ 4.0 2.0++IDS: 49205 57671 SCORE: -8.00 NSCORE: -1.60 WORD: p o WORD: p a m p a+ p a m p a+ - - - p o+ -4.0 -4.0 -4.0 4.0 0.0++IDS: 57671 57671 SCORE: 16.00 NSCORE: 3.20 WORD: p a m p a WORD: p a m p a+ p a m p a+ p a m p a+ 4.0 2.0 4.0 4.0 2.0+
+ tests/global-unigram-PTSP800-PTsoloSPsuelo-1.golden view
@@ -0,0 +1,15 @@+IDS: 3 3 SCORE: 11.00 NSCORE: 2.75 WORD: s o l o WORD: s o l o+ s o l o+ s o l o+ 4.0 2.0 3.0 2.0++IDS: 3 1683 SCORE: 5.00 NSCORE: 1.00 WORD: s o l o WORD: s u e l o+ s u e l o+ s - o l o+ 4.0 -4.0 0.0 3.0 2.0++IDS: 1683 1683 SCORE: 13.00 NSCORE: 2.60 WORD: s u e l o WORD: s u e l o+ s u e l o+ s u e l o+ 4.0 2.0 2.0 3.0 2.0+
+ tests/global-unigram-example-example-1.golden view
@@ -0,0 +1,105 @@+IDS: 1 1 SCORE: 16.00 NSCORE: 2.67 WORD: a a a a h h WORD: a a a a h h+ a a a a h h+ a a a a h h+ 2.0 2.0 2.0 2.0 4.0 4.0++IDS: 1 2 SCORE: -8.00 NSCORE: -1.33 WORD: a a a a h h WORD: h h a a a a+ h h a a a a - -+ - - a a a a h h+ -4.0 -4.0 2.0 2.0 2.0 2.0 -4.0 -4.0++IDS: 1 3 SCORE: -26.00 NSCORE: -2.36 WORD: a a a a h h WORD: s c h e i s s h a u s+ s c h e i s s h a u - s+ - - - a a - - - a a h h+ -4.0 -4.0 -4.0 0.0 0.0 -4.0 -4.0 -4.0 2.0 0.0 -4.0 0.0++IDS: 1 4 SCORE: -16.00 NSCORE: -2.29 WORD: a a a a h h WORD: h a u s b a u+ h a u s b a u - -+ - a a - - a a h h+ -4.0 2.0 0.0 -4.0 -4.0 2.0 0.0 -4.0 -4.0++IDS: 1 5 SCORE: -12.00 NSCORE: -1.50 WORD: a a a a h h WORD: b a u m h a u s+ b a u m h a u - s+ - a a - - a a h h+ -4.0 2.0 0.0 -4.0 -4.0 2.0 0.0 -4.0 0.0++IDS: 1 6 SCORE: -28.00 NSCORE: -3.11 WORD: a a a a h h WORD: s h i t h o u s e+ s h i t h o u s e - -+ - - a - - a a - a h h+ -4.0 -4.0 0.0 -4.0 -4.0 0.0 0.0 -4.0 0.0 -4.0 -4.0++IDS: 2 2 SCORE: 16.00 NSCORE: 2.67 WORD: h h a a a a WORD: h h a a a a+ h h a a a a+ h h a a a a+ 4.0 4.0 2.0 2.0 2.0 2.0++IDS: 2 3 SCORE: -14.00 NSCORE: -1.27 WORD: h h a a a a WORD: s c h e i s s h a u s+ s c h e i s s h a u s+ - h h a a - - - a a -+ -4.0 0.0 4.0 0.0 0.0 -4.0 -4.0 -4.0 2.0 0.0 -4.0++IDS: 2 4 SCORE: -4.00 NSCORE: -0.57 WORD: h h a a a a WORD: h a u s b a u+ - h a u s b a u+ h h a a - - a a+ -4.0 4.0 2.0 0.0 -4.0 -4.0 2.0 0.0++IDS: 2 5 SCORE: -12.00 NSCORE: -1.50 WORD: h h a a a a WORD: b a u m h a u s+ - b a u m h a u s+ h h a a - - a a -+ -4.0 0.0 2.0 0.0 -4.0 -4.0 2.0 0.0 -4.0++IDS: 2 6 SCORE: -8.00 NSCORE: -0.89 WORD: h h a a a a WORD: s h i t h o u s e+ s h i t h o u s e+ h h a - - a a - a+ 0.0 4.0 0.0 -4.0 -4.0 0.0 0.0 -4.0 0.0++IDS: 3 3 SCORE: 36.00 NSCORE: 3.27 WORD: s c h e i s s h a u s WORD: s c h e i s s h a u s+ s c h e i s s h a u s+ s c h e i s s h a u s+ 4.0 4.0 4.0 2.0 2.0 4.0 4.0 4.0 2.0 2.0 4.0++IDS: 3 4 SCORE: -4.00 NSCORE: -0.36 WORD: s c h e i s s h a u s WORD: h a u s b a u+ - - h a u - s b a u -+ s c h e i s s h a u s+ -4.0 -4.0 4.0 0.0 0.0 -4.0 4.0 0.0 2.0 2.0 -4.0++IDS: 3 5 SCORE: 0.00 NSCORE: 0.00 WORD: s c h e i s s h a u s WORD: b a u m h a u s+ - - b a u - m h a u s+ s c h e i s s h a u s+ -4.0 -4.0 0.0 0.0 0.0 -4.0 0.0 4.0 2.0 2.0 4.0++IDS: 3 6 SCORE: 4.00 NSCORE: 0.36 WORD: s c h e i s s h a u s WORD: s h i t h o u s e+ s - h - i - t h o u s e+ s c h e i s s h a u s -+ 4.0 -4.0 4.0 -4.0 2.0 -4.0 0.0 4.0 0.0 2.0 4.0 -4.0++IDS: 4 4 SCORE: 20.00 NSCORE: 2.86 WORD: h a u s b a u WORD: h a u s b a u+ h a u s b a u+ h a u s b a u+ 4.0 2.0 2.0 4.0 4.0 2.0 2.0++IDS: 4 5 SCORE: 4.00 NSCORE: 0.50 WORD: h a u s b a u WORD: b a u m h a u s+ b a u m h a u s+ h a u s b a u -+ 0.0 2.0 2.0 0.0 0.0 2.0 2.0 -4.0++IDS: 4 6 SCORE: -10.00 NSCORE: -1.11 WORD: h a u s b a u WORD: s h i t h o u s e+ s h - i t h o u s e+ - h a u s b a u - -+ -4.0 4.0 -4.0 0.0 0.0 0.0 0.0 2.0 -4.0 -4.0++IDS: 5 5 SCORE: 24.00 NSCORE: 3.00 WORD: b a u m h a u s WORD: b a u m h a u s+ b a u m h a u s+ b a u m h a u s+ 4.0 2.0 2.0 4.0 4.0 2.0 2.0 4.0++IDS: 5 6 SCORE: -2.00 NSCORE: -0.22 WORD: b a u m h a u s WORD: s h i t h o u s e+ s h - i t h o u s e+ - b a u m h a u s -+ -4.0 0.0 -4.0 0.0 0.0 4.0 0.0 2.0 4.0 -4.0++IDS: 6 6 SCORE: 28.00 NSCORE: 3.11 WORD: s h i t h o u s e WORD: s h i t h o u s e+ s h i t h o u s e+ s h i t h o u s e+ 4.0 4.0 2.0 4.0 4.0 2.0 2.0 4.0 2.0+
+ tests/order.bgdef view
@@ -0,0 +1,14 @@+-- a gap in linear scoring global alignments+Gap -4+-- gap opening in affine languages+GapOpen -4+-- gap extension in affine languages+GapExtend -2+-- opening the prefix or suffix in overhang languages+PreSufOpen -1+-- extending the prefix or suffix in overhang languages+PreSufExtend 0+-- score to return for bigram mismatches+Mismatch -3+-- needs to be set, but is not used (need better simple scoring parser)+Match -999999
+ tests/order.bgms view
@@ -0,0 +1,2 @@+German English a b a c 99.0+English German a b a c -99.0
+ tests/order.words view
@@ -0,0 +1,3 @@+1 1.000 German 3 a b D E+2 1.100 English 9 a c E N+3 1.000 German 3 a b D E
tests/properties.hs view
@@ -1,11 +1,130 @@ module Main where -import Test.Framework.Providers.QuickCheck2-import Test.Framework.TH+import Control.Monad (forM,unless)+import Data.List (intersperse)+import Data.List.Split (splitOneOf)+import Debug.Trace+import qualified Data.ByteString.Builder as BB+import qualified Data.ByteString.Lazy.Char8 as BL+import qualified Data.Set as S+import qualified Data.Text.IO as TIO+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Builder as TL+import qualified Data.Text.Lazy.Encoding as TLE+import qualified Data.Text.Lazy.IO as TL+import System.FilePath ((</>),(<.>))+import System.IO (stdout)+import Test.Tasty+import Test.Tasty.QuickCheck as QC+import Test.Tasty.Silver as S+import Test.Tasty.Silver.Interactive as SI+import Test.Tasty.TH+import Data.List (isInfixOf) +import Linguistics.WordAlignment.Bigram+import Linguistics.WordAlignment.Word (parseWord,Word(..),addWordDelims,wordLazyTextWS,wordLazyTextWSB, FastChars(..))+import NLP.Scoring.SimpleUnigram+import NLP.Scoring.SimpleUnigram.Import +import Linguistics.WordAlignment+import Linguistics.WordAlignment.FastLookups +++{-+infixBigramTest = do+ ws <- (map parseWord . BL.lines) <$> BL.readFile "tests/example.words"+ simpleScoring <- simpleScoreFromFile "scores/defaultBigramScoring"+ let chkLs = S.fromList . map wordLang $ ws+ bigramScoring <- BL.readFile "tests/example.bgms" >>= return . mkBigramMap chkLs (-999999)+ ts <- forM ws $ \x -> forM ws $ \y -> do+ let fc = FastChars mempty 8+ let fd = FastDoubles mempty 8+ let !sco = getScores2 False bigramScoring (wordLang x) (wordLang y)+ let (d,bts) = alignInfixBigram2 simpleScoring sco fc fd 8 1 (wordWord x) (wordWord y)+ let ali = buildAlignmentBuilder 0 ([x,y],(d, bts))+ let hndl = stdout+ return $ BB.toLazyByteString ali+ return . TL.toStrict . TLE.decodeUtf8 . mconcat $ concat ts++goldenInfixBigramTest+ = S.goldenVsAction+ "Infix-Bigram"+ "tests/infix-bigram.golden"+ infixBigramTest+ id+-}++-- Test files are split according to this scheme:+--+-- @+-- directory name "/"+-- grammartype+-- unigram or bigram+-- unigram or bigram score name+-- words file+-- ".golden"+-- @++runSingleTest gldn [dir,grammar,"unigram",ugms,wrds,cnt,suffix] = do+ let c = read cnt+ unless (grammar `elem` ["global","infix"]) $ error gldn+ -- words file+ ws <- (map parseWord . BL.lines) <$> BL.readFile (dir </> wrds <.> "words")+ -- the bigram-associated simple scoring file+ simpleScoring <- simpleScoreFromFile $ dir </> ugms <.> "ugdef"+ -- a particular way we do scores for all inputs+ let xys = [ (x,y) | x <- ws, y <- ws, x <= y ]+ ts <- forM xys $ \ (x,y) -> do+ let fc = FastChars mempty 8+ let fd = FastDoubles mempty 8+ let (d,bts) = case grammar of+ "global" -> alignGlobalSimple2 simpleScoring fc fd 8 c (wordWord x) (wordWord y)+-- "infix" -> alignInfixSimple2 simpleScoring fc fd 8 c (wordWord x) (wordWord y)+ let ali = buildAlignmentBuilder 0 ([x,y],(d, bts))+ let hndl = stdout+ return $ BB.toLazyByteString ali+ let res = TL.toStrict . TLE.decodeUtf8 . mconcat $ ts+ return res+++runSingleTest gldn [dir,grammar,"bigram",bgms,wrds,cnt,suffix] = do+ let c = read cnt+ unless (grammar `elem` ["global","infix"]) $ error gldn+ -- words file+ ws <- (map parseWord . BL.lines) <$> BL.readFile (dir </> wrds <.> "words")+ -- the bigram scoring file+ let chkLs = S.fromList . map wordLang $ ws+ bigramScoring <- BL.readFile (dir </> bgms <.> "bgms") >>= return . mkBigramMap chkLs (-999999)+ -- the bigram-associated simple scoring file+ simpleScoring <- simpleScoreFromFile $ dir </> bgms <.> "bgdef"+ -- a particular way we do scores for all inputs+ let xys = [ (x,y) | x <- ws, y <- ws, x <= y ]+ ts <- forM xys $ \ (x,y) -> do+ let fc = FastChars mempty 8+ let fd = FastDoubles mempty 8+ let !sco = getScores2 False bigramScoring (wordLang x) (wordLang y)+ let (d,bts) = case grammar of+ "global" -> alignGlobalBigram2 simpleScoring sco fc fd 8 c (wordWord x) (wordWord y)+-- "infix" -> alignInfixBigram2 simpleScoring sco fc fd 8 c (wordWord x) (wordWord y)+ let ali = buildAlignmentBuilder 0 ([x,y],(d, bts))+ let hndl = stdout+ return $ BB.toLazyByteString ali+ let res = TL.toStrict . TLE.decodeUtf8 . mconcat $ ts+ return res++runSingleTest gldn xs = do+ error $ "don't know how to execute test based on: " ++ gldn++testWrapper gldn = S.goldenVsAction name gldn (runSingleTest gldn xs) id+ where name = concat . intersperse "-" . drop 1 . take (length xs - 2) $ xs+ xs = splitOneOf "/-." gldn+ main :: IO ()-main = $(defaultMainGenerator)+main = do+ gg <- testGroup "Known good alignments"+ <$> (fmap testWrapper) -- . filter ("order" `isInfixOf`))+ <$> S.findByExtension [".golden"] "tests"+ SI.defaultMain gg