NaturalLanguageAlphabets 0.0.2.0 → 0.1.0.0
raw patch · 10 files changed
+107/−386 lines, 10 filesdep +LinguisticsTypesdep ~hashtablesdep ~unordered-containersPVP ok
version bump matches the API change (PVP)
Dependencies added: LinguisticsTypes
Dependency ranges changed: hashtables, unordered-containers
API changes (from Hackage documentation)
- NLP.Alphabet.IMMC: IMMC :: Int -> IMMC
- NLP.Alphabet.IMMC: [getIMMC] :: IMMC -> Int
- NLP.Alphabet.IMMC: immc :: InternedMultiChar -> IMMC
- NLP.Alphabet.IMMC: instance Control.DeepSeq.NFData NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Aeson.Types.Class.FromJSON NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Aeson.Types.Class.ToJSON NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Binary.Class.Binary NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Hashable.Class.Hashable NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Serialize.Serialize NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.String.IsString NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Stringable.Stringable NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Vector.Generic.Base.Vector Data.Vector.Unboxed.Base.Vector NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Vector.Generic.Mutable.Base.MVector Data.Vector.Unboxed.Base.MVector NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Vector.Unboxed.Base.Unbox NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Classes.Eq NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Classes.Ord NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Generics.Constructor NLP.Alphabet.IMMC.C1_0IMMC
- NLP.Alphabet.IMMC: instance GHC.Generics.Datatype NLP.Alphabet.IMMC.D1IMMC
- NLP.Alphabet.IMMC: instance GHC.Generics.Generic NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Generics.Selector NLP.Alphabet.IMMC.S1_0_0IMMC
- NLP.Alphabet.IMMC: instance GHC.Read.Read NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Show.Show NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: newtype IMMC
- NLP.Alphabet.IMMC.Internal: immcBimap :: IORef (Bimap InternedMultiChar Int)
- NLP.Alphabet.IMMC.Internal: immcBimapAdd :: InternedMultiChar -> Int
- NLP.Alphabet.IMMC.Internal: immcBimapLookupInt :: Int -> InternedMultiChar
- NLP.Alphabet.MultiChar: InternedMultiChar :: {-# UNPACK #-} !Id -> {-# UNPACK #-} !MultiChar -> InternedMultiChar
- NLP.Alphabet.MultiChar: MultiChar :: Text -> MultiChar
- NLP.Alphabet.MultiChar: [getMultiChar] :: MultiChar -> Text
- NLP.Alphabet.MultiChar: [internedMultiCharId] :: InternedMultiChar -> {-# UNPACK #-} !Id
- NLP.Alphabet.MultiChar: [uninternMultiChar] :: InternedMultiChar -> {-# UNPACK #-} !MultiChar
- NLP.Alphabet.MultiChar: data InternedMultiChar
- NLP.Alphabet.MultiChar: imcCache :: Cache InternedMultiChar
- NLP.Alphabet.MultiChar: instance Control.DeepSeq.NFData NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Control.DeepSeq.NFData NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance Data.Data.Data NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.Data.Data NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance Data.Hashable.Class.Hashable (Data.Interned.Internal.Description NLP.Alphabet.MultiChar.InternedMultiChar)
- NLP.Alphabet.MultiChar: instance Data.Hashable.Class.Hashable NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.Hashable.Class.Hashable NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance Data.Interned.Internal.Interned NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.String.IsString NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.String.IsString NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance Data.Stringable.Stringable NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.Stringable.Stringable NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Classes.Eq (Data.Interned.Internal.Description NLP.Alphabet.MultiChar.InternedMultiChar)
- NLP.Alphabet.MultiChar: instance GHC.Classes.Eq NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Classes.Eq NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Classes.Ord NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Classes.Ord NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Constructor NLP.Alphabet.MultiChar.C1_0InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Constructor NLP.Alphabet.MultiChar.C1_0MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Datatype NLP.Alphabet.MultiChar.D1InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Datatype NLP.Alphabet.MultiChar.D1MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Generic NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Generic NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Selector NLP.Alphabet.MultiChar.S1_0_0InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Selector NLP.Alphabet.MultiChar.S1_0_0MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Selector NLP.Alphabet.MultiChar.S1_0_1InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Read.Read NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Read.Read NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Show.Show NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Show.Show NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: internMultiChar :: MultiChar -> MultiChar
- NLP.Alphabet.MultiChar: newtype MultiChar
+ NLP.Scoring.SimpleUnigram: instance Data.Aeson.Types.Class.FromJSON NLP.Scoring.SimpleUnigram.SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance Data.Aeson.Types.Class.ToJSON NLP.Scoring.SimpleUnigram.SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Classes.Eq NLP.Scoring.SimpleUnigram.SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Constructor NLP.Scoring.SimpleUnigram.C1_0SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Datatype NLP.Scoring.SimpleUnigram.D1SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Generic NLP.Scoring.SimpleUnigram.SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_0SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_1SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_2SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_3SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_4SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_5SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Read.Read NLP.Scoring.SimpleUnigram.SimpleScoring
- NLP.Scoring.SimpleUnigram: SimpleScoring :: !(BasicHashTable (IMMC, IMMC) Double) -> !Double -> !Double -> !Double -> !Double -> !Double -> SimpleScoring
+ NLP.Scoring.SimpleUnigram: SimpleScoring :: !(HashMap (BTI, BTI) Double) -> !Double -> !Double -> !Double -> !Double -> !Double -> SimpleScoring
- NLP.Scoring.SimpleUnigram: [simpleScore] :: SimpleScoring -> !(BasicHashTable (IMMC, IMMC) Double)
+ NLP.Scoring.SimpleUnigram: [simpleScore] :: SimpleScoring -> !(HashMap (BTI, BTI) Double)
- NLP.Scoring.SimpleUnigram: scoreUnigram :: SimpleScoring -> IMMC -> IMMC -> Double
+ NLP.Scoring.SimpleUnigram: scoreUnigram :: SimpleScoring -> BTI -> BTI -> Double
- NLP.Scoring.SimpleUnigram.Import: PLeqset :: Text -> [IMMC] -> ParsedLine
+ NLP.Scoring.SimpleUnigram.Import: PLeqset :: Text -> [BTI] -> ParsedLine
- NLP.Scoring.SimpleUnigram.Import: PLset :: Text -> [IMMC] -> ParsedLine
+ NLP.Scoring.SimpleUnigram.Import: PLset :: Text -> [BTI] -> ParsedLine
Files
- NLP/Alphabet/IMMC.hs +0/−96
- NLP/Alphabet/IMMC/Internal.hs +0/−43
- NLP/Alphabet/MultiChar.hs +0/−140
- NLP/Scoring/SimpleUnigram.hs +30/−11
- NLP/Scoring/SimpleUnigram/Import.hs +7/−13
- NaturalLanguageAlphabets.cabal +7/−29
- README.md +7/−5
- changelog.md +9/−0
- tests/Benchmark.hs +23/−23
- tests/properties.hs +24/−26
− NLP/Alphabet/IMMC.hs
@@ -1,96 +0,0 @@---- | An implementation of @Int@-mapped @MultiChar@s with internalization.--module NLP.Alphabet.IMMC where--import Control.Applicative-import Control.DeepSeq (NFData(..))-import Data.Aeson as A-import Data.Binary as DB-import Data.Hashable-import Data.Serialize as DS-import Data.Serialize.Text-import Data.Stringable as SA-import Data.String as IS-import Data.Text.Binary-import Data.Vector.Unboxed.Deriving-import GHC.Generics-import qualified Data.ByteString.Short as BS-import qualified Data.Text as T-import qualified Data.Text.Encoding as T--import NLP.Alphabet.IMMC.Internal-import NLP.Alphabet.MultiChar (InternedMultiChar)------ * A somewhat fragile (?) encoding of multichars using the internalized--- @Id@. Should only be used via it's wrapper. The mapped @Int@s are not--- consecutive.--newtype IMMC = IMMC { getIMMC :: Int }- deriving (Eq,Generic)--derivingUnbox "IMMC"- [t| IMMC -> Int |]- [| getIMMC |]- [| IMMC |]--instance Ord IMMC where- IMMC l `compare` IMMC r = immcBimapLookupInt l `compare` immcBimapLookupInt r- {-# Inline compare #-}--immc :: InternedMultiChar -> IMMC-immc s = IMMC $! immcBimapAdd s-{-# Inline immc #-}--instance IsString IMMC where- fromString = immc . IS.fromString- {-# Inline fromString #-}--instance Show IMMC where- showsPrec p i r = showsPrec p (toString i) r- {-# Inline showsPrec #-}--instance Read IMMC where- readsPrec p str = [ (immc $ IS.fromString s, y) | (s,y) <- readsPrec p str ]- {-# Inline readsPrec #-}--instance Hashable IMMC--instance Stringable IMMC where- toString = toString . immcBimapLookupInt . getIMMC- fromString = immc . SA.fromString- length = SA.length . immcBimapLookupInt . getIMMC- toText = toText . immcBimapLookupInt . getIMMC- fromText = immc . fromText- {-# Inline toString #-}- {-# Inline fromString #-}- {-# Inline length #-}- {-# Inline toText #-}- {-# Inline fromText #-}--instance NFData IMMC where- rnf = rnf . getIMMC- {-# Inline rnf #-}--instance Binary IMMC where- put = DB.put . toText- get = fromText <$> DB.get- {-# Inline put #-}- {-# Inline get #-}--instance Serialize IMMC where- put = DS.put . toText- get = fromText <$> DS.get- {-# Inline put #-}- {-# Inline get #-}--instance FromJSON IMMC where- parseJSON s = fromText <$> parseJSON s- {-# Inline parseJSON #-}--instance ToJSON IMMC where- toJSON = toJSON . toText- {-# Inline toJSON #-}-
− NLP/Alphabet/IMMC/Internal.hs
@@ -1,43 +0,0 @@---- | This module keeps a persistent @bimap@ between @InternedMultiChar@s--- and @Int@s------ TODO make this a bimap @Text <-> Vector@. Compare performance when--- printing backtracking results. (Do this after the Builder-based--- backtracking is online)--module NLP.Alphabet.IMMC.Internal where--import Data.IORef (newIORef,IORef,readIORef,atomicWriteIORef,atomicModifyIORef')-import System.IO.Unsafe (unsafePerformIO,unsafeDupablePerformIO)--import Data.Bijection.Hash (Bimap,empty,lookupL,lookupR,size,insert)--import NLP.Alphabet.MultiChar----immcBimap :: IORef (Bimap InternedMultiChar Int)-immcBimap = unsafePerformIO $ newIORef empty-{-# NoInline immcBimap #-}---- | Add @InternedMultiChar@ and return @Int@ key. Will return key for--- existing string and thereby serves for lookup in left-to-right--- direction.--immcBimapAdd :: InternedMultiChar -> Int-immcBimapAdd k = unsafeDupablePerformIO $ atomicModifyIORef' immcBimap $ \m ->- case lookupL m k of Just i -> (m,i)- Nothing -> let s = size m- in (insert m (k,s) , s)-{-# Inline immcBimapAdd #-}---- | Lookup the @InternedMultiChar@ based on an @Int@ key. Unsafe totality--- assumption.--immcBimapLookupInt :: Int -> InternedMultiChar-immcBimapLookupInt r = seq r . unsafeDupablePerformIO $ atomicModifyIORef' immcBimap $ \m ->- case lookupR m r of Just l -> (m,l)- Nothing -> error "immcBimapLookupInt: totality assumption invalidated"-{-# Inline immcBimapLookupInt #-}-
− NLP/Alphabet/MultiChar.hs
@@ -1,140 +0,0 @@---- | An alphabet, where each character is a short piece of @Text@.--module NLP.Alphabet.MultiChar where--import Control.DeepSeq (NFData(..))-import Data.Function (on)-import Data.Hashable-import Data.Interned-import Data.Interned.Internal (getCache)-import Data.Stringable-import Data.String (IsString)-import qualified Data.ByteString.Short as BS-import qualified Data.ByteString.Short.Internal as BS-import qualified Data.String as S-import qualified Data.Text as T-import qualified Data.Text.Encoding as T-import GHC.Generics-import qualified Data.HashMap.Strict as HM-import qualified Data.Array as A-import Data.Typeable (Typeable)-import Data.Data (Data)------ * 'MultiChar's capture UTF characters that are encoded using one or more--- symbols.---- | Interns a 'MultiChar' character.--internMultiChar :: MultiChar -> MultiChar-internMultiChar = uninternMultiChar . intern-{-# Inline internMultiChar #-}---- | Wrap a short bytestring. Read and Show instances behave like for normal--- strings.--newtype MultiChar = MultiChar { getMultiChar :: T.Text }- deriving (Eq,Ord,Generic,Data,Typeable)--instance Show MultiChar where- showsPrec p (MultiChar mc) r = showsPrec p (toString mc) r- {-# Inline showsPrec #-}--instance Read MultiChar where- readsPrec p str = [ (MultiChar x, y) | (x,y) <- readsPrec p str ]- {-# Inline readsPrec #-}--instance Hashable MultiChar--instance IsString MultiChar where- fromString = MultiChar . S.fromString- {-# Inline fromString #-}--instance Stringable MultiChar where- toString = T.unpack . getMultiChar- fromString = MultiChar . T.pack- length = T.length . getMultiChar- fromText = MultiChar- toText = getMultiChar- {-# Inline toString #-}- {-# Inline fromString #-}- {-# Inline length #-}- {-# Inline fromText #-}- {-# Inline toText #-}--instance NFData MultiChar where- rnf = rnf . getMultiChar- {-# Inline rnf #-}------ * Interned---- | Interned 'MultiChar'.------ TODO Check 'Ord' instance. We @compare `on` uninternMultiChar@.--data InternedMultiChar = InternedMultiChar- { internedMultiCharId :: {-# UNPACK #-} !Id- , uninternMultiChar :: {-# UNPACK #-} !MultiChar- }- deriving (Generic,Data,Typeable)--instance IsString InternedMultiChar where- fromString = intern . S.fromString- {-# Inline fromString #-}--instance Eq InternedMultiChar where- (==) = (==) `on` internedMultiCharId- {-# Inline (==) #-}--instance Ord InternedMultiChar where- compare = compare `on` uninternMultiChar -- internedMultiCharId- {-# Inline compare #-}--instance Read InternedMultiChar where- readsPrec p str = [ (intern x, y) | (x,y) <- readsPrec p str ]- {-# Inline readsPrec #-}--instance Show InternedMultiChar where- showsPrec d (InternedMultiChar _ mc) = showsPrec d mc- {-# Inline showsPrec #-}--instance Hashable InternedMultiChar where- hashWithSalt salt = hashWithSalt salt . internedMultiCharId- hash = hash . internedMultiCharId- {-# Inline hashWithSalt #-}- {-# Inline hash #-}--instance Interned InternedMultiChar where- type Uninterned InternedMultiChar = MultiChar- newtype Description InternedMultiChar = DMC MultiChar deriving (Eq,Hashable)- describe = DMC . MultiChar . T.copy . getMultiChar -- @DMC@ alone is type-correct. With 'T.copy' we make sure not to keep long @Text@s. TODO benchmark!- identify = InternedMultiChar- cache = imcCache- {-# Inline describe #-}- {-# Inline identify #-}- {-# Inline cache #-}--imcCache :: Cache InternedMultiChar-imcCache = mkCache-{-# NOINLINE imcCache #-}--instance Stringable InternedMultiChar where- toString = toString . uninternMultiChar- fromString = intern . fromString- length = Data.Stringable.length . uninternMultiChar- toText = toText . uninternMultiChar- fromText = intern . fromText- {-# Inline toString #-}- {-# Inline fromString #-}- {-# Inline length #-}- {-# Inline toText #-}- {-# Inline fromText #-}--instance NFData InternedMultiChar where- rnf (InternedMultiChar i c) = rnf i `seq` rnf c- {-# Inline rnf #-}-
NLP/Scoring/SimpleUnigram.hs view
@@ -3,35 +3,54 @@ module NLP.Scoring.SimpleUnigram where -import Data.HashTable.IO (BasicHashTable)-import qualified Data.HashTable.IO as H-import System.IO.Unsafe (unsafePerformIO)+import GHC.Generics+import Data.HashMap.Strict+import Data.Aeson -import NLP.Alphabet.IMMC+import NLP.Text.BTI --- | Score 'MultiChar's @x@ and @y@ based on the simple scoring system: (i)+-- | Score 'BTI's @x@ and @y@ based on the simple scoring system: (i) -- lookup (x,y) and use the score if found; (ii) if (x,y) is not in the -- database, then return the default matching 'defMatch' score if @x==y@, -- otherwise return the default mismatch 'defMismatch' score. -scoreUnigram :: SimpleScoring -> IMMC -> IMMC -> Double+scoreUnigram :: SimpleScoring -> BTI -> BTI -> Double scoreUnigram SimpleScoring {..} x y =- maybe (if x==y then defMatch else defMismatch)- id- (unsafePerformIO $ H.lookup simpleScore (x,y))+ lookupDefault (if x==y then defMatch else defMismatch) (x,y) simpleScore {-# INLINE scoreUnigram #-} -- | Collect the hashtable and scalar values for simple scoring.+--+-- TODO binary and cereal instances data SimpleScoring = SimpleScoring- { simpleScore :: !(BasicHashTable (IMMC,IMMC) Double)+ { simpleScore :: !(HashMap (BTI,BTI) Double) , gapScore :: !Double , gapOpen :: !Double , gapExtend :: !Double , defMatch :: !Double , defMismatch :: !Double }- deriving (Show)+ deriving (Read,Show,Eq,Generic)++instance FromJSON SimpleScoring where+ parseJSON (Object v) = SimpleScoring <$>+ (fromList `fmap` (v .: "simpleScore")) <*>+ v .: "gapScore" <*>+ v .: "gapOpen" <*>+ v .: "gapExtend" <*>+ v .: "defMatch" <*>+ v .: "defMismatch"++instance ToJSON SimpleScoring where+ toJSON (SimpleScoring ss gs go ge dm di)+ = object [ "simpleScore" .= toList ss+ , "gapScore" .= gs+ , "gapOpen" .= go+ , "gapExtend" .= ge+ , "defMatch" .= dm+ , "defMismatch" .= di+ ]
NLP/Scoring/SimpleUnigram/Import.hs view
@@ -2,21 +2,15 @@ module NLP.Scoring.SimpleUnigram.Import where import Control.Applicative---import Data.ByteString.Char8 (ByteString)-import Data.HashTable.IO (BasicHashTable)+import Data.HashMap.Strict (fromList) import Data.Stringable---import qualified Data.Attoparsec.ByteString as AB---import qualified Data.Attoparsec.ByteString.Char8 as AB hiding (takeWhile1,skipWhile)---import qualified Data.ByteString.Char8 as B-import qualified Data.HashTable.IO as H-import System.IO.Unsafe (unsafePerformIO)--import qualified Data.Attoparsec.Text as AT import Data.Text (Text)+import qualified Data.Attoparsec.Text as AT import qualified Data.Text as T import qualified Data.Text.IO as T -import NLP.Alphabet.IMMC+import NLP.Text.BTI+ import NLP.Scoring.SimpleUnigram @@ -28,9 +22,9 @@ -- circular imports) data ParsedLine- = PLset Text [IMMC]+ = PLset Text [BTI] | PLeq Text Double- | PLeqset Text [IMMC]+ | PLeqset Text [BTI] | PLinset Text Text Double | PLgap Double | PLgapopen Double@@ -68,7 +62,7 @@ genSimpleScoring :: Text -> SimpleScoring genSimpleScoring l = SimpleScoring t g go ge dm di where- t = unsafePerformIO $ H.fromListWithSizeHint (Prelude.length ys) ys+ t = fromList ys ls = T.lines l xs = map parseLine ls ys = concatMap genPairs $ iss ++ eqs
NaturalLanguageAlphabets.cabal view
@@ -1,5 +1,5 @@ name: NaturalLanguageAlphabets-version: 0.0.2.0+version: 0.1.0.0 author: Christian Hoener zu Siederdissen maintainer: choener@bioinf.uni-leipzig.de homepage: https://github.com/choener/NaturalLanguageAlphabets@@ -12,19 +12,9 @@ stability: experimental cabal-version: >= 1.10.0 tested-with: GHC == 7.8.4, GHC == 7.10.2-synopsis: Alphabet and word representations+synopsis: Simple scoring schemes for word alignments description:- Provides different encoding for characters and words in natural- language processing. A character will often be encoded as a- unicode text string as we deal with multi-symbol characters.- .- Internal encoding of IMMC symbols are 0-based integers, which- allows for the use of unboxed containers.- .- A very simple unigram-based scoring scheme and DSL to write- such schemes are also provided.- .- <https://github.com/choener/NaturalLanguageAlphabets/blob/master/README.md>+ Provides a simple scoring scheme for word alignments. @@ -35,13 +25,6 @@ -flag llvm- description: build the benchmark using LLVM- default: False- manual: True--- library build-depends: base > 4.7 && < 4.9 , aeson >= 0.8 && < 0.11@@ -55,8 +38,8 @@ , deepseq >= 1.3 && < 1.5 , file-embed >= 0.0.6 && < 0.0.10 , hashable >= 1.2 && < 1.3- , hashtables >= 1.1 && < 1.3 , intern >= 0.9 && < 0.10+ , LinguisticsTypes >= 0.0.0 && < 0.0.1 , QuickCheck >= 2.7 && < 2.9 , stringable >= 0.1.2 && < 0.2 , system-filepath >= 0.4.9 && < 0.5@@ -67,9 +50,6 @@ , vector-th-unbox >= 0.2 && < 0.3 exposed-modules:- NLP.Alphabet.IMMC- NLP.Alphabet.IMMC.Internal- NLP.Alphabet.MultiChar NLP.Scoring.SimpleUnigram NLP.Scoring.SimpleUnigram.Default NLP.Scoring.SimpleUnigram.Import@@ -98,6 +78,7 @@ , criterion >= 1.0.2 && < 1.1.1 , deepseq , hashtables+ , LinguisticsTypes , mwc-random >= 0.13 && < 0.14 , NaturalLanguageAlphabets , random >= 1.0 && < 1.2@@ -119,11 +100,6 @@ -funbox-strict-fields -funfolding-use-threshold1000 -funfolding-keeness-factor1000- if flag(llvm)- ghc-options:- -fllvm- -optlo-O3 -optlo-std-compile-opts- -fllvm-tbaa @@ -144,6 +120,7 @@ , aeson , binary , cereal+ , LinguisticsTypes , NaturalLanguageAlphabets , QuickCheck , stringable@@ -151,6 +128,7 @@ , test-framework-quickcheck2 >= 0.3 && < 0.4 , test-framework-th >= 0.2 && < 0.3 , text+ , unordered-containers
README.md view
@@ -2,11 +2,13 @@ # Natural Language Alphabets -Efficient, alphabet symbols. The symbols are interned, and hashed. This is-quite useful for k-gram scoring, where we have different sets of symbols with-different scores. IMMC symbols are internally represented via Ints in the range-[0..]. This makes it possible to use unboxed containers when handling IMMC-symbols.+This library provides a simple scoring method for alignment of words with+natural language alphabets. The underlying /character/ type is BTI from the+LinguisticsTypes package, which provides efficient encoding of arbitrarily+complex atomic characters.++The actual alignment algorithms can be found in the+[WordAlignment](http://hackage.haskell.org/package/WordAlignment) package.
changelog.md view
@@ -1,3 +1,12 @@+0.1.0.0+-------++- moved IMMC to LinguisticsTypes library and renamed to BTI+- removed MultiChar type+- using HashMap from unordered-containers instead of HashTable from hashtables.+ Now we do not have to unsafePerformIO anymore.+- JSON (de)serialization for SimpleScoring scheme+ 0.0.2.0 -------
tests/Benchmark.hs view
@@ -7,24 +7,24 @@ module Main where -import Criterion.Main-import qualified Data.Vector.Unboxed as VU-import qualified Data.Vector.Generic as VG-import qualified Data.Vector as VV-import Text.Printf-import Data.Tuple (swap) import Control.Applicative ((<$>))-import System.Random.MWC-import System.Random import Control.DeepSeq import Control.Monad-import qualified Data.HashMap.Strict as UCHS-import qualified Data.IntMap.Strict as CIS+import Criterion.Main import Data.String+import Data.Tuple (swap)+import qualified Data.HashMap.Strict as UCHS import qualified Data.HashTable.IO as HT+import qualified Data.IntMap.Strict as CIS+import qualified Data.Vector as VV+import qualified Data.Vector.Generic as VG+import qualified Data.Vector.Unboxed as VU import System.IO.Unsafe (unsafePerformIO)+import System.Random+import System.Random.MWC+import Text.Printf -import NLP.Alphabet.IMMC+import NLP.Text.BTI @@ -36,18 +36,18 @@ (UCHS.size hmsK) (CIS.size cisK) (-1 :: Int) defaultMain [ bgroup " 100 keys" [ bench "HashMap.Strict" $ whnf (\k -> UCHS.lookupDefault 0 k hmsH) (VU.head keys)- , bench "IntMap.Strict" $ whnf (\k -> CIS.findWithDefault 0 k cisH) (getIMMC $ VU.head keys)+ , bench "IntMap.Strict" $ whnf (\k -> CIS.findWithDefault 0 k cisH) (getBTI $ VU.head keys) , bench "HashTable" $ whnf (\k -> htLookup htH k) (VU.head keys) ] , bgroup "10000 keys" [ bench "HashMap.Strict" $ whnf (\k -> UCHS.lookupDefault 0 k hmsK) (VU.head keys)- , bench "IntMap.Strict" $ whnf (\k -> CIS.findWithDefault 0 k cisK) (getIMMC $ VU.head keys)+ , bench "IntMap.Strict" $ whnf (\k -> CIS.findWithDefault 0 k cisK) (getBTI $ VU.head keys) , bench "HashTable" $ whnf (\k -> htLookup htK k) (VU.head keys) ] -- , bgroup " 100 known" [ bench "uchsH" $ whnf (\ks -> VU.sum $ VU.map (\k -> UCHS.lookupDefault 0 k hmsH) ks) (VU.take 100 keys)--- , bench " cisH" $ whnf (\ks -> VU.sum $ VU.map (\k -> CIS.findWithDefault 0 k cisH) ks) (VU.map getIMMC $ VU.take 100 keys)+-- , bench " cisH" $ whnf (\ks -> VU.sum $ VU.map (\k -> CIS.findWithDefault 0 k cisH) ks) (VU.map getBTI $ VU.take 100 keys) -- ] -- , bgroup " 1 unknown" [ bench "uchsH" $ whnf (\k -> UCHS.lookupDefault 0 k hmsH) (VU.head unks)--- , bench " cisH" $ whnf (\k -> CIS.findWithDefault 0 k cisH) (getIMMC $ VU.head unks)+-- , bench " cisH" $ whnf (\k -> CIS.findWithDefault 0 k cisH) (getBTI $ VU.head unks) -- ] ] @@ -59,25 +59,25 @@ {-# Inline htLookup #-} -type HTB = HT.BasicHashTable IMMC Double+type HTB = HT.BasicHashTable BTI Double setupEnv = do -- create keys strs :: [String] <- replicateM 10000 rString -- scores to look up scrs :: [Double] <- replicateM 10000 $ randomRIO (0 , 9)- -- create IMMC keys- let keys = map (immc . fromString) strs+ -- create BTI keys+ let keys = map (bti . fromString) strs -- create random keys, mostly not in strs unks :: [String] <- replicateM 10000 rString- let sknu = map (immc . fromString) unks+ let sknu = map (bti . fromString) unks -- for 100 keys- let hmsK = UCHS.fromList $ take 100 $ zip keys scrs- let cisK = CIS.fromList $ take 100 $ zip (map getIMMC keys) scrs+ let hmsK = UCHS.fromList $ take 100 $ zip keys scrs+ let cisK = CIS.fromList $ take 100 $ zip (map getBTI keys) scrs htH :: HTB <- HT.fromList $ take 100 $ zip keys scrs -- for 10 000 keys- let hmsM = UCHS.fromList $ zip keys scrs- let cisM = CIS.fromList $ zip (map getIMMC keys) scrs+ let hmsM = UCHS.fromList $ zip keys scrs+ let cisM = CIS.fromList $ zip (map getBTI keys) scrs htK :: HTB <- HT.fromList $ zip keys scrs return (VU.fromList keys , VU.fromList sknu , (hmsK,cisK,htH) , (hmsM,cisM,htK) )
tests/properties.hs view
@@ -2,46 +2,44 @@ module Main where import Control.Applicative-import Data.String-import Data.Stringable hiding (fromString)+import Data.Stringable import Debug.Trace import qualified Data.Aeson as A import qualified Data.Binary as B import qualified Data.Serialize as S import Test.Framework.Providers.QuickCheck2 import Test.Framework.TH+import Data.HashMap.Strict (fromList,union) -import NLP.Alphabet.IMMC+import NLP.Text.BTI +import NLP.Scoring.SimpleUnigram+import NLP.Scoring.SimpleUnigram.Default --- * IMMC --- basic property of interning+-- | Test aeson conversion. We add random @(key,value)@ pairs in the form+-- of @xs@ to the default scoring. We also randomize the individual scoring+-- constants.+--+-- Testing is done by serialization followed by deserialization and testing+-- for equality. -prop_IMMC (t :: String)- | t == u = True- | otherwise = traceShow (t, getIMMC i, u) False- where i :: IMMC = fromString t- u = toString i+prop_Aeson ( xs :: [((String,String),Double)]+ , (gs :: Double, go :: Double, ge :: Double, dm :: Double, di :: Double)+ )+ = Just def' == A.decode (A.encode def')+ where def = clvDefaults+ xs' = fromList $ map (\((x,y),s) -> ((fromString x,fromString y),s)) xs+ def' = def { simpleScore = simpleScore def `union` xs'+ , gapScore = gs+ , gapOpen = go+ , gapExtend = ge+ , defMatch = dm+ , defMismatch = di+ } --- binary -prop_Binary (t :: String) = t == toString j- where i :: IMMC = fromString t- j :: IMMC = B.decode $ B.encode i---- cereal--prop_Serialize (t :: String) = Right t == (toString <$> j)- where i :: IMMC = fromString t- j :: Either String IMMC = S.decode $ S.encode i---- aeson (more complicated to due the json format!--prop_Aeson (t :: String) = Just [t] == (map toString <$> j)- where i :: [IMMC] = [fromString t]- j :: Maybe [IMMC] = A.decode $ A.encode i main :: IO () main = $(defaultMainGenerator)