packages feed

NaturalLanguageAlphabets 0.0.2.0 → 0.1.0.0

raw patch · 10 files changed

+107/−386 lines, 10 filesdep +LinguisticsTypesdep ~hashtablesdep ~unordered-containersPVP ok

version bump matches the API change (PVP)

Dependencies added: LinguisticsTypes

Dependency ranges changed: hashtables, unordered-containers

API changes (from Hackage documentation)

- NLP.Alphabet.IMMC: IMMC :: Int -> IMMC
- NLP.Alphabet.IMMC: [getIMMC] :: IMMC -> Int
- NLP.Alphabet.IMMC: immc :: InternedMultiChar -> IMMC
- NLP.Alphabet.IMMC: instance Control.DeepSeq.NFData NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Aeson.Types.Class.FromJSON NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Aeson.Types.Class.ToJSON NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Binary.Class.Binary NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Hashable.Class.Hashable NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Serialize.Serialize NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.String.IsString NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Stringable.Stringable NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Vector.Generic.Base.Vector Data.Vector.Unboxed.Base.Vector NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Vector.Generic.Mutable.Base.MVector Data.Vector.Unboxed.Base.MVector NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance Data.Vector.Unboxed.Base.Unbox NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Classes.Eq NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Classes.Ord NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Generics.Constructor NLP.Alphabet.IMMC.C1_0IMMC
- NLP.Alphabet.IMMC: instance GHC.Generics.Datatype NLP.Alphabet.IMMC.D1IMMC
- NLP.Alphabet.IMMC: instance GHC.Generics.Generic NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Generics.Selector NLP.Alphabet.IMMC.S1_0_0IMMC
- NLP.Alphabet.IMMC: instance GHC.Read.Read NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: instance GHC.Show.Show NLP.Alphabet.IMMC.IMMC
- NLP.Alphabet.IMMC: newtype IMMC
- NLP.Alphabet.IMMC.Internal: immcBimap :: IORef (Bimap InternedMultiChar Int)
- NLP.Alphabet.IMMC.Internal: immcBimapAdd :: InternedMultiChar -> Int
- NLP.Alphabet.IMMC.Internal: immcBimapLookupInt :: Int -> InternedMultiChar
- NLP.Alphabet.MultiChar: InternedMultiChar :: {-# UNPACK #-} !Id -> {-# UNPACK #-} !MultiChar -> InternedMultiChar
- NLP.Alphabet.MultiChar: MultiChar :: Text -> MultiChar
- NLP.Alphabet.MultiChar: [getMultiChar] :: MultiChar -> Text
- NLP.Alphabet.MultiChar: [internedMultiCharId] :: InternedMultiChar -> {-# UNPACK #-} !Id
- NLP.Alphabet.MultiChar: [uninternMultiChar] :: InternedMultiChar -> {-# UNPACK #-} !MultiChar
- NLP.Alphabet.MultiChar: data InternedMultiChar
- NLP.Alphabet.MultiChar: imcCache :: Cache InternedMultiChar
- NLP.Alphabet.MultiChar: instance Control.DeepSeq.NFData NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Control.DeepSeq.NFData NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance Data.Data.Data NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.Data.Data NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance Data.Hashable.Class.Hashable (Data.Interned.Internal.Description NLP.Alphabet.MultiChar.InternedMultiChar)
- NLP.Alphabet.MultiChar: instance Data.Hashable.Class.Hashable NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.Hashable.Class.Hashable NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance Data.Interned.Internal.Interned NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.String.IsString NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.String.IsString NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance Data.Stringable.Stringable NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance Data.Stringable.Stringable NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Classes.Eq (Data.Interned.Internal.Description NLP.Alphabet.MultiChar.InternedMultiChar)
- NLP.Alphabet.MultiChar: instance GHC.Classes.Eq NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Classes.Eq NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Classes.Ord NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Classes.Ord NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Constructor NLP.Alphabet.MultiChar.C1_0InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Constructor NLP.Alphabet.MultiChar.C1_0MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Datatype NLP.Alphabet.MultiChar.D1InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Datatype NLP.Alphabet.MultiChar.D1MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Generic NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Generic NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Selector NLP.Alphabet.MultiChar.S1_0_0InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Selector NLP.Alphabet.MultiChar.S1_0_0MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Generics.Selector NLP.Alphabet.MultiChar.S1_0_1InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Read.Read NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Read.Read NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: instance GHC.Show.Show NLP.Alphabet.MultiChar.InternedMultiChar
- NLP.Alphabet.MultiChar: instance GHC.Show.Show NLP.Alphabet.MultiChar.MultiChar
- NLP.Alphabet.MultiChar: internMultiChar :: MultiChar -> MultiChar
- NLP.Alphabet.MultiChar: newtype MultiChar
+ NLP.Scoring.SimpleUnigram: instance Data.Aeson.Types.Class.FromJSON NLP.Scoring.SimpleUnigram.SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance Data.Aeson.Types.Class.ToJSON NLP.Scoring.SimpleUnigram.SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Classes.Eq NLP.Scoring.SimpleUnigram.SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Constructor NLP.Scoring.SimpleUnigram.C1_0SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Datatype NLP.Scoring.SimpleUnigram.D1SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Generic NLP.Scoring.SimpleUnigram.SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_0SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_1SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_2SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_3SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_4SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Generics.Selector NLP.Scoring.SimpleUnigram.S1_0_5SimpleScoring
+ NLP.Scoring.SimpleUnigram: instance GHC.Read.Read NLP.Scoring.SimpleUnigram.SimpleScoring
- NLP.Scoring.SimpleUnigram: SimpleScoring :: !(BasicHashTable (IMMC, IMMC) Double) -> !Double -> !Double -> !Double -> !Double -> !Double -> SimpleScoring
+ NLP.Scoring.SimpleUnigram: SimpleScoring :: !(HashMap (BTI, BTI) Double) -> !Double -> !Double -> !Double -> !Double -> !Double -> SimpleScoring
- NLP.Scoring.SimpleUnigram: [simpleScore] :: SimpleScoring -> !(BasicHashTable (IMMC, IMMC) Double)
+ NLP.Scoring.SimpleUnigram: [simpleScore] :: SimpleScoring -> !(HashMap (BTI, BTI) Double)
- NLP.Scoring.SimpleUnigram: scoreUnigram :: SimpleScoring -> IMMC -> IMMC -> Double
+ NLP.Scoring.SimpleUnigram: scoreUnigram :: SimpleScoring -> BTI -> BTI -> Double
- NLP.Scoring.SimpleUnigram.Import: PLeqset :: Text -> [IMMC] -> ParsedLine
+ NLP.Scoring.SimpleUnigram.Import: PLeqset :: Text -> [BTI] -> ParsedLine
- NLP.Scoring.SimpleUnigram.Import: PLset :: Text -> [IMMC] -> ParsedLine
+ NLP.Scoring.SimpleUnigram.Import: PLset :: Text -> [BTI] -> ParsedLine

Files

− NLP/Alphabet/IMMC.hs
@@ -1,96 +0,0 @@---- | An implementation of @Int@-mapped @MultiChar@s with internalization.--module NLP.Alphabet.IMMC where--import           Control.Applicative-import           Control.DeepSeq (NFData(..))-import           Data.Aeson as A-import           Data.Binary      as DB-import           Data.Hashable-import           Data.Serialize   as DS-import           Data.Serialize.Text-import           Data.Stringable as SA-import           Data.String as IS-import           Data.Text.Binary-import           Data.Vector.Unboxed.Deriving-import           GHC.Generics-import qualified Data.ByteString.Short as BS-import qualified Data.Text as T-import qualified Data.Text.Encoding as T--import           NLP.Alphabet.IMMC.Internal-import           NLP.Alphabet.MultiChar (InternedMultiChar)------ * A somewhat fragile (?) encoding of multichars using the internalized--- @Id@. Should only be used via it's wrapper. The mapped @Int@s are not--- consecutive.--newtype IMMC = IMMC { getIMMC :: Int }-  deriving (Eq,Generic)--derivingUnbox "IMMC"-  [t| IMMC -> Int |]-  [|  getIMMC     |]-  [|  IMMC        |]--instance Ord IMMC where-  IMMC l `compare` IMMC r = immcBimapLookupInt l `compare` immcBimapLookupInt r-  {-# Inline compare #-}--immc :: InternedMultiChar -> IMMC-immc s = IMMC $! immcBimapAdd s-{-# Inline immc #-}--instance IsString IMMC where-  fromString = immc . IS.fromString-  {-# Inline fromString #-}--instance Show IMMC where-  showsPrec p i r = showsPrec p (toString i) r-  {-# Inline showsPrec #-}--instance Read IMMC where-  readsPrec p str = [ (immc $ IS.fromString s, y) | (s,y) <- readsPrec p str ]-  {-# Inline readsPrec #-}--instance Hashable IMMC--instance Stringable IMMC where-  toString   = toString . immcBimapLookupInt . getIMMC-  fromString = immc . SA.fromString-  length     = SA.length . immcBimapLookupInt . getIMMC-  toText     = toText . immcBimapLookupInt . getIMMC-  fromText   = immc . fromText-  {-# Inline toString   #-}-  {-# Inline fromString #-}-  {-# Inline length     #-}-  {-# Inline toText     #-}-  {-# Inline fromText   #-}--instance NFData IMMC where-  rnf = rnf . getIMMC-  {-# Inline rnf #-}--instance Binary IMMC where-  put = DB.put . toText-  get = fromText <$> DB.get-  {-# Inline put #-}-  {-# Inline get #-}--instance Serialize IMMC where-  put = DS.put . toText-  get = fromText <$> DS.get-  {-# Inline put #-}-  {-# Inline get #-}--instance FromJSON IMMC where-  parseJSON s = fromText <$> parseJSON s-  {-# Inline parseJSON #-}--instance ToJSON IMMC where-  toJSON = toJSON . toText-  {-# Inline toJSON #-}-
− NLP/Alphabet/IMMC/Internal.hs
@@ -1,43 +0,0 @@---- | This module keeps a persistent @bimap@ between @InternedMultiChar@s--- and @Int@s------ TODO make this a bimap @Text <-> Vector@. Compare performance when--- printing backtracking results. (Do this after the Builder-based--- backtracking is online)--module NLP.Alphabet.IMMC.Internal where--import Data.IORef (newIORef,IORef,readIORef,atomicWriteIORef,atomicModifyIORef')-import System.IO.Unsafe (unsafePerformIO,unsafeDupablePerformIO)--import Data.Bijection.Hash (Bimap,empty,lookupL,lookupR,size,insert)--import NLP.Alphabet.MultiChar----immcBimap :: IORef (Bimap InternedMultiChar Int)-immcBimap = unsafePerformIO $ newIORef empty-{-# NoInline immcBimap #-}---- | Add @InternedMultiChar@ and return @Int@ key. Will return key for--- existing string and thereby serves for lookup in left-to-right--- direction.--immcBimapAdd :: InternedMultiChar -> Int-immcBimapAdd k = unsafeDupablePerformIO $ atomicModifyIORef' immcBimap $ \m ->-  case lookupL m k of Just i  -> (m,i)-                      Nothing -> let s = size m-                                 in  (insert m (k,s) , s)-{-# Inline immcBimapAdd #-}---- | Lookup the @InternedMultiChar@ based on an @Int@ key. Unsafe totality--- assumption.--immcBimapLookupInt :: Int -> InternedMultiChar-immcBimapLookupInt r = seq r . unsafeDupablePerformIO $ atomicModifyIORef' immcBimap $ \m ->-  case lookupR m r of Just l  -> (m,l)-                      Nothing -> error "immcBimapLookupInt: totality assumption invalidated"-{-# Inline immcBimapLookupInt #-}-
− NLP/Alphabet/MultiChar.hs
@@ -1,140 +0,0 @@---- | An alphabet, where each character is a short piece of @Text@.--module NLP.Alphabet.MultiChar where--import           Control.DeepSeq (NFData(..))-import           Data.Function (on)-import           Data.Hashable-import           Data.Interned-import           Data.Interned.Internal (getCache)-import           Data.Stringable-import           Data.String (IsString)-import qualified Data.ByteString.Short as BS-import qualified Data.ByteString.Short.Internal as BS-import qualified Data.String as S-import qualified Data.Text as T-import qualified Data.Text.Encoding as T-import           GHC.Generics-import qualified Data.HashMap.Strict as HM-import qualified Data.Array as A-import           Data.Typeable (Typeable)-import           Data.Data (Data)------ * 'MultiChar's capture UTF characters that are encoded using one or more--- symbols.---- | Interns a 'MultiChar' character.--internMultiChar :: MultiChar -> MultiChar-internMultiChar = uninternMultiChar . intern-{-# Inline internMultiChar #-}---- | Wrap a short bytestring. Read and Show instances behave like for normal--- strings.--newtype MultiChar = MultiChar { getMultiChar :: T.Text }-  deriving (Eq,Ord,Generic,Data,Typeable)--instance Show MultiChar where-  showsPrec p (MultiChar mc) r = showsPrec p (toString mc) r-  {-# Inline showsPrec #-}--instance Read MultiChar where-  readsPrec p str = [ (MultiChar x, y) | (x,y) <- readsPrec p str ]-  {-# Inline readsPrec #-}--instance Hashable MultiChar--instance IsString MultiChar where-  fromString = MultiChar . S.fromString-  {-# Inline fromString #-}--instance Stringable MultiChar where-  toString   = T.unpack . getMultiChar-  fromString = MultiChar . T.pack-  length     = T.length . getMultiChar-  fromText   = MultiChar-  toText     = getMultiChar-  {-# Inline toString   #-}-  {-# Inline fromString #-}-  {-# Inline length     #-}-  {-# Inline fromText   #-}-  {-# Inline toText     #-}--instance NFData MultiChar where-  rnf = rnf . getMultiChar-  {-# Inline rnf #-}------ * Interned---- | Interned 'MultiChar'.------ TODO Check 'Ord' instance. We @compare `on` uninternMultiChar@.--data InternedMultiChar = InternedMultiChar-  { internedMultiCharId :: {-# UNPACK #-} !Id-  , uninternMultiChar   :: {-# UNPACK #-} !MultiChar-  }-  deriving (Generic,Data,Typeable)--instance IsString InternedMultiChar where-  fromString = intern . S.fromString-  {-# Inline fromString #-}--instance Eq InternedMultiChar where-  (==) = (==) `on` internedMultiCharId-  {-# Inline (==) #-}--instance Ord InternedMultiChar where-  compare = compare `on` uninternMultiChar -- internedMultiCharId-  {-# Inline compare #-}--instance Read InternedMultiChar where-  readsPrec p str = [ (intern x, y) | (x,y) <- readsPrec p str ]-  {-# Inline readsPrec #-}--instance Show InternedMultiChar where-  showsPrec d (InternedMultiChar _ mc) = showsPrec d mc-  {-# Inline showsPrec #-}--instance Hashable InternedMultiChar where-  hashWithSalt salt = hashWithSalt salt . internedMultiCharId-  hash              = hash . internedMultiCharId-  {-# Inline hashWithSalt #-}-  {-# Inline hash         #-}--instance Interned InternedMultiChar where-  type Uninterned InternedMultiChar = MultiChar-  newtype Description InternedMultiChar = DMC MultiChar deriving (Eq,Hashable)-  describe = DMC . MultiChar . T.copy . getMultiChar -- @DMC@ alone is type-correct. With 'T.copy' we make sure not to keep long @Text@s. TODO benchmark!-  identify = InternedMultiChar-  cache    = imcCache-  {-# Inline describe #-}-  {-# Inline identify #-}-  {-# Inline cache    #-}--imcCache :: Cache InternedMultiChar-imcCache = mkCache-{-# NOINLINE imcCache #-}--instance Stringable InternedMultiChar where-  toString   = toString . uninternMultiChar-  fromString = intern . fromString-  length     = Data.Stringable.length . uninternMultiChar-  toText     = toText . uninternMultiChar-  fromText   = intern . fromText-  {-# Inline toString   #-}-  {-# Inline fromString #-}-  {-# Inline length     #-}-  {-# Inline toText     #-}-  {-# Inline fromText   #-}--instance NFData InternedMultiChar where-  rnf (InternedMultiChar i c) = rnf i `seq` rnf c-  {-# Inline rnf #-}-
NLP/Scoring/SimpleUnigram.hs view
@@ -3,35 +3,54 @@  module NLP.Scoring.SimpleUnigram where -import           Data.HashTable.IO (BasicHashTable)-import qualified Data.HashTable.IO as H-import           System.IO.Unsafe (unsafePerformIO)+import GHC.Generics+import Data.HashMap.Strict+import Data.Aeson -import           NLP.Alphabet.IMMC+import NLP.Text.BTI   --- | Score 'MultiChar's @x@ and @y@ based on the simple scoring system: (i)+-- | Score 'BTI's @x@ and @y@ based on the simple scoring system: (i) -- lookup (x,y) and use the score if found; (ii) if (x,y) is not in the -- database, then return the default matching 'defMatch' score if @x==y@, -- otherwise return the default mismatch 'defMismatch' score. -scoreUnigram :: SimpleScoring -> IMMC -> IMMC -> Double+scoreUnigram :: SimpleScoring -> BTI -> BTI -> Double scoreUnigram SimpleScoring {..} x y =-  maybe (if x==y then defMatch else defMismatch)-  id-  (unsafePerformIO $ H.lookup simpleScore (x,y))+  lookupDefault (if x==y then defMatch else defMismatch) (x,y) simpleScore {-# INLINE scoreUnigram #-}  -- | Collect the hashtable and scalar values for simple scoring.+--+-- TODO binary and cereal instances  data SimpleScoring = SimpleScoring-  { simpleScore  :: !(BasicHashTable (IMMC,IMMC) Double)+  { simpleScore  :: !(HashMap (BTI,BTI) Double)   , gapScore     :: !Double   , gapOpen      :: !Double   , gapExtend    :: !Double   , defMatch     :: !Double   , defMismatch  :: !Double   }-  deriving (Show)+  deriving (Read,Show,Eq,Generic)++instance FromJSON SimpleScoring where+  parseJSON (Object v) = SimpleScoring <$>+                          (fromList `fmap` (v .: "simpleScore")) <*>+                          v .: "gapScore"    <*>+                          v .: "gapOpen"     <*>+                          v .: "gapExtend"   <*>+                          v .: "defMatch"    <*>+                          v .: "defMismatch"++instance ToJSON SimpleScoring where+  toJSON (SimpleScoring ss gs go ge dm di)+    = object [ "simpleScore" .= toList ss+             , "gapScore"    .= gs+             , "gapOpen"     .= go+             , "gapExtend"   .= ge+             , "defMatch"    .= dm+             , "defMismatch" .= di+             ] 
NLP/Scoring/SimpleUnigram/Import.hs view
@@ -2,21 +2,15 @@ module NLP.Scoring.SimpleUnigram.Import where  import           Control.Applicative---import           Data.ByteString.Char8 (ByteString)-import           Data.HashTable.IO (BasicHashTable)+import           Data.HashMap.Strict (fromList) import           Data.Stringable---import qualified Data.Attoparsec.ByteString as AB---import qualified Data.Attoparsec.ByteString.Char8 as AB hiding (takeWhile1,skipWhile)---import qualified Data.ByteString.Char8 as B-import qualified Data.HashTable.IO as H-import           System.IO.Unsafe (unsafePerformIO)--import qualified Data.Attoparsec.Text as AT import           Data.Text (Text)+import qualified Data.Attoparsec.Text as AT import qualified Data.Text as T import qualified Data.Text.IO as T -import           NLP.Alphabet.IMMC+import           NLP.Text.BTI+ import           NLP.Scoring.SimpleUnigram  @@ -28,9 +22,9 @@ -- circular imports)  data ParsedLine-  = PLset Text [IMMC]+  = PLset Text [BTI]   | PLeq Text Double-  | PLeqset Text [IMMC]+  | PLeqset Text [BTI]   | PLinset Text Text Double   | PLgap Double   | PLgapopen Double@@ -68,7 +62,7 @@ genSimpleScoring :: Text -> SimpleScoring genSimpleScoring l = SimpleScoring t g go ge dm di   where-    t    = unsafePerformIO $ H.fromListWithSizeHint (Prelude.length ys) ys+    t    = fromList ys     ls   = T.lines l     xs   = map parseLine ls     ys   = concatMap genPairs $ iss ++ eqs
NaturalLanguageAlphabets.cabal view
@@ -1,5 +1,5 @@ name:           NaturalLanguageAlphabets-version:        0.0.2.0+version:        0.1.0.0 author:         Christian Hoener zu Siederdissen maintainer:     choener@bioinf.uni-leipzig.de homepage:       https://github.com/choener/NaturalLanguageAlphabets@@ -12,19 +12,9 @@ stability:      experimental cabal-version:  >= 1.10.0 tested-with:    GHC == 7.8.4, GHC == 7.10.2-synopsis:       Alphabet and word representations+synopsis:       Simple scoring schemes for word alignments description:-                Provides different encoding for characters and words in natural-                language processing. A character will often be encoded as a-                unicode text string as we deal with multi-symbol characters.-                .-                Internal encoding of IMMC symbols are 0-based integers, which-                allows for the use of unboxed containers.-                .-                A very simple unigram-based scoring scheme and DSL to write-                such schemes are also provided.-                .-                <https://github.com/choener/NaturalLanguageAlphabets/blob/master/README.md>+                Provides a simple scoring scheme for word alignments.   @@ -35,13 +25,6 @@   -flag llvm-  description:  build the benchmark using LLVM-  default:      False-  manual:       True--- library   build-depends: base                   >  4.7      && < 4.9                , aeson                  >= 0.8      && < 0.11@@ -55,8 +38,8 @@                , deepseq                >= 1.3      && < 1.5                , file-embed             >= 0.0.6    && < 0.0.10                , hashable               >= 1.2      && < 1.3-               , hashtables             >= 1.1      && < 1.3                , intern                 >= 0.9      && < 0.10+               , LinguisticsTypes       >= 0.0.0    && < 0.0.1                , QuickCheck             >= 2.7      && < 2.9                , stringable             >= 0.1.2    && < 0.2                , system-filepath        >= 0.4.9    && < 0.5@@ -67,9 +50,6 @@                , vector-th-unbox        >= 0.2      && < 0.3    exposed-modules:-    NLP.Alphabet.IMMC-    NLP.Alphabet.IMMC.Internal-    NLP.Alphabet.MultiChar     NLP.Scoring.SimpleUnigram     NLP.Scoring.SimpleUnigram.Default     NLP.Scoring.SimpleUnigram.Import@@ -98,6 +78,7 @@                , criterion                >= 1.0.2  && < 1.1.1                , deepseq                , hashtables+               , LinguisticsTypes                , mwc-random               >= 0.13   && < 0.14                , NaturalLanguageAlphabets                , random                   >= 1.0    && < 1.2@@ -119,11 +100,6 @@     -funbox-strict-fields     -funfolding-use-threshold1000     -funfolding-keeness-factor1000-  if flag(llvm)-    ghc-options:-      -fllvm-      -optlo-O3 -optlo-std-compile-opts-      -fllvm-tbaa   @@ -144,6 +120,7 @@                , aeson                , binary                , cereal+               , LinguisticsTypes                , NaturalLanguageAlphabets                , QuickCheck                , stringable@@ -151,6 +128,7 @@                , test-framework-quickcheck2   >= 0.3  && < 0.4                , test-framework-th            >= 0.2  && < 0.3                , text+               , unordered-containers   
README.md view
@@ -2,11 +2,13 @@  # Natural Language Alphabets -Efficient, alphabet symbols. The symbols are interned, and hashed. This is-quite useful for k-gram scoring, where we have different sets of symbols with-different scores. IMMC symbols are internally represented via Ints in the range-[0..]. This makes it possible to use unboxed containers when handling IMMC-symbols.+This library provides a simple scoring method for alignment of words with+natural language alphabets.  The underlying /character/ type is BTI from the+LinguisticsTypes package, which provides efficient encoding of arbitrarily+complex atomic characters.++The actual alignment algorithms can be found in the+[WordAlignment](http://hackage.haskell.org/package/WordAlignment) package.   
changelog.md view
@@ -1,3 +1,12 @@+0.1.0.0+-------++- moved IMMC to LinguisticsTypes library and renamed to BTI+- removed MultiChar type+- using HashMap from unordered-containers instead of HashTable from hashtables.+  Now we do not have to unsafePerformIO anymore.+- JSON (de)serialization for SimpleScoring scheme+ 0.0.2.0 ------- 
tests/Benchmark.hs view
@@ -7,24 +7,24 @@  module Main where -import           Criterion.Main-import qualified Data.Vector.Unboxed as VU-import qualified Data.Vector.Generic as VG-import qualified Data.Vector as VV-import           Text.Printf-import           Data.Tuple (swap) import           Control.Applicative ((<$>))-import           System.Random.MWC-import           System.Random import           Control.DeepSeq import           Control.Monad-import qualified Data.HashMap.Strict as UCHS-import qualified Data.IntMap.Strict  as CIS+import           Criterion.Main import           Data.String+import           Data.Tuple (swap)+import qualified Data.HashMap.Strict as UCHS import qualified Data.HashTable.IO as HT+import qualified Data.IntMap.Strict  as CIS+import qualified Data.Vector as VV+import qualified Data.Vector.Generic as VG+import qualified Data.Vector.Unboxed as VU import           System.IO.Unsafe (unsafePerformIO)+import           System.Random+import           System.Random.MWC+import           Text.Printf -import NLP.Alphabet.IMMC+import NLP.Text.BTI   @@ -36,18 +36,18 @@                     (UCHS.size hmsK) (CIS.size cisK) (-1 :: Int)           defaultMain             [ bgroup "  100 keys" [ bench "HashMap.Strict" $ whnf (\k -> UCHS.lookupDefault  0 k hmsH) (VU.head keys)-                                  , bench "IntMap.Strict" $ whnf (\k -> CIS.findWithDefault 0 k cisH) (getIMMC $ VU.head keys)+                                  , bench "IntMap.Strict" $ whnf (\k -> CIS.findWithDefault 0 k cisH) (getBTI $ VU.head keys)                                   , bench "HashTable" $ whnf (\k -> htLookup htH k)               (VU.head keys)                                   ]             , bgroup "10000 keys" [ bench "HashMap.Strict" $ whnf (\k -> UCHS.lookupDefault  0 k hmsK) (VU.head keys)-                                  , bench "IntMap.Strict" $ whnf (\k -> CIS.findWithDefault 0 k cisK) (getIMMC $ VU.head keys)+                                  , bench "IntMap.Strict" $ whnf (\k -> CIS.findWithDefault 0 k cisK) (getBTI $ VU.head keys)                                   , bench "HashTable" $ whnf (\k -> htLookup htK k)               (VU.head keys)                                   ] --            , bgroup "  100 known"   [ bench "uchsH" $ whnf (\ks -> VU.sum $ VU.map (\k -> UCHS.lookupDefault  0 k hmsH) ks) (VU.take 100 keys)---                                     , bench " cisH" $ whnf (\ks -> VU.sum $ VU.map (\k -> CIS.findWithDefault 0 k cisH) ks) (VU.map getIMMC $ VU.take 100 keys)+--                                     , bench " cisH" $ whnf (\ks -> VU.sum $ VU.map (\k -> CIS.findWithDefault 0 k cisH) ks) (VU.map getBTI $ VU.take 100 keys) --                                     ] --            , bgroup "    1 unknown" [ bench "uchsH" $ whnf (\k -> UCHS.lookupDefault  0 k hmsH) (VU.head unks)---                                     , bench " cisH" $ whnf (\k -> CIS.findWithDefault 0 k cisH) (getIMMC $ VU.head unks)+--                                     , bench " cisH" $ whnf (\k -> CIS.findWithDefault 0 k cisH) (getBTI $ VU.head unks) --                                     ]             ] @@ -59,25 +59,25 @@ {-# Inline htLookup #-}  -type HTB = HT.BasicHashTable IMMC Double+type HTB = HT.BasicHashTable BTI Double  setupEnv = do   -- create keys   strs :: [String] <- replicateM 10000 rString   -- scores to look up   scrs :: [Double] <- replicateM 10000 $ randomRIO (0 , 9)-  -- create IMMC keys-  let keys = map (immc . fromString) strs+  -- create BTI keys+  let keys = map (bti . fromString) strs   -- create random keys, mostly not in strs   unks :: [String] <- replicateM 10000 rString-  let sknu = map (immc . fromString) unks+  let sknu = map (bti . fromString) unks   -- for 100 keys-  let hmsK = UCHS.fromList $ take 100 $ zip keys               scrs-  let cisK =  CIS.fromList $ take 100 $ zip (map getIMMC keys) scrs+  let hmsK = UCHS.fromList $ take 100 $ zip keys              scrs+  let cisK =  CIS.fromList $ take 100 $ zip (map getBTI keys) scrs   htH :: HTB <- HT.fromList $ take 100 $ zip keys scrs   -- for 10 000 keys-  let hmsM = UCHS.fromList $ zip keys               scrs-  let cisM =  CIS.fromList $ zip (map getIMMC keys) scrs+  let hmsM = UCHS.fromList $ zip keys              scrs+  let cisM =  CIS.fromList $ zip (map getBTI keys) scrs   htK :: HTB <- HT.fromList $ zip keys scrs   return (VU.fromList keys , VU.fromList sknu , (hmsK,cisK,htH) , (hmsM,cisM,htK) ) 
tests/properties.hs view
@@ -2,46 +2,44 @@ module Main where  import           Control.Applicative-import           Data.String-import           Data.Stringable hiding (fromString)+import           Data.Stringable import           Debug.Trace import qualified Data.Aeson as A import qualified Data.Binary as B import qualified Data.Serialize as S import           Test.Framework.Providers.QuickCheck2 import           Test.Framework.TH+import           Data.HashMap.Strict (fromList,union) -import           NLP.Alphabet.IMMC+import           NLP.Text.BTI +import           NLP.Scoring.SimpleUnigram+import           NLP.Scoring.SimpleUnigram.Default  --- * IMMC --- basic property of interning+-- | Test aeson conversion. We add random @(key,value)@ pairs in the form+-- of @xs@ to the default scoring. We also randomize the individual scoring+-- constants.+--+-- Testing is done by serialization followed by deserialization and testing+-- for equality. -prop_IMMC (t :: String)-  | t == u    = True-  | otherwise = traceShow (t, getIMMC i, u) False-  where i :: IMMC = fromString t-        u         = toString   i+prop_Aeson ( xs :: [((String,String),Double)]+           , (gs :: Double, go :: Double, ge :: Double, dm :: Double, di :: Double)+           )+  = Just def' == A.decode (A.encode def')+  where def  = clvDefaults+        xs'  = fromList $ map (\((x,y),s) -> ((fromString x,fromString y),s)) xs+        def' = def { simpleScore  = simpleScore def `union` xs'+                   , gapScore     = gs+                   , gapOpen      = go+                   , gapExtend    = ge+                   , defMatch     = dm+                   , defMismatch  = di+                   } --- binary -prop_Binary (t :: String) = t == toString j-  where i :: IMMC = fromString t-        j :: IMMC = B.decode $ B.encode i---- cereal--prop_Serialize (t :: String) = Right t == (toString <$> j)-  where i ::               IMMC = fromString t-        j :: Either String IMMC = S.decode $ S.encode i---- aeson (more complicated to due the json format!--prop_Aeson (t :: String) = Just [t] == (map toString <$> j)-  where i ::       [IMMC] = [fromString t]-        j :: Maybe [IMMC] = A.decode $ A.encode i  main :: IO () main = $(defaultMainGenerator)