ngrams-loader (empty) → 0.1.0.0
raw patch · 10 files changed
+670/−0 lines, 10 filesdep +attoparsecdep +basedep +machinessetup-changed
Dependencies added: attoparsec, base, machines, mtl, ngrams-loader, parseargs, resourcet, sqlite-simple, text
Files
- LICENSE +21/−0
- Main.hs +103/−0
- README.md +107/−0
- Setup.hs +2/−0
- lib/Data/Ngrams.hs +43/−0
- lib/Data/Ngrams/Database/Sqlite.hs +134/−0
- lib/Data/Ngrams/Parser.hs +65/−0
- lib/Data/Ngrams/Process.hs +96/−0
- lib/Data/Ngrams/Type.hs +50/−0
- ngrams-loader.cabal +49/−0
+ LICENSE view
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2014 Yorick Laupa++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.
+ Main.hs view
@@ -0,0 +1,103 @@+-----------------------------------------------------------------------------+-- |+-- Module : Main+-- Copyright : (C) 2014 Yorick Laupa+-- License : (see the file LICENSE)+--+-- Maintainer : Yorick Laupa <yo.eight@gmail.com>+-- Stability : provisional+-- Portability : non-portable+--+----------------------------------------------------------------------------+module Main where++----------------------------------------------------------------------------+import Data.List (find)++----------------------------------------------------------------------------+import System.Console.ParseArgs++----------------------------------------------------------------------------+import Data.Ngrams++----------------------------------------------------------------------------+-- | Command-line Option type+data Option = Bi+ | Tri+ | Quad+ | Penta+ | Create+ | File+ | DB deriving (Ord, Eq, Show, Enum)++----------------------------------------------------------------------------+main :: IO ()+main = do+ as <- parseArgsIO ArgsComplete desc+ let nType = parserType as+ file = getRequiredArg as File+ db = getRequiredArg as DB+ create = gotArg as Create+ mk = mkProcess file db create++ ngram Bi = mk parserBigram bigramCommand+ ngram Tri = mk parserTrigram trigramCommand+ ngram Quad = mk parserQuadgram quadgramCommand+ ngram Penta = mk parserPentagram pentagramCommand+ ngram _ = error "The impossible happens"++ case nType of+ Nothing -> print "Please toggled one of N-grams type"+ Just p -> runProcess_ $ ngram p++----------------------------------------------------------------------------+-- | Command-line Options parser description+desc :: [Arg Option]+desc = [ Arg { argIndex = Bi+ , argAbbr = Just '2'+ , argName = Just "bigram"+ , argData = Nothing+ , argDesc = "Parses bigrams"+ }+ , Arg { argIndex = Tri+ , argAbbr = Just '3'+ , argName = Just "trigram"+ , argData = Nothing+ , argDesc = "Parses trigrams"+ }+ , Arg { argIndex = Quad+ , argAbbr = Just '4'+ , argName = Just "quadgram"+ , argData = Nothing+ , argDesc = "Parses 4-grams"+ }+ , Arg { argIndex = Penta+ , argAbbr = Just '5'+ , argName = Just "pentagram"+ , argData = Nothing+ , argDesc = "Parses 5-grams"+ }+ , Arg { argIndex = Create+ , argAbbr = Just 'c'+ , argName = Just "create"+ , argData = Nothing+ , argDesc = "Creates table before inserts"+ }+ , Arg { argIndex = File+ , argAbbr = Nothing+ , argName = Nothing+ , argData = argDataRequired "n-grams file" ArgtypeString+ , argDesc = "N-grams file"+ }+ , Arg { argIndex = DB+ , argAbbr = Nothing+ , argName = Nothing+ , argData = argDataRequired "SQLite file" ArgtypeString+ , argDesc = "SQlite db file"+ }+ ]++----------------------------------------------------------------------------+-- | Gets toggled Ngrams flag+parserType :: Args Option -> Maybe Option+parserType as = find (gotArg as) [Bi .. Penta]
+ README.md view
@@ -0,0 +1,107 @@+ngrams-loader+=============++Ngrams loader based on http://www.ngrams.info format++Installation+------------+Supposed you have at least `cabal 1.18` installed++```+$ cabal sandbox init+$ cabal install --only-dependencies+$ cabal configure+$ cabal install++-- program located in ~/.cabal-sandbox/bin+```++Usage+-----++```+usage: ngrams-loader [options] <n-grams file> <SQLite file>+ [-2,--bigram] Parses bigrams+ [-3,--trigram] Parses trigrams+ [-4,--quadgram] Parses 4-grams+ [-5,--pentagram] Parses 5-grams+ [-c,--create] Creates table before inserts+ <n-grams file> N-grams file+ <SQLite file> SQlite db file+```++Example+-------++```+ngrams-loader --bigram --create w2.txt bigram.db++```+It parses each line of `w2.txt` as a bigram, create bigram table before performing inserts and saves everything in `bigram.db`++Figures+-------++Specs++- Core i7 3770 @ 3.4GHz+- Gentoo with 3.12.13 Linux kernel (64bits)+- 1.055.386 lines bigram file+ +`ngrams-loader --bigram --create w2.txt bigram.db` gets++```+real 0m16.244s+user 0m15.597s+sys 0m0.143s++```++Sql Schemas+-----------++Bigram++```sql+create table bigrams(+ frequence int,+ word1 varchar(100),+ word2 varchar(100)+);+```++Trigram++```sql+create table tridgrams(+ frequence int,+ word1 varchar(100),+ word2 varchar(100),+ word3 varchar(100)+);+```++4-gram++```sql+create table quadgrams(+ frequence int,+ word1 varchar(100),+ word2 varchar(100),+ word3 varchar(100),+ word4 varchar(100)+);+```++5-gram++```sql+create table pentagrams(+ frequence int,+ word1 varchar(100),+ word2 varchar(100),+ word3 varchar(100),+ word4 varchar(100),+ word5 varchar(100)+);+```
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ lib/Data/Ngrams.hs view
@@ -0,0 +1,43 @@+-----------------------------------------------------------------------------+-- |+-- Module : Data.Ngrams+-- Copyright : (C) 2014 Yorick Laupa+-- License : (see the file LICENSE)+--+-- Maintainer : Yorick Laupa <yo.eight@gmail.com>+-- Stability : provisional+-- Portability : non-portable+--+----------------------------------------------------------------------------+module Data.Ngrams+ ( module Data.Ngrams.Database.Sqlite+ , module Data.Ngrams.Parser+ , module Data.Ngrams.Process+ , mkProcess+ , runProcess_+ ) where++----------------------------------------------------------------------------+import Control.Monad.Trans.Resource (ResourceT, runResourceT)+import Data.Attoparsec.Text (Parser)+import Data.Machine (ProcessT, runT_, (~>))+import Data.Text (Text)++----------------------------------------------------------------------------+import Data.Ngrams.Database.Sqlite+import Data.Ngrams.Parser+import Data.Ngrams.Process++----------------------------------------------------------------------------+mkProcess :: FilePath+ -> FilePath+ -> Bool+ -> Parser a+ -> Command a+ -> ProcessT (ResourceT IO) Text ()+mkProcess file db create parser cmd =+ sourceLines file ~> parseLine parser ~> saveDB create db cmd++----------------------------------------------------------------------------+runProcess_ :: ProcessT (ResourceT IO) Text a -> IO ()+runProcess_ = runResourceT . runT_
+ lib/Data/Ngrams/Database/Sqlite.hs view
@@ -0,0 +1,134 @@+{-# LANGUAGE OverloadedStrings #-}+-----------------------------------------------------------------------------+-- |+-- Module : Data.Ngrams.Database.Sqlite+-- Copyright : (C) 2014 Yorick Laupa+-- License : (see the file LICENSE)+--+-- Maintainer : Yorick Laupa <yo.eight@gmail.com>+-- Stability : provisional+-- Portability : non-portable+--+----------------------------------------------------------------------------+module Data.Ngrams.Database.Sqlite+ ( Command+ , cmdExec+ , cmdCreateTable+ , bigramCommand+ , trigramCommand+ , quadgramCommand+ , pentagramCommand+ ) where++----------------------------------------------------------------------------+import Database.SQLite.Simple++----------------------------------------------------------------------------+import Data.Ngrams.Type++----------------------------------------------------------------------------+data Command a =+ Command+ { cmdCreateTable :: Query+ , cmdExec :: a -> Connection -> IO ()+ }++----------------------------------------------------------------------------+bigramCommand :: Command Bigram+bigramCommand = Command bigramCreateTable bigramEx++----------------------------------------------------------------------------+trigramCommand :: Command Trigram+trigramCommand = Command trigramCreateTable trigramEx++----------------------------------------------------------------------------+quadgramCommand :: Command Quadgram+quadgramCommand = Command quadgramCreateTable quadgramEx++----------------------------------------------------------------------------+pentagramCommand :: Command Pentagram+pentagramCommand = Command pentagramCreateTable pentagramEx++----------------------------------------------------------------------------+bigramCreateTable :: Query+bigramCreateTable =+ "create table bigrams(\+ \ frequence int,\+ \ word1 varchar(100),\+ \ word2 varchar(100)\+ \);"++----------------------------------------------------------------------------+trigramCreateTable :: Query+trigramCreateTable =+ "create table trigrams(\+ \ frequence int,\+ \ word1 varchar(100),\+ \ word2 varchar(100),\+ \ word3 varchar(100)\+ \);"++----------------------------------------------------------------------------+quadgramCreateTable :: Query+quadgramCreateTable =+ "create table quadgrams(\+ \ frequence int,\+ \ word1 varchar(100),\+ \ word2 varchar(100),\+ \ word3 varchar(100),\+ \ word4 varchar(100)\+ \);"++----------------------------------------------------------------------------+pentagramCreateTable :: Query+pentagramCreateTable =+ "create table pentagrams(\+ \ frequence int,\+ \ word1 varchar(100),\+ \ word2 varchar(100),\+ \ word3 varchar(100),\+ \ word4 varchar(100),\+ \ word5 varchar(100)\+ \);"++----------------------------------------------------------------------------+bigramInsert :: Query+bigramInsert =+ "insert into bigrams (frequence, word1, word2) values (?,?,?);"++----------------------------------------------------------------------------+trigramInsert :: Query+trigramInsert =+ "insert into trigrams (frequence, word1, word2, word3) values (?,?,?,?);"++----------------------------------------------------------------------------+quadgramInsert :: Query+quadgramInsert =+ "insert into quadgrams (frequence, word1, word2, word3, word4) values \+ \(?,?,?,?,?);"++----------------------------------------------------------------------------+pentagramInsert :: Query+pentagramInsert =+ "insert into pentagrams (frequence, word1, word2, word3, word4, word5) \+ \values (?,?,?,?,?);"++----------------------------------------------------------------------------+bigramEx:: Bigram -> Connection -> IO ()+bigramEx (Bigram freq w1 w2) con =+ execute con bigramInsert (freq, w1, w2)++----------------------------------------------------------------------------+trigramEx :: Trigram -> Connection -> IO ()+trigramEx (Trigram freq w1 w2 w3) con =+ execute con trigramInsert (freq, w1, w2, w3)++----------------------------------------------------------------------------+quadgramEx :: Quadgram -> Connection -> IO ()+quadgramEx (Quadgram freq w1 w2 w3 w4) con =+ execute con quadgramInsert (freq, w1, w2, w3, w4)++----------------------------------------------------------------------------+pentagramEx :: Pentagram -> Connection -> IO ()+pentagramEx (Pentagram freq w1 w2 w3 w4 w5) con =+ execute con pentagramInsert (freq, w1, w2, w3, w4, w5)
+ lib/Data/Ngrams/Parser.hs view
@@ -0,0 +1,65 @@+-----------------------------------------------------------------------------+-- |+-- Module : Data.Ngrams.Parser+-- Copyright : (C) 2014 Yorick Laupa+-- License : (see the file LICENSE)+--+-- Maintainer : Yorick Laupa <yo.eight@gmail.com>+-- Stability : provisional+-- Portability : non-portable+--+----------------------------------------------------------------------------+module Data.Ngrams.Parser+ ( parserBigram+ , parserTrigram+ , parserQuadgram+ , parserPentagram+ ) where++----------------------------------------------------------------------------+import Data.Char (isSpace)++----------------------------------------------------------------------------+import Data.Attoparsec.Text (Parser, decimal, skipSpace, takeWhile1)+import Data.Text (Text)++----------------------------------------------------------------------------+import Data.Ngrams.Type++----------------------------------------------------------------------------+parserBigram :: Parser Bigram+parserBigram = do+ freq <- decimal+ skipSpace+ w1 <- _word+ skipSpace+ w2 <- _word+ return $ Bigram freq w1 w2++----------------------------------------------------------------------------+parserTrigram :: Parser Trigram+parserTrigram = do+ Bigram freq w1 w2 <- parserBigram+ skipSpace+ w3 <- _word+ return $ Trigram freq w1 w2 w3++----------------------------------------------------------------------------+parserQuadgram :: Parser Quadgram+parserQuadgram = do+ Trigram freq w1 w2 w3 <- parserTrigram+ skipSpace+ w4 <- _word+ return $ Quadgram freq w1 w2 w3 w4++----------------------------------------------------------------------------+parserPentagram :: Parser Pentagram+parserPentagram = do+ Quadgram freq w1 w2 w3 w4 <- parserQuadgram+ skipSpace+ w5 <- _word+ return $ Pentagram freq w1 w2 w3 w4 w5++----------------------------------------------------------------------------+_word :: Parser Text+_word = takeWhile1 (not . isSpace)
+ lib/Data/Ngrams/Process.hs view
@@ -0,0 +1,96 @@+{-# LANGUAGE DeriveDataTypeable #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RankNTypes #-}+-----------------------------------------------------------------------------+-- |+-- Module : Data.Ngrams.Process+-- Copyright : (C) 2014 Yorick Laupa+-- License : (see the file LICENSE)+--+-- Maintainer : Yorick Laupa <yo.eight@gmail.com>+-- Stability : provisional+-- Portability : non-portable+--+----------------------------------------------------------------------------+module Data.Ngrams.Process where++----------------------------------------------------------------------------+import Control.Applicative ((<|>))+import Control.Monad (when)+import Control.Exception+import Data.Typeable+import System.IO (IOMode(..), hClose, hIsEOF, openFile)++----------------------------------------------------------------------------+import Data.Attoparsec.Text (Parser, parseOnly)+import Database.SQLite.Simple+import Control.Monad.Trans (lift, liftIO)+import Control.Monad.Trans.Resource+import Data.Machine+import qualified Data.Text as T+import qualified Data.Text.IO as T++----------------------------------------------------------------------------+import Data.Ngrams.Database.Sqlite++----------------------------------------------------------------------------+data ParseException = ParseException !String deriving (Show, Typeable)++instance Exception ParseException++----------------------------------------------------------------------------+-- | Creates a `Source` from the lines of a file, using the `ResourceT`+-- monad to ensure the file is closed when processing the stream of lines+-- is finished+sourceLines :: FilePath -> SourceT (ResourceT IO) T.Text+sourceLines path = construct $ do+ (key, h) <- lift $ allocate (openFile path ReadMode) hClose+ go key h+ where+ go key h =+ let readLine = lift $ liftIO $ T.hGetLine h+ isEOF = lift $ liftIO $ hIsEOF h++ closing = do+ lift $ release key+ stop++ yielding = do+ line <- readLine+ yield line+ go key h in++ isEOF >>= \eof -> if eof then closing else yielding++----------------------------------------------------------------------------+-- | Applies a `Parser` on each line. If an error occurs, an Exception is+-- raised+parseLine :: MonadThrow m => Parser a -> ProcessT m T.Text a+parseLine p = repeatedly $ do+ line <- await+ case parseOnly p line of+ Left e -> lift $ monadThrow $ ParseException e+ Right a -> yield a++----------------------------------------------------------------------------+-- | Saves each entry in a SQLite database with submitted `Command`+saveDB :: Bool -> FilePath -> Command a -> ProcessT (ResourceT IO) a ()+saveDB create path c = construct $ do+ (ckey, con) <- lift $ allocate (open path) close+ when create $ liftIO $ execute_ con createTable+ lift $ liftIO $ execute_ con "begin"+ loop ckey con+ where+ exec = cmdExec c+ createTable = cmdCreateTable c++ loop ckey con =+ let closing = do+ lift $ liftIO $ do+ execute_ con "end"+ release ckey+ stop in++ do a <- await <|> closing+ lift $ liftIO $ exec a con+ loop ckey con
+ lib/Data/Ngrams/Type.hs view
@@ -0,0 +1,50 @@+-----------------------------------------------------------------------------+-- |+-- Module : Data.Ngrams.Type+-- Copyright : (C) 2014 Yorick Laupa+-- License : (see the file LICENSE)+--+-- Maintainer : Yorick Laupa <yo.eight@gmail.com>+-- Stability : provisional+-- Portability : non-portable+--+----------------------------------------------------------------------------+module Data.Ngrams.Type where++----------------------------------------------------------------------------+import Data.Text (Text)++----------------------------------------------------------------------------+data Bigram =+ Bigram+ { biFreq :: !Integer+ , biWord1 :: !Text+ , biWord2 :: !Text+ }++data Trigram =+ Trigram+ { triFreq :: !Integer+ , triWord1 :: !Text+ , triWord2 :: !Text+ , triWord3 :: !Text+ }++data Quadgram =+ Quadgram+ { quadFreq :: !Integer+ , quadWord1 :: !Text+ , quadWord2 :: !Text+ , quadWord3 :: !Text+ , quadWord4 :: !Text+ }++data Pentagram =+ Pentagram+ { pentaFreq :: !Integer+ , pentaWord1 :: !Text+ , pentaWord2 :: !Text+ , pentaWord3 :: !Text+ , pentaWord4 :: !Text+ , pentaWord5 :: !Text+ }
+ ngrams-loader.cabal view
@@ -0,0 +1,49 @@+name: ngrams-loader+version: 0.1.0.0+synopsis: Ngrams loader based on http://www.ngrams.info format+description: Ngrams loader based on http://www.ngrams.info format+license: MIT+license-file: LICENSE+author: Yorick Laupa+maintainer: Yorick Laupa <yo.eight@gmail.com>+homepage: http://github.com/YoEight/ngrams-loader+bug-reports: https://github.com/YoEight/ngrams-loader/issues+copyright: Copyright (C) 2014 Yorick Laupa++category: Data+build-type: Simple+extra-source-files: README.md+cabal-version: >=1.10+tested-with: GHC == 7.6.3++source-repository head+ type: git+ location: git://github.com/YoEight/ngrams-loader.git++library+ hs-source-dirs: lib+ build-depends: base >=4.6 && <4.7+ , attoparsec >=0.11.1 && <0.11.2+ , machines >= 0.2.5 && <0.3+ , mtl+ , resourcet >=0.4.3 && <0.5+ , sqlite-simple >= 0.4.5 && <0.5+ , text >=0.11 && <1.2++ exposed-modules: Data.Ngrams+ Data.Ngrams.Database.Sqlite+ other-modules: Data.Ngrams.Process+ Data.Ngrams.Parser+ Data.Ngrams.Type++ ghc-options: -Wall -O2+ default-language: Haskell2010++executable ngrams-loader+ main-is: Main.hs+ build-depends: base+ , ngrams-loader+ , parseargs ==0.1.5.*++ ghc-options: -Wall -O2+ default-language: Haskell2010