diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Yorick Laupa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Main.hs b/Main.hs new file mode 100644 --- /dev/null +++ b/Main.hs @@ -0,0 +1,103 @@ +----------------------------------------------------------------------------- +-- | +-- Module : Main +-- Copyright : (C) 2014 Yorick Laupa +-- License : (see the file LICENSE) +-- +-- Maintainer : Yorick Laupa +-- Stability : provisional +-- Portability : non-portable +-- +---------------------------------------------------------------------------- +module Main where + +---------------------------------------------------------------------------- +import Data.List (find) + +---------------------------------------------------------------------------- +import System.Console.ParseArgs + +---------------------------------------------------------------------------- +import Data.Ngrams + +---------------------------------------------------------------------------- +-- | Command-line Option type +data Option = Bi + | Tri + | Quad + | Penta + | Create + | File + | DB deriving (Ord, Eq, Show, Enum) + +---------------------------------------------------------------------------- +main :: IO () +main = do + as <- parseArgsIO ArgsComplete desc + let nType = parserType as + file = getRequiredArg as File + db = getRequiredArg as DB + create = gotArg as Create + mk = mkProcess file db create + + ngram Bi = mk parserBigram bigramCommand + ngram Tri = mk parserTrigram trigramCommand + ngram Quad = mk parserQuadgram quadgramCommand + ngram Penta = mk parserPentagram pentagramCommand + ngram _ = error "The impossible happens" + + case nType of + Nothing -> print "Please toggled one of N-grams type" + Just p -> runProcess_ $ ngram p + +---------------------------------------------------------------------------- +-- | Command-line Options parser description +desc :: [Arg Option] +desc = [ Arg { argIndex = Bi + , argAbbr = Just '2' + , argName = Just "bigram" + , argData = Nothing + , argDesc = "Parses bigrams" + } + , Arg { argIndex = Tri + , argAbbr = Just '3' + , argName = Just "trigram" + , argData = Nothing + , argDesc = "Parses trigrams" + } + , Arg { argIndex = Quad + , argAbbr = Just '4' + , argName = Just "quadgram" + , argData = Nothing + , argDesc = "Parses 4-grams" + } + , Arg { argIndex = Penta + , argAbbr = Just '5' + , argName = Just "pentagram" + , argData = Nothing + , argDesc = "Parses 5-grams" + } + , Arg { argIndex = Create + , argAbbr = Just 'c' + , argName = Just "create" + , argData = Nothing + , argDesc = "Creates table before inserts" + } + , Arg { argIndex = File + , argAbbr = Nothing + , argName = Nothing + , argData = argDataRequired "n-grams file" ArgtypeString + , argDesc = "N-grams file" + } + , Arg { argIndex = DB + , argAbbr = Nothing + , argName = Nothing + , argData = argDataRequired "SQLite file" ArgtypeString + , argDesc = "SQlite db file" + } + ] + +---------------------------------------------------------------------------- +-- | Gets toggled Ngrams flag +parserType :: Args Option -> Maybe Option +parserType as = find (gotArg as) [Bi .. Penta] diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +ngrams-loader +============= + +Ngrams loader based on http://www.ngrams.info format + +Installation +------------ +Supposed you have at least `cabal 1.18` installed + +``` +$ cabal sandbox init +$ cabal install --only-dependencies +$ cabal configure +$ cabal install + +-- program located in ~/.cabal-sandbox/bin +``` + +Usage +----- + +``` +usage: ngrams-loader [options] + [-2,--bigram] Parses bigrams + [-3,--trigram] Parses trigrams + [-4,--quadgram] Parses 4-grams + [-5,--pentagram] Parses 5-grams + [-c,--create] Creates table before inserts + N-grams file + SQlite db file +``` + +Example +------- + +``` +ngrams-loader --bigram --create w2.txt bigram.db + +``` +It parses each line of `w2.txt` as a bigram, create bigram table before performing inserts and saves everything in `bigram.db` + +Figures +------- + +Specs + +- Core i7 3770 @ 3.4GHz +- Gentoo with 3.12.13 Linux kernel (64bits) +- 1.055.386 lines bigram file + +`ngrams-loader --bigram --create w2.txt bigram.db` gets + +``` +real 0m16.244s +user 0m15.597s +sys 0m0.143s + +``` + +Sql Schemas +----------- + +Bigram + +```sql +create table bigrams( + frequence int, + word1 varchar(100), + word2 varchar(100) +); +``` + +Trigram + +```sql +create table tridgrams( + frequence int, + word1 varchar(100), + word2 varchar(100), + word3 varchar(100) +); +``` + +4-gram + +```sql +create table quadgrams( + frequence int, + word1 varchar(100), + word2 varchar(100), + word3 varchar(100), + word4 varchar(100) +); +``` + +5-gram + +```sql +create table pentagrams( + frequence int, + word1 varchar(100), + word2 varchar(100), + word3 varchar(100), + word4 varchar(100), + word5 varchar(100) +); +``` diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/lib/Data/Ngrams.hs b/lib/Data/Ngrams.hs new file mode 100644 --- /dev/null +++ b/lib/Data/Ngrams.hs @@ -0,0 +1,43 @@ +----------------------------------------------------------------------------- +-- | +-- Module : Data.Ngrams +-- Copyright : (C) 2014 Yorick Laupa +-- License : (see the file LICENSE) +-- +-- Maintainer : Yorick Laupa +-- Stability : provisional +-- Portability : non-portable +-- +---------------------------------------------------------------------------- +module Data.Ngrams + ( module Data.Ngrams.Database.Sqlite + , module Data.Ngrams.Parser + , module Data.Ngrams.Process + , mkProcess + , runProcess_ + ) where + +---------------------------------------------------------------------------- +import Control.Monad.Trans.Resource (ResourceT, runResourceT) +import Data.Attoparsec.Text (Parser) +import Data.Machine (ProcessT, runT_, (~>)) +import Data.Text (Text) + +---------------------------------------------------------------------------- +import Data.Ngrams.Database.Sqlite +import Data.Ngrams.Parser +import Data.Ngrams.Process + +---------------------------------------------------------------------------- +mkProcess :: FilePath + -> FilePath + -> Bool + -> Parser a + -> Command a + -> ProcessT (ResourceT IO) Text () +mkProcess file db create parser cmd = + sourceLines file ~> parseLine parser ~> saveDB create db cmd + +---------------------------------------------------------------------------- +runProcess_ :: ProcessT (ResourceT IO) Text a -> IO () +runProcess_ = runResourceT . runT_ diff --git a/lib/Data/Ngrams/Database/Sqlite.hs b/lib/Data/Ngrams/Database/Sqlite.hs new file mode 100644 --- /dev/null +++ b/lib/Data/Ngrams/Database/Sqlite.hs @@ -0,0 +1,134 @@ +{-# LANGUAGE OverloadedStrings #-} +----------------------------------------------------------------------------- +-- | +-- Module : Data.Ngrams.Database.Sqlite +-- Copyright : (C) 2014 Yorick Laupa +-- License : (see the file LICENSE) +-- +-- Maintainer : Yorick Laupa +-- Stability : provisional +-- Portability : non-portable +-- +---------------------------------------------------------------------------- +module Data.Ngrams.Database.Sqlite + ( Command + , cmdExec + , cmdCreateTable + , bigramCommand + , trigramCommand + , quadgramCommand + , pentagramCommand + ) where + +---------------------------------------------------------------------------- +import Database.SQLite.Simple + +---------------------------------------------------------------------------- +import Data.Ngrams.Type + +---------------------------------------------------------------------------- +data Command a = + Command + { cmdCreateTable :: Query + , cmdExec :: a -> Connection -> IO () + } + +---------------------------------------------------------------------------- +bigramCommand :: Command Bigram +bigramCommand = Command bigramCreateTable bigramEx + +---------------------------------------------------------------------------- +trigramCommand :: Command Trigram +trigramCommand = Command trigramCreateTable trigramEx + +---------------------------------------------------------------------------- +quadgramCommand :: Command Quadgram +quadgramCommand = Command quadgramCreateTable quadgramEx + +---------------------------------------------------------------------------- +pentagramCommand :: Command Pentagram +pentagramCommand = Command pentagramCreateTable pentagramEx + +---------------------------------------------------------------------------- +bigramCreateTable :: Query +bigramCreateTable = + "create table bigrams(\ + \ frequence int,\ + \ word1 varchar(100),\ + \ word2 varchar(100)\ + \);" + +---------------------------------------------------------------------------- +trigramCreateTable :: Query +trigramCreateTable = + "create table trigrams(\ + \ frequence int,\ + \ word1 varchar(100),\ + \ word2 varchar(100),\ + \ word3 varchar(100)\ + \);" + +---------------------------------------------------------------------------- +quadgramCreateTable :: Query +quadgramCreateTable = + "create table quadgrams(\ + \ frequence int,\ + \ word1 varchar(100),\ + \ word2 varchar(100),\ + \ word3 varchar(100),\ + \ word4 varchar(100)\ + \);" + +---------------------------------------------------------------------------- +pentagramCreateTable :: Query +pentagramCreateTable = + "create table pentagrams(\ + \ frequence int,\ + \ word1 varchar(100),\ + \ word2 varchar(100),\ + \ word3 varchar(100),\ + \ word4 varchar(100),\ + \ word5 varchar(100)\ + \);" + +---------------------------------------------------------------------------- +bigramInsert :: Query +bigramInsert = + "insert into bigrams (frequence, word1, word2) values (?,?,?);" + +---------------------------------------------------------------------------- +trigramInsert :: Query +trigramInsert = + "insert into trigrams (frequence, word1, word2, word3) values (?,?,?,?);" + +---------------------------------------------------------------------------- +quadgramInsert :: Query +quadgramInsert = + "insert into quadgrams (frequence, word1, word2, word3, word4) values \ + \(?,?,?,?,?);" + +---------------------------------------------------------------------------- +pentagramInsert :: Query +pentagramInsert = + "insert into pentagrams (frequence, word1, word2, word3, word4, word5) \ + \values (?,?,?,?,?);" + +---------------------------------------------------------------------------- +bigramEx:: Bigram -> Connection -> IO () +bigramEx (Bigram freq w1 w2) con = + execute con bigramInsert (freq, w1, w2) + +---------------------------------------------------------------------------- +trigramEx :: Trigram -> Connection -> IO () +trigramEx (Trigram freq w1 w2 w3) con = + execute con trigramInsert (freq, w1, w2, w3) + +---------------------------------------------------------------------------- +quadgramEx :: Quadgram -> Connection -> IO () +quadgramEx (Quadgram freq w1 w2 w3 w4) con = + execute con quadgramInsert (freq, w1, w2, w3, w4) + +---------------------------------------------------------------------------- +pentagramEx :: Pentagram -> Connection -> IO () +pentagramEx (Pentagram freq w1 w2 w3 w4 w5) con = + execute con pentagramInsert (freq, w1, w2, w3, w4, w5) diff --git a/lib/Data/Ngrams/Parser.hs b/lib/Data/Ngrams/Parser.hs new file mode 100644 --- /dev/null +++ b/lib/Data/Ngrams/Parser.hs @@ -0,0 +1,65 @@ +----------------------------------------------------------------------------- +-- | +-- Module : Data.Ngrams.Parser +-- Copyright : (C) 2014 Yorick Laupa +-- License : (see the file LICENSE) +-- +-- Maintainer : Yorick Laupa +-- Stability : provisional +-- Portability : non-portable +-- +---------------------------------------------------------------------------- +module Data.Ngrams.Parser + ( parserBigram + , parserTrigram + , parserQuadgram + , parserPentagram + ) where + +---------------------------------------------------------------------------- +import Data.Char (isSpace) + +---------------------------------------------------------------------------- +import Data.Attoparsec.Text (Parser, decimal, skipSpace, takeWhile1) +import Data.Text (Text) + +---------------------------------------------------------------------------- +import Data.Ngrams.Type + +---------------------------------------------------------------------------- +parserBigram :: Parser Bigram +parserBigram = do + freq <- decimal + skipSpace + w1 <- _word + skipSpace + w2 <- _word + return $ Bigram freq w1 w2 + +---------------------------------------------------------------------------- +parserTrigram :: Parser Trigram +parserTrigram = do + Bigram freq w1 w2 <- parserBigram + skipSpace + w3 <- _word + return $ Trigram freq w1 w2 w3 + +---------------------------------------------------------------------------- +parserQuadgram :: Parser Quadgram +parserQuadgram = do + Trigram freq w1 w2 w3 <- parserTrigram + skipSpace + w4 <- _word + return $ Quadgram freq w1 w2 w3 w4 + +---------------------------------------------------------------------------- +parserPentagram :: Parser Pentagram +parserPentagram = do + Quadgram freq w1 w2 w3 w4 <- parserQuadgram + skipSpace + w5 <- _word + return $ Pentagram freq w1 w2 w3 w4 w5 + +---------------------------------------------------------------------------- +_word :: Parser Text +_word = takeWhile1 (not . isSpace) diff --git a/lib/Data/Ngrams/Process.hs b/lib/Data/Ngrams/Process.hs new file mode 100644 --- /dev/null +++ b/lib/Data/Ngrams/Process.hs @@ -0,0 +1,96 @@ +{-# LANGUAGE DeriveDataTypeable #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} +----------------------------------------------------------------------------- +-- | +-- Module : Data.Ngrams.Process +-- Copyright : (C) 2014 Yorick Laupa +-- License : (see the file LICENSE) +-- +-- Maintainer : Yorick Laupa +-- Stability : provisional +-- Portability : non-portable +-- +---------------------------------------------------------------------------- +module Data.Ngrams.Process where + +---------------------------------------------------------------------------- +import Control.Applicative ((<|>)) +import Control.Monad (when) +import Control.Exception +import Data.Typeable +import System.IO (IOMode(..), hClose, hIsEOF, openFile) + +---------------------------------------------------------------------------- +import Data.Attoparsec.Text (Parser, parseOnly) +import Database.SQLite.Simple +import Control.Monad.Trans (lift, liftIO) +import Control.Monad.Trans.Resource +import Data.Machine +import qualified Data.Text as T +import qualified Data.Text.IO as T + +---------------------------------------------------------------------------- +import Data.Ngrams.Database.Sqlite + +---------------------------------------------------------------------------- +data ParseException = ParseException !String deriving (Show, Typeable) + +instance Exception ParseException + +---------------------------------------------------------------------------- +-- | Creates a `Source` from the lines of a file, using the `ResourceT` +-- monad to ensure the file is closed when processing the stream of lines +-- is finished +sourceLines :: FilePath -> SourceT (ResourceT IO) T.Text +sourceLines path = construct $ do + (key, h) <- lift $ allocate (openFile path ReadMode) hClose + go key h + where + go key h = + let readLine = lift $ liftIO $ T.hGetLine h + isEOF = lift $ liftIO $ hIsEOF h + + closing = do + lift $ release key + stop + + yielding = do + line <- readLine + yield line + go key h in + + isEOF >>= \eof -> if eof then closing else yielding + +---------------------------------------------------------------------------- +-- | Applies a `Parser` on each line. If an error occurs, an Exception is +-- raised +parseLine :: MonadThrow m => Parser a -> ProcessT m T.Text a +parseLine p = repeatedly $ do + line <- await + case parseOnly p line of + Left e -> lift $ monadThrow $ ParseException e + Right a -> yield a + +---------------------------------------------------------------------------- +-- | Saves each entry in a SQLite database with submitted `Command` +saveDB :: Bool -> FilePath -> Command a -> ProcessT (ResourceT IO) a () +saveDB create path c = construct $ do + (ckey, con) <- lift $ allocate (open path) close + when create $ liftIO $ execute_ con createTable + lift $ liftIO $ execute_ con "begin" + loop ckey con + where + exec = cmdExec c + createTable = cmdCreateTable c + + loop ckey con = + let closing = do + lift $ liftIO $ do + execute_ con "end" + release ckey + stop in + + do a <- await <|> closing + lift $ liftIO $ exec a con + loop ckey con diff --git a/lib/Data/Ngrams/Type.hs b/lib/Data/Ngrams/Type.hs new file mode 100644 --- /dev/null +++ b/lib/Data/Ngrams/Type.hs @@ -0,0 +1,50 @@ +----------------------------------------------------------------------------- +-- | +-- Module : Data.Ngrams.Type +-- Copyright : (C) 2014 Yorick Laupa +-- License : (see the file LICENSE) +-- +-- Maintainer : Yorick Laupa +-- Stability : provisional +-- Portability : non-portable +-- +---------------------------------------------------------------------------- +module Data.Ngrams.Type where + +---------------------------------------------------------------------------- +import Data.Text (Text) + +---------------------------------------------------------------------------- +data Bigram = + Bigram + { biFreq :: !Integer + , biWord1 :: !Text + , biWord2 :: !Text + } + +data Trigram = + Trigram + { triFreq :: !Integer + , triWord1 :: !Text + , triWord2 :: !Text + , triWord3 :: !Text + } + +data Quadgram = + Quadgram + { quadFreq :: !Integer + , quadWord1 :: !Text + , quadWord2 :: !Text + , quadWord3 :: !Text + , quadWord4 :: !Text + } + +data Pentagram = + Pentagram + { pentaFreq :: !Integer + , pentaWord1 :: !Text + , pentaWord2 :: !Text + , pentaWord3 :: !Text + , pentaWord4 :: !Text + , pentaWord5 :: !Text + } diff --git a/ngrams-loader.cabal b/ngrams-loader.cabal new file mode 100644 --- /dev/null +++ b/ngrams-loader.cabal @@ -0,0 +1,49 @@ +name: ngrams-loader +version: 0.1.0.0 +synopsis: Ngrams loader based on http://www.ngrams.info format +description: Ngrams loader based on http://www.ngrams.info format +license: MIT +license-file: LICENSE +author: Yorick Laupa +maintainer: Yorick Laupa +homepage: http://github.com/YoEight/ngrams-loader +bug-reports: https://github.com/YoEight/ngrams-loader/issues +copyright: Copyright (C) 2014 Yorick Laupa + +category: Data +build-type: Simple +extra-source-files: README.md +cabal-version: >=1.10 +tested-with: GHC == 7.6.3 + +source-repository head + type: git + location: git://github.com/YoEight/ngrams-loader.git + +library + hs-source-dirs: lib + build-depends: base >=4.6 && <4.7 + , attoparsec >=0.11.1 && <0.11.2 + , machines >= 0.2.5 && <0.3 + , mtl + , resourcet >=0.4.3 && <0.5 + , sqlite-simple >= 0.4.5 && <0.5 + , text >=0.11 && <1.2 + + exposed-modules: Data.Ngrams + Data.Ngrams.Database.Sqlite + other-modules: Data.Ngrams.Process + Data.Ngrams.Parser + Data.Ngrams.Type + + ghc-options: -Wall -O2 + default-language: Haskell2010 + +executable ngrams-loader + main-is: Main.hs + build-depends: base + , ngrams-loader + , parseargs ==0.1.5.* + + ghc-options: -Wall -O2 + default-language: Haskell2010