diff --git a/src/Wiki4e/Commands.hs b/src/Wiki4e/Commands.hs new file mode 100644 --- /dev/null +++ b/src/Wiki4e/Commands.hs @@ -0,0 +1,137 @@ +module Wiki4e.Commands where +import Network.Wikipedia +import Web.Firefox +import System.FilePath +import System.Directory +import Network.URL +import Control.Monad (liftM, filterM) +import Data.List (nub, foldl') +import Data.Char (toLower) +import System.IO +import Network.HTTP +import Codec.EBook +import qualified Data.ByteString.Lazy as B + +-- Fetching state where first number is total and second is current item. +type FetchingState = (Int,Int) +type FetchingMethod = (URL -> IO String) + +fs2str :: FetchingState -> String +fs2str (total,current) = "["++(show current)++"/"++(show total)++"] " + +wiki4e_fetch :: FilePath -> FetchingMethod -> FetchingState -> URL -> IO () +wiki4e_fetch outf fm fs x = do + e <- doesFileExist outf + if not e then do + putStrLn $ (fs2str fs) ++ "Fetching : " ++ exportURL x + c <- fm x + withBinaryFile outf WriteMode (flip hPutStr c) + else putStrLn $ (fs2str fs) ++ "Already cached. Skipping download. " ++ outf + +wiki4e_fetchArticles :: FilePath -> [URL] -> IO () +wiki4e_fetchArticles oud xs = mapM_ (\(st,x) -> wiki4e_fetch (nm x) fm st x) $ zip fsls xs + where + fsls = [ (total,i) | i <- [1..total] ] + total = length xs + nm x = oud (articleURL2Title x) + fm x = do + (WikiArticleHTML _ c) <- fetchArticle x + return c + +wiki4e_fetchImages :: FilePath -> [URL] -> IO () +wiki4e_fetchImages oud xs = mapM_ (\(st,x) -> wiki4e_fetch (nm x) fm st x) $ zip fsls xs + where + fsls = [ (total,i) | i <- [1..total] ] + total = length xs + nm x = oud (sanitizeFileName $ takeFileName (url_path x)) + fm x = do + rsp <- Network.HTTP.simpleHTTP (getRequest (exportURL x)) + c <- (getResponseBody rsp) + return c + +wiki4e_fetchArticleTree :: String -> Int -> IO () +wiki4e_fetchArticleTree xs l = undefined + +wiki4e_sanitizeArticle :: FilePath -> FilePath -> IO () +wiki4e_sanitizeArticle inf ouf = withFile inf ReadMode (\hi -> do + hSetEncoding hi utf8 + withFile ouf WriteMode (\ho -> do + hSetEncoding ho utf8 + c <- hGetContents hi + let a = sanitizeArticle $ WikiArticleHTML "" c + hPutStr ho $ waContent a + ) + ) + +wiki4e_listArticleImages :: FilePath -> IO [URL] +wiki4e_listArticleImages x = do + h <- openFile x ReadMode + hSetEncoding h utf8 + c <- hGetContents h + return $ getArticleImages (WikiArticleHTML "" c) + +wiki4e_listArticlesImages :: FilePath -> IO [URL] +wiki4e_listArticlesImages ind = do + xs <- wiki4e_listFiles ind + ys <- mapM (wiki4e_listArticleImages) xs + return $ nub $ concat ys + +wiki4e_sanitizeArticles :: FilePath -> FilePath -> IO () +wiki4e_sanitizeArticles ind oud = wiki4e_listFiles ind >>= + mapM_ (\x -> wiki4e_sanitizeArticle x (oud (takeFileName x))) + +wiki4e_listFiles :: FilePath -> IO [FilePath] +wiki4e_listFiles xs = do + nonDot <- liftM (filter (\(c:_) -> c /= '.')) $ getDirectoryContents xs + dirs <- filterM (doesDirectoryExist) (map (xs ) nonDot) + fils <- filterM (\e -> liftM (not) $ doesDirectoryExist e ) (map (xs ) nonDot) + ys <- mapM (wiki4e_listFiles) dirs + return $ (concat ys) ++ fils + +wiki4e_listFirefoxURLs :: IO [URL] +wiki4e_listFirefoxURLs = do + xs <- listAllHistoryURLs + return $ nub $ filter isArticleURL xs + +wiki4e_createEpub :: String -> FilePath -> FilePath -> IO () +wiki4e_createEpub bookName srcDir imgDir = do + let book = emptyBook { + bookID = "http://localhost/"++bookName, + bookAuthor = "wiki4e-firefox-epub", + bookTitle = bookName + } + xs <- wiki4e_listFiles srcDir + itemsA <- mapM (\(i,x) -> loadArticleFile i "wiki" x) $ zip [1..] xs + ys <- wiki4e_listFiles imgDir + itemsI <- mapM (\(i,x) -> loadImgFile i "wiki/img" x) $ zip [(length xs + 1)..] ys + let bookFull = foldl' addItem2Book book (itemsA ++ itemsI) + let epubFName = bookName++".epub" + outdata <- book2Bin' bookFull + B.writeFile epubFName outdata + putStrLn $ epubFName ++ " constructed." + +-- Support Functions + +loadArticleFile :: Int -> FilePath -> FilePath -> IO BookItem +loadArticleFile i bookDir fname = do + cs <- B.readFile fname + return (BookItem aid bfile cs opsMediatype (Just (ChapterMetadata name))) + where + aid = show i + bfile = bookDir name + name = takeFileName $ normalise fname + +loadImgFile :: Int -> FilePath -> FilePath -> IO BookItem +loadImgFile i bookDir fname = do + cs <- B.readFile fname + return (BookItem aid bfile cs mimeType Nothing) + where + mimeType | hasExt ".png" name = "image/png" + | hasExt ".jpeg" name = "image/jpeg" + | hasExt ".jpg" name = "image/jpeg" + | hasExt ".gif" name = "image/gif" + | otherwise = "image/png" + hasExt x f = (map toLower $ takeExtension f) == x + aid = show i + bfile = bookDir name + name = takeFileName $ normalise fname diff --git a/src/Wikipedia4epub/Commands.hs b/src/Wikipedia4epub/Commands.hs deleted file mode 100644 --- a/src/Wikipedia4epub/Commands.hs +++ /dev/null @@ -1,132 +0,0 @@ -module Wikipedia4epub.Commands where -import Network.Wikipedia -import Web.Firefox -import System.FilePath -import System.Directory -import Network.URL -import Control.Monad (liftM, filterM) -import Data.List (nub, foldl') -import Data.Char (toLower) -import System.IO -import Network.HTTP -import Codec.EBook -import qualified Data.ByteString.Lazy as B - -wiki4e_fetchArticle :: FilePath -> URL -> IO () -wiki4e_fetchArticle oud x = do - e <- doesFileExist filename - if not e then do - putStrLn $ "Fetching : " ++ exportURL x - (WikiArticleHTML _ c) <- fetchArticle x - withBinaryFile filename WriteMode (flip hPutStr c) - else putStrLn $ "File already exists. Skipping download: " ++ filename - where - filename = oud title - title = articleURL2Title x - -wiki4e_fetchArticles :: FilePath -> [URL] -> IO () -wiki4e_fetchArticles oud xs = mapM_ (wiki4e_fetchArticle oud) xs - -wiki4e_fetchArticleTree :: String -> Int -> IO () -wiki4e_fetchArticleTree xs l = undefined - -wiki4e_sanitizeArticle :: FilePath -> FilePath -> IO () -wiki4e_sanitizeArticle inf ouf = withFile inf ReadMode (\hi -> do - hSetEncoding hi utf8 - withFile ouf WriteMode (\ho -> do - hSetEncoding ho utf8 - c <- hGetContents hi - let a = sanitizeArticle $ WikiArticleHTML "" c - hPutStr ho $ waContent a - ) - ) - -wiki4e_listArticleImages :: FilePath -> IO [URL] -wiki4e_listArticleImages x = do - h <- openFile x ReadMode - hSetEncoding h utf8 - c <- hGetContents h - return $ getArticleImages (WikiArticleHTML "" c) - -wiki4e_listArticlesImages :: FilePath -> IO [URL] -wiki4e_listArticlesImages ind = do - xs <- wiki4e_listFiles ind - ys <- mapM (wiki4e_listArticleImages) xs - return $ nub $ concat ys - -wiki4e_fetchImage :: FilePath -> URL -> IO () -wiki4e_fetchImage oud x = do - e <- doesFileExist filename - if not e then do - createDirectoryIfMissing True (takeDirectory filename) - putStrLn $ "Fetching : " ++ exportURL x - rsp <- Network.HTTP.simpleHTTP (getRequest (exportURL x)) - c <- (getResponseBody rsp) - withBinaryFile filename WriteMode (flip hPutStr c) - else putStrLn $ "File already exists. Skipping download: " ++ filename - where - filename = oud name - name = sanitizeFileName $ takeFileName (url_path x) - -wiki4e_fetchImages :: FilePath -> [URL] -> IO () -wiki4e_fetchImages oud xs = mapM_ (wiki4e_fetchImage oud) xs - -wiki4e_sanitizeArticles :: FilePath -> FilePath -> IO () -wiki4e_sanitizeArticles ind oud = wiki4e_listFiles ind >>= - mapM_ (\x -> wiki4e_sanitizeArticle x (oud (takeFileName x))) - -wiki4e_listFiles :: FilePath -> IO [FilePath] -wiki4e_listFiles xs = do - nonDot <- liftM (filter (\(c:_) -> c /= '.')) $ getDirectoryContents xs - dirs <- filterM (doesDirectoryExist) (map (xs ) nonDot) - fils <- filterM (\e -> liftM (not) $ doesDirectoryExist e ) (map (xs ) nonDot) - ys <- mapM (wiki4e_listFiles) dirs - return $ (concat ys) ++ fils - -wiki4e_listFirefoxURLs :: IO [URL] -wiki4e_listFirefoxURLs = do - xs <- listAllHistoryURLs - return $ nub $ filter isArticleURL xs - -wiki4e_createEpub :: String -> FilePath -> FilePath -> IO () -wiki4e_createEpub bookName srcDir imgDir = do - let book = emptyBook { - bookID = "http://localhost/"++bookName, - bookAuthor = "wiki4e-firefox-epub", - bookTitle = bookName - } - xs <- wiki4e_listFiles srcDir - itemsA <- mapM (\(i,x) -> loadArticleFile i "wiki" x) $ zip [1..] xs - ys <- wiki4e_listFiles imgDir - itemsI <- mapM (\(i,x) -> loadImgFile i "wiki/img" x) $ zip [(length xs + 1)..] ys - let bookFull = foldl' addItem2Book book (itemsA ++ itemsI) - let epubFName = bookName++".epub" - outdata <- book2Bin' bookFull - B.writeFile epubFName outdata - putStrLn $ epubFName ++ " constructed." - --- Support Functions - -loadArticleFile :: Int -> FilePath -> FilePath -> IO BookItem -loadArticleFile i bookDir fname = do - cs <- B.readFile fname - return (BookItem aid bfile cs opsMediatype (Just (ChapterMetadata name))) - where - aid = show i - bfile = bookDir name - name = takeFileName $ normalise fname - -loadImgFile :: Int -> FilePath -> FilePath -> IO BookItem -loadImgFile i bookDir fname = do - cs <- B.readFile fname - return (BookItem aid bfile cs mimeType Nothing) - where - mimeType | hasExt ".png" name = "image/png" - | hasExt ".jpeg" name = "image/jpeg" - | hasExt ".jpg" name = "image/jpeg" - | hasExt ".gif" name = "image/gif" - | otherwise = "image/png" - hasExt x f = (map toLower $ takeExtension f) == x - aid = show i - bfile = bookDir name - name = takeFileName $ normalise fname diff --git a/src/wiki4e-firefox-epub.hs b/src/wiki4e-firefox-epub.hs deleted file mode 100644 --- a/src/wiki4e-firefox-epub.hs +++ /dev/null @@ -1,39 +0,0 @@ -import Text.HTML.TagSoup -import System.Environment -import System.Directory -import System.FilePath -import Wikipedia4epub.Commands -import System.IO -import Network.URL - -main = do - name <- getProgName - args <- getArgs - case args of - [] -> firefox2epub "Wikipedia_Articles_From_Firefox" - ['-':_] -> usageHelp name - ['/':_] -> usageHelp name - [xs] -> firefox2epub xs - _ -> usageHelp name - -usageHelp name = putStrLn $ "Usage: " ++ name ++ " []" - -firefox2epub bookName = do - xs <- wiki4e_listFirefoxURLs - tmpDir <- getTemporaryDirectory - let tmpDirFetch = tmpDir </> "wiki4e_fetch" - let tmpDirSanitize = tmpDir </> "wiki4e_sanitize" - let tmpDirImgs = tmpDir </> "wiki4e_images" - createDirectoryIfMissing True tmpDirFetch - createDirectoryIfMissing True tmpDirSanitize - createDirectoryIfMissing True tmpDirImgs - putStrLn $ "Fetching "++show (length xs)++" Articles..." - wiki4e_fetchArticles tmpDirFetch xs - putStrLn "Sanitize Articles..." - wiki4e_sanitizeArticles tmpDirFetch tmpDirSanitize - putStrLn "Download Images..." - imgs <- wiki4e_listArticlesImages tmpDirFetch - wiki4e_fetchImages tmpDirImgs imgs - putStrLn "Constructing EPUB..." - wiki4e_createEpub bookName tmpDirSanitize tmpDirImgs - putStrLn "Done." diff --git a/src/wiki4e-mkepub-firefox.hs b/src/wiki4e-mkepub-firefox.hs new file mode 100644 --- /dev/null +++ b/src/wiki4e-mkepub-firefox.hs @@ -0,0 +1,43 @@ +import Text.HTML.TagSoup +import System.Environment +import System.Directory +import System.FilePath +import Wiki4e.Commands +import System.IO +import Network.URL + +defaultEbookName = "Wikipedia_Articles_From_Firefox" + +main = do + name <- getProgName + args <- getArgs + case args of + [] -> firefox2epub defaultEbookName + ['-':_] -> usageHelp name + ['/':_] -> usageHelp name + [xs] -> firefox2epub xs + _ -> usageHelp name + +usageHelp name = putStrLn $ "Usage: " ++ name ++ " [<Title Name of new e-book>]" + +getWiki4eDir = getAppUserDataDirectory "wiki4e" + +firefox2epub bookName = do + xs <- wiki4e_listFirefoxURLs + tmpDir <- getWiki4eDir + let tmpDirFetch = tmpDir </> "wiki4e_fetch" + let tmpDirSanitize = tmpDir </> "wiki4e_sanitize" + let tmpDirImgs = tmpDir </> "wiki4e_images" + createDirectoryIfMissing True tmpDirFetch + createDirectoryIfMissing True tmpDirSanitize + createDirectoryIfMissing True tmpDirImgs + putStrLn "# STAGE 1/4 - Download Articles..." + wiki4e_fetchArticles tmpDirFetch xs + putStrLn "# STAGE 2/4 - Sanitize Articles..." + wiki4e_sanitizeArticles tmpDirFetch tmpDirSanitize + putStrLn "# STAGE 3/4 - Download Images..." + imgs <- wiki4e_listArticlesImages tmpDirFetch + wiki4e_fetchImages tmpDirImgs imgs + putStrLn "# STAGE 4/4 - Constructing EPUB..." + wiki4e_createEpub bookName tmpDirSanitize tmpDirImgs + putStrLn "Done." diff --git a/wikipedia4epub.cabal b/wikipedia4epub.cabal --- a/wikipedia4epub.cabal +++ b/wikipedia4epub.cabal @@ -1,5 +1,5 @@ Name: wikipedia4epub -Version: 0.0.3 +Version: 0.0.4 License: BSD3 License-File: LICENSE Homepage: http://rampa.sk/static/wikipedia4epub.html @@ -46,10 +46,10 @@ Ghc-Options: -Wall -fno-warn-orphans Exposed-modules: - Wikipedia4epub.Commands + Wiki4e.Commands Network.Wikipedia Web.Firefox -Executable wiki4e-firefox-epub +Executable wiki4e-mkepub-firefox Hs-Source-Dirs: src - Main-Is: wiki4e-firefox-epub.hs + Main-Is: wiki4e-mkepub-firefox.hs