wikipedia4epub 0.0.3 → 0.0.4

raw patch · 5 files changed

+184/−175 lines, 5 filesnew-component:exe:wiki4e-mkepub-firefoxPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• src/Wiki4e/Commands.hs +137/−0
• src/Wikipedia4epub/Commands.hs +0/−132
• src/wiki4e-firefox-epub.hs +0/−39
• src/wiki4e-mkepub-firefox.hs +43/−0
• wikipedia4epub.cabal +4/−4

@@ -0,0 +1,137 @@+module Wiki4e.Commands where+import Network.Wikipedia+import Web.Firefox+import System.FilePath+import System.Directory+import Network.URL+import Control.Monad (liftM, filterM)+import Data.List (nub, foldl')+import Data.Char (toLower)+import System.IO+import Network.HTTP+import Codec.EBook+import qualified Data.ByteString.Lazy as B++-- Fetching state where first number is total and second is current item.+type FetchingState = (Int,Int)+type FetchingMethod = (URL -> IO String)++fs2str :: FetchingState -> String+fs2str (total,current) = "["++(show current)++"/"++(show total)++"] " ++wiki4e_fetch :: FilePath -> FetchingMethod -> FetchingState -> URL -> IO ()+wiki4e_fetch outf fm fs x = do+  e <- doesFileExist outf+  if not e then do+    putStrLn $ (fs2str fs) ++ "Fetching : " ++ exportURL x+    c <- fm x+    withBinaryFile outf WriteMode (flip hPutStr c)+    else putStrLn $ (fs2str fs) ++ "Already cached. Skipping download. " ++ outf++wiki4e_fetchArticles :: FilePath -> [URL] -> IO ()+wiki4e_fetchArticles oud xs = mapM_ (\(st,x) -> wiki4e_fetch (nm x) fm st x) $ zip fsls xs+    where+      fsls = [ (total,i) | i <- [1..total] ]+      total = length xs+      nm x = oud </> (articleURL2Title x)+      fm x = do+        (WikiArticleHTML _ c) <- fetchArticle x+        return c++wiki4e_fetchImages :: FilePath -> [URL] -> IO ()+wiki4e_fetchImages oud xs = mapM_ (\(st,x) -> wiki4e_fetch (nm x) fm st x) $ zip fsls xs+    where+      fsls = [ (total,i) | i <- [1..total] ]+      total = length xs+      nm x = oud </> (sanitizeFileName $ takeFileName (url_path x))+      fm x = do+        rsp <- Network.HTTP.simpleHTTP (getRequest (exportURL x))+        c <- (getResponseBody rsp)+        return c++wiki4e_fetchArticleTree :: String -> Int -> IO ()+wiki4e_fetchArticleTree xs l = undefined++wiki4e_sanitizeArticle :: FilePath -> FilePath -> IO ()+wiki4e_sanitizeArticle inf ouf = withFile inf ReadMode (\hi -> do+          hSetEncoding hi utf8+          withFile ouf WriteMode (\ho -> do +                  hSetEncoding ho utf8+                  c <- hGetContents hi+                  let a = sanitizeArticle $ WikiArticleHTML "" c+                  hPutStr ho $ waContent a+               )+         )++wiki4e_listArticleImages :: FilePath -> IO [URL]+wiki4e_listArticleImages x = do+          h <- openFile x ReadMode+          hSetEncoding h utf8+          c <- hGetContents h+          return $ getArticleImages (WikiArticleHTML "" c)++wiki4e_listArticlesImages :: FilePath -> IO [URL]+wiki4e_listArticlesImages ind = do +     xs <- wiki4e_listFiles ind+     ys <- mapM (wiki4e_listArticleImages) xs+     return $ nub $ concat ys++wiki4e_sanitizeArticles :: FilePath -> FilePath -> IO ()+wiki4e_sanitizeArticles ind oud = wiki4e_listFiles ind >>= +     mapM_ (\x -> wiki4e_sanitizeArticle x (oud </> (takeFileName x)))++wiki4e_listFiles :: FilePath -> IO [FilePath]+wiki4e_listFiles xs = do+  nonDot <- liftM (filter (\(c:_) -> c /= '.')) $ getDirectoryContents xs+  dirs <- filterM (doesDirectoryExist) (map (xs </>) nonDot)+  fils <- filterM (\e -> liftM (not) $ doesDirectoryExist e ) (map (xs </>) nonDot)+  ys <- mapM (wiki4e_listFiles) dirs+  return $ (concat ys) ++ fils++wiki4e_listFirefoxURLs :: IO [URL]+wiki4e_listFirefoxURLs = do+     xs <- listAllHistoryURLs+     return $ nub $ filter isArticleURL xs++wiki4e_createEpub :: String -> FilePath -> FilePath -> IO ()+wiki4e_createEpub bookName srcDir imgDir = do+     let book = emptyBook { +                  bookID = "http://localhost/"++bookName,+                  bookAuthor = "wiki4e-firefox-epub",+                  bookTitle = bookName+                }+     xs <- wiki4e_listFiles srcDir+     itemsA <- mapM (\(i,x) -> loadArticleFile i "wiki" x) $ zip [1..] xs+     ys <- wiki4e_listFiles imgDir+     itemsI <- mapM (\(i,x) -> loadImgFile i "wiki/img" x) $ zip [(length xs + 1)..] ys+     let bookFull = foldl' addItem2Book book (itemsA ++ itemsI)+     let epubFName = bookName++".epub"+     outdata <- book2Bin' bookFull+     B.writeFile epubFName  outdata+     putStrLn $ epubFName ++ " constructed."++-- Support Functions++loadArticleFile :: Int -> FilePath -> FilePath -> IO BookItem+loadArticleFile i bookDir fname = do+   cs <- B.readFile fname+   return (BookItem aid bfile cs opsMediatype (Just (ChapterMetadata name))) +   where+      aid = show i+      bfile = bookDir </> name+      name = takeFileName $ normalise fname++loadImgFile :: Int -> FilePath -> FilePath -> IO BookItem+loadImgFile i bookDir fname = do+   cs <- B.readFile fname+   return (BookItem aid bfile cs mimeType Nothing) +   where+      mimeType | hasExt ".png"  name = "image/png"+               | hasExt ".jpeg" name = "image/jpeg"+               | hasExt ".jpg"  name = "image/jpeg"+               | hasExt ".gif"  name = "image/gif"+               | otherwise = "image/png"+      hasExt x f = (map toLower $ takeExtension f) == x+      aid = show i+      bfile = bookDir </> name+      name = takeFileName $ normalise fname

@@ -1,132 +0,0 @@-module Wikipedia4epub.Commands where-import Network.Wikipedia-import Web.Firefox-import System.FilePath-import System.Directory-import Network.URL-import Control.Monad (liftM, filterM)-import Data.List (nub, foldl')-import Data.Char (toLower)-import System.IO-import Network.HTTP-import Codec.EBook-import qualified Data.ByteString.Lazy as B--wiki4e_fetchArticle :: FilePath -> URL -> IO ()-wiki4e_fetchArticle oud x = do-  e <- doesFileExist filename-  if not e then do-    putStrLn $ "Fetching : " ++ exportURL x-    (WikiArticleHTML _ c) <- fetchArticle x-    withBinaryFile filename WriteMode (flip hPutStr c)-    else putStrLn $ "File already exists. Skipping download: " ++ filename-  where-    filename = oud </> title-    title    = articleURL2Title x--wiki4e_fetchArticles :: FilePath -> [URL] -> IO ()-wiki4e_fetchArticles oud xs = mapM_ (wiki4e_fetchArticle oud) xs--wiki4e_fetchArticleTree :: String -> Int -> IO ()-wiki4e_fetchArticleTree xs l = undefined--wiki4e_sanitizeArticle :: FilePath -> FilePath -> IO ()-wiki4e_sanitizeArticle inf ouf = withFile inf ReadMode (\hi -> do-          hSetEncoding hi utf8-          withFile ouf WriteMode (\ho -> do -                  hSetEncoding ho utf8-                  c <- hGetContents hi-                  let a = sanitizeArticle $ WikiArticleHTML "" c-                  hPutStr ho $ waContent a-               )-         )--wiki4e_listArticleImages :: FilePath -> IO [URL]-wiki4e_listArticleImages x = do-          h <- openFile x ReadMode-          hSetEncoding h utf8-          c <- hGetContents h-          return $ getArticleImages (WikiArticleHTML "" c)--wiki4e_listArticlesImages :: FilePath -> IO [URL]-wiki4e_listArticlesImages ind = do -     xs <- wiki4e_listFiles ind-     ys <- mapM (wiki4e_listArticleImages) xs-     return $ nub $ concat ys--wiki4e_fetchImage :: FilePath -> URL -> IO ()-wiki4e_fetchImage oud x = do-  e <- doesFileExist filename-  if not e then do-    createDirectoryIfMissing True (takeDirectory filename)-    putStrLn $ "Fetching : " ++ exportURL x-    rsp <- Network.HTTP.simpleHTTP (getRequest (exportURL x))-    c <- (getResponseBody rsp)-    withBinaryFile filename WriteMode (flip hPutStr c)-    else putStrLn $ "File already exists. Skipping download: " ++ filename-  where-    filename = oud </> name-    name     = sanitizeFileName $ takeFileName (url_path x)--wiki4e_fetchImages :: FilePath -> [URL] -> IO ()-wiki4e_fetchImages oud xs = mapM_ (wiki4e_fetchImage oud) xs--wiki4e_sanitizeArticles :: FilePath -> FilePath -> IO ()-wiki4e_sanitizeArticles ind oud = wiki4e_listFiles ind >>= -     mapM_ (\x -> wiki4e_sanitizeArticle x (oud </> (takeFileName x)))--wiki4e_listFiles :: FilePath -> IO [FilePath]-wiki4e_listFiles xs = do-  nonDot <- liftM (filter (\(c:_) -> c /= '.')) $ getDirectoryContents xs-  dirs <- filterM (doesDirectoryExist) (map (xs </>) nonDot)-  fils <- filterM (\e -> liftM (not) $ doesDirectoryExist e ) (map (xs </>) nonDot)-  ys <- mapM (wiki4e_listFiles) dirs-  return $ (concat ys) ++ fils--wiki4e_listFirefoxURLs :: IO [URL]-wiki4e_listFirefoxURLs = do-     xs <- listAllHistoryURLs-     return $ nub $ filter isArticleURL xs--wiki4e_createEpub :: String -> FilePath -> FilePath -> IO ()-wiki4e_createEpub bookName srcDir imgDir = do-     let book = emptyBook { -                  bookID = "http://localhost/"++bookName,-                  bookAuthor = "wiki4e-firefox-epub",-                  bookTitle = bookName-                }-     xs <- wiki4e_listFiles srcDir-     itemsA <- mapM (\(i,x) -> loadArticleFile i "wiki" x) $ zip [1..] xs-     ys <- wiki4e_listFiles imgDir-     itemsI <- mapM (\(i,x) -> loadImgFile i "wiki/img" x) $ zip [(length xs + 1)..] ys-     let bookFull = foldl' addItem2Book book (itemsA ++ itemsI)-     let epubFName = bookName++".epub"-     outdata <- book2Bin' bookFull-     B.writeFile epubFName  outdata-     putStrLn $ epubFName ++ " constructed."---- Support Functions--loadArticleFile :: Int -> FilePath -> FilePath -> IO BookItem-loadArticleFile i bookDir fname = do-   cs <- B.readFile fname-   return (BookItem aid bfile cs opsMediatype (Just (ChapterMetadata name))) -   where-      aid = show i-      bfile = bookDir </> name-      name = takeFileName $ normalise fname--loadImgFile :: Int -> FilePath -> FilePath -> IO BookItem-loadImgFile i bookDir fname = do-   cs <- B.readFile fname-   return (BookItem aid bfile cs mimeType Nothing) -   where-      mimeType | hasExt ".png"  name = "image/png"-               | hasExt ".jpeg" name = "image/jpeg"-               | hasExt ".jpg"  name = "image/jpeg"-               | hasExt ".gif"  name = "image/gif"-               | otherwise = "image/png"-      hasExt x f = (map toLower $ takeExtension f) == x-      aid = show i-      bfile = bookDir </> name-      name = takeFileName $ normalise fname

@@ -1,39 +0,0 @@-import Text.HTML.TagSoup-import System.Environment-import System.Directory-import System.FilePath-import Wikipedia4epub.Commands-import System.IO -import Network.URL--main = do-  name <- getProgName-  args <- getArgs-  case args of -      []      -> firefox2epub "Wikipedia_Articles_From_Firefox"-      ['-':_] -> usageHelp name-      ['/':_] -> usageHelp name-      [xs]    -> firefox2epub xs-      _       -> usageHelp name--usageHelp name = putStrLn $ "Usage: " ++ name ++ " [<Title Name of new e-book>]"--firefox2epub bookName = do-  xs <- wiki4e_listFirefoxURLs-  tmpDir <- getTemporaryDirectory-  let tmpDirFetch    = tmpDir </> "wiki4e_fetch"     -  let tmpDirSanitize = tmpDir </> "wiki4e_sanitize"-  let tmpDirImgs     = tmpDir </> "wiki4e_images"-  createDirectoryIfMissing True tmpDirFetch-  createDirectoryIfMissing True tmpDirSanitize-  createDirectoryIfMissing True tmpDirImgs-  putStrLn $ "Fetching "++show (length xs)++" Articles..."-  wiki4e_fetchArticles tmpDirFetch xs-  putStrLn "Sanitize Articles..."-  wiki4e_sanitizeArticles tmpDirFetch tmpDirSanitize-  putStrLn "Download Images..."-  imgs <- wiki4e_listArticlesImages tmpDirFetch-  wiki4e_fetchImages tmpDirImgs imgs-  putStrLn "Constructing EPUB..."-  wiki4e_createEpub bookName tmpDirSanitize tmpDirImgs-  putStrLn "Done."

@@ -0,0 +1,43 @@+import Text.HTML.TagSoup+import System.Environment+import System.Directory+import System.FilePath+import Wiki4e.Commands+import System.IO +import Network.URL++defaultEbookName = "Wikipedia_Articles_From_Firefox"++main = do+  name <- getProgName+  args <- getArgs+  case args of +      []      -> firefox2epub defaultEbookName+      ['-':_] -> usageHelp name+      ['/':_] -> usageHelp name+      [xs]    -> firefox2epub xs+      _       -> usageHelp name++usageHelp name = putStrLn $ "Usage: " ++ name ++ " [<Title Name of new e-book>]"++getWiki4eDir = getAppUserDataDirectory "wiki4e"++firefox2epub bookName = do+  xs <- wiki4e_listFirefoxURLs+  tmpDir <- getWiki4eDir+  let tmpDirFetch    = tmpDir </> "wiki4e_fetch"     +  let tmpDirSanitize = tmpDir </> "wiki4e_sanitize"+  let tmpDirImgs     = tmpDir </> "wiki4e_images"+  createDirectoryIfMissing True tmpDirFetch+  createDirectoryIfMissing True tmpDirSanitize+  createDirectoryIfMissing True tmpDirImgs+  putStrLn "# STAGE 1/4 - Download Articles..."+  wiki4e_fetchArticles tmpDirFetch xs+  putStrLn "# STAGE 2/4 - Sanitize Articles..."+  wiki4e_sanitizeArticles tmpDirFetch tmpDirSanitize+  putStrLn "# STAGE 3/4 - Download Images..."+  imgs <- wiki4e_listArticlesImages tmpDirFetch+  wiki4e_fetchImages tmpDirImgs imgs+  putStrLn "# STAGE 4/4 - Constructing EPUB..."+  wiki4e_createEpub bookName tmpDirSanitize tmpDirImgs+  putStrLn "Done."

@@ -1,5 +1,5 @@ Name:            wikipedia4epub-Version:         0.0.3+Version:         0.0.4 License:         BSD3 License-File:    LICENSE Homepage:        http://rampa.sk/static/wikipedia4epub.html@@ -46,10 +46,10 @@   Ghc-Options:      -Wall -fno-warn-orphans    Exposed-modules:-        Wikipedia4epub.Commands+        Wiki4e.Commands         Network.Wikipedia         Web.Firefox -Executable wiki4e-firefox-epub+Executable wiki4e-mkepub-firefox   Hs-Source-Dirs:  src-  Main-Is:         wiki4e-firefox-epub.hs+  Main-Is:         wiki4e-mkepub-firefox.hs