wikipedia4epub 0.0.6 → 0.0.7
raw patch · 6 files changed
+16/−106 lines, 6 filesdep +zlib
Dependencies added: zlib
Files
- src/Network/Wikipedia.hs +12/−5
- src/Wiki4e/Commands/Articles.hs +2/−4
- src/Wiki4e/Commands/Common.hs +0/−1
- src/wiki4e-mkepub-cache.hs +0/−45
- src/wiki4e-mkepub-firefox.hs +0/−43
- wikipedia4epub.cabal +2/−8
src/Network/Wikipedia.hs view
@@ -17,14 +17,16 @@ import System.FilePath import Data.List (nub) import Data.Maybe (mapMaybe, fromJust)+import Data.Either+import qualified Data.ByteString.Lazy.Char8 as BC+import qualified Data.ByteString.Lazy as ByteString+import qualified Codec.Compression.GZip as GZip data WikiArticle = WikiArticleHTML { waTitle :: String, waContent :: String } | WikiArticleSRC { waTitle :: String, waContent :: String } deriving (Show, Ord, Eq) sanitizeFileName :: FilePath -> FilePath-sanitizeFileName cs = map (unPercent) $ urlEncode cs- where- unPercent c = if (c == '%' || c == ':') then 'X' else c+sanitizeFileName cs = makeValid cs isArticleURL :: URL -> Bool isArticleURL (URL (Absolute (Host (HTTP False) xs Nothing)) ph []) = (xs =~ ".*en[.]wikipedia.org$") && (ph =~ "wiki/[^/]+$" )@@ -134,6 +136,11 @@ fetchArticle :: URL -> IO WikiArticle fetchArticle xs = do- rsp <- Network.HTTP.simpleHTTP (getRequest (exportURL xs))+ rsp <- Network.HTTP.simpleHTTP (insertHeader HdrAcceptEncoding "identity" $ getRequest (exportURL xs))+ -- rsp <- Network.HTTP.simpleHTTP (getRequest (exportURL xs)) contents <- (getResponseBody rsp)- return (WikiArticleHTML (articleURL2Title xs) contents)+ let header = either (\_ -> []) rspHeaders rsp+ let fincts = case lookupHeader HdrContentEncoding header of + Nothing -> contents+ Just mm -> if mm == "gzip" then BC.unpack (GZip.decompress (BC.pack contents)) else contents+ return (WikiArticleHTML (articleURL2Title xs) fincts)
src/Wiki4e/Commands/Articles.hs view
@@ -22,10 +22,8 @@ import Wiki4e.Commands.Fetching wiki4e_sanitizeArticle :: [String] -> FilePath -> FilePath -> IO ()-wiki4e_sanitizeArticle alnk inf ouf = withFile inf ReadMode (\hi -> do- hSetEncoding hi utf8- withFile ouf WriteMode (\ho -> do - hSetEncoding ho utf8+wiki4e_sanitizeArticle alnk inf ouf = withBinaryFile inf ReadMode (\hi -> do+ withBinaryFile ouf WriteMode (\ho -> do c <- hGetContents hi let a = sanitizeArticle alnk $ WikiArticleHTML "" c hPutStr ho $ waContent a
src/Wiki4e/Commands/Common.hs view
@@ -33,7 +33,6 @@ readFileUTF8 :: FilePath -> IO String readFileUTF8 x = do h <- openBinaryFile x ReadMode- hSetEncoding h utf8 c <- hGetContents h forceList c `seq` hClose h return c
− src/wiki4e-mkepub-cache.hs
@@ -1,45 +0,0 @@-import Text.HTML.TagSoup-import System.Environment-import System.Directory-import System.FilePath-import Data.Maybe (catMaybes)-import Control.Monad (liftM)-import System.IO -import Network.URL-import Wiki4e.Commands-import Network.Wikipedia (isArticleURL)--defaultEbookName = "Wikipedia_Articles_From_Cache"--main = do- name <- getProgName- args <- getArgs- case args of - [] -> cache2epub defaultEbookName- ['-':_] -> usageHelp name- ['/':_] -> usageHelp name- [xs] -> cache2epub xs- _ -> usageHelp name--usageHelp name = putStrLn $ "Usage: " ++ name ++ " [<Title Name of new e-book>]"--cache2epub bookName = do- config <- wiki4e_initConfig- arts <- wiki4e_listCacheURLs config- putStrLn "# STAGE 1/5 - Verify Articles..."- wiki4e_fetchArticles config arts- putStrLn "# STAGE 2/5 - Sanitize Articles..."- wiki4e_sanitizeArticles config arts- putStrLn "# STAGE 3/5 - Inspect Articles for Images..."- imgs <- wiki4e_listArticlesImages config arts- putStrLn "# STAGE 4/5 - Download Images..."- wiki4e_fetchImages config imgs- putStrLn "# STAGE 5/5 - Constructing EPUB..."- wiki4e_createEpub config bookName arts imgs- putStrLn "Done."---- | It is expected that all articles are from english wikipedia-wiki4e_listCacheURLs :: Wiki4eConfig -> IO [URL]-wiki4e_listCacheURLs config = do- files <- liftM (filter (\(c:_) -> c /= '.')) $ getDirectoryContents (w4confDirFetch config)- return $ catMaybes $ map (importURL) (map (\x -> "http://en.wikipedia.org/wiki/"++x) files)
− src/wiki4e-mkepub-firefox.hs
@@ -1,43 +0,0 @@-import Text.HTML.TagSoup-import System.Environment-import System.Directory-import System.FilePath-import Data.List (nub)-import System.IO -import Network.URL-import Wiki4e.Commands-import Network.Wikipedia (isArticleURL)-import Web.Firefox--defaultEbookName = "Wikipedia_Articles_From_Firefox"--main = do- name <- getProgName- args <- getArgs- case args of - [] -> firefox2epub defaultEbookName- ['-':_] -> usageHelp name- ['/':_] -> usageHelp name- [xs] -> firefox2epub xs- _ -> usageHelp name--usageHelp name = putStrLn $ "Usage: " ++ name ++ " [<Title Name of new e-book>]"--firefox2epub bookName = do- arts <- wiki4e_listFirefoxURLs- config <- wiki4e_initConfig- putStrLn "# STAGE 1/4 - Download Articles..."- wiki4e_fetchArticles config arts- putStrLn "# STAGE 2/4 - Sanitize Articles..."- wiki4e_sanitizeArticles config arts- putStrLn "# STAGE 3/4 - Download Images..."- imgs <- wiki4e_listArticlesImages config arts- wiki4e_fetchImages config imgs- putStrLn "# STAGE 4/4 - Constructing EPUB..."- wiki4e_createEpub config bookName arts imgs- putStrLn "Done."--wiki4e_listFirefoxURLs :: IO [URL]-wiki4e_listFirefoxURLs = do- xs <- listAllHistoryURLs- return $ nub $ filter isArticleURL xs
wikipedia4epub.cabal view
@@ -1,5 +1,5 @@ Name: wikipedia4epub-Version: 0.0.6+Version: 0.0.7 License: BSD3 License-File: LICENSE Homepage: http://rampa.sk/static/wikipedia4epub.html@@ -39,6 +39,7 @@ tagsoup >= 0.8, HTTP, zip-archive,+ zlib, xml, Cabal >= 1.5 && < 1.9 Hs-Source-Dirs: src@@ -55,14 +56,7 @@ Network.Wikipedia Web.Firefox -Executable wiki4e-mkepub-firefox- Hs-Source-Dirs: src- Main-Is: wiki4e-mkepub-firefox.hs- Executable wiki4e-mkepub-subtree Hs-Source-Dirs: src Main-Is: wiki4e-mkepub-subtree.hs -Executable wiki4e-mkepub-cache- Hs-Source-Dirs: src- Main-Is: wiki4e-mkepub-cache.hs