wikipedia4epub 0.0.5 → 0.0.6

raw patch · 5 files changed

+20/−16 lines, 5 files

Files

• src/Network/Wikipedia.hs +7/−6
• src/Wiki4e/Commands/Articles.hs +6/−5
• src/wiki4e-mkepub-cache.hs +5/−4
• src/wiki4e-mkepub-subtree.hs +1/−0
• wikipedia4epub.cabal +1/−1

@@ -24,10 +24,10 @@ sanitizeFileName :: FilePath -> FilePath sanitizeFileName cs = map (unPercent) $ urlEncode cs     where-      unPercent c = if (c == '%') then 'X' else c+      unPercent c = if (c == '%' || c == ':') then 'X' else c  isArticleURL :: URL -> Bool-isArticleURL (URL (Absolute (Host (HTTP False) xs Nothing)) ph []) = (xs =~ ".*en[.]wikipedia.org$") && (ph =~ "wiki/[^:/]+$" )+isArticleURL (URL (Absolute (Host (HTTP False) xs Nothing)) ph []) = (xs =~ ".*en[.]wikipedia.org$") && (ph =~ "wiki/[^/]+$" ) isArticleURL _ = False  articleURL2Title :: URL -> String@@ -35,6 +35,7 @@                    | otherwise      = ""  articleRelURL2Title :: String -> String +articleRelURL2Title ('#':xs) = '#':xs articleRelURL2Title x = case importURL ("http://en.wikipedia.org"++x) of                            Nothing -> ""                           Just  u -> articleURL2Title u@@ -45,8 +46,8 @@ getArticleImages :: WikiArticle -> [URL] getArticleImages x = let inTags = parseTags (waContent x)                          imgTags = filter (isTagOpenName "img") inTags-                         imgSrcs = filter (isArticleImgURL) $ map (fromAttrib "src") imgTags -                     in mapMaybe importURL imgSrcs+                         imgSrcs = filter (isArticleImgURL) (map (fromAttrib "src") imgTags)+                     in imgSrcs `seq` mapMaybe importURL imgSrcs  getArticleLinks :: WikiArticle -> [URL] getArticleLinks xs = let inTags = parseTags (waContent xs)@@ -86,7 +87,7 @@                                    | otherwise     = TagOpen s (removeStyleAttr $ removeEmptyAttr ys)      processAttrs t = t      removeEmptyAttr xs = filter (not . null . snd) xs -     removeStyleAttr xs = filter (\x -> (fst x) `notElem` ["style", "id", "class"]) xs+     removeStyleAttr xs = filter (\x -> (fst x) `notElem` ["style", "class"]) xs   filterAllTags tgs xs = filter (not . isTagComment) $ foldr (filterTags) xs tgs @@ -104,7 +105,7 @@ procHrefTags alnk (x:xs) | isTagOpenName "a" x = let relP   = fromAttrib "href" x                                                      title   = articleRelURL2Title relP                                                      imgOk  = (TagOpen "a" [("href",title)]):(procHrefTags alnk xs)-                                                     isInBook = elem (title) alnk +                                                     isInBook = elem (title) alnk || ((length title) > 0 && (head title == '#'))                                                      imgNok = let pre  = takeWhile (not . isTagCloseName "a") xs                                                                   post = tail $ dropWhile (not . isTagCloseName "a") xs                                                               in pre ++ (procHrefTags alnk post)

@@ -53,14 +53,15 @@ wiki4e_listArticleImages :: FilePath -> IO [URL] wiki4e_listArticleImages x = do           c <- readFileUTF8 x-          return $ nub $ getArticleImages (WikiArticleHTML "" c)+          let imgs = nub (getArticleImages (WikiArticleHTML "" c))+          putStrLn ((show $ length imgs) ++ " images at "++x)+          return $! (imgs `seq` imgs)  -- | Method expects already sanitized articles wiki4e_listArticlesImages :: Wiki4eConfig -> [URL] -> IO [URL]-wiki4e_listArticlesImages config urls = do -     let files = wiki4e_getArticleFiles config urls-     images <- mapM (wiki4e_listArticleImages) files-     return $ nub $ concat images+wiki4e_listArticlesImages config urls = do+     images <- mapM (wiki4e_listArticleImages) (wiki4e_getArticleFiles config urls)+     return $! nub (images `seq` concat images)  wiki4e_crawlArticlesLinks :: Wiki4eConfig -> [URL] -> Int -> IO [URL] wiki4e_crawlArticlesLinks _ _  0 = return []

@@ -26,14 +26,15 @@ cache2epub bookName = do   config <- wiki4e_initConfig   arts <- wiki4e_listCacheURLs config-  putStrLn "# STAGE 1/4 - Verify Articles..."+  putStrLn "# STAGE 1/5 - Verify Articles..."   wiki4e_fetchArticles config arts-  putStrLn "# STAGE 2/4 - Sanitize Articles..."+  putStrLn "# STAGE 2/5 - Sanitize Articles..."   wiki4e_sanitizeArticles config arts-  putStrLn "# STAGE 3/4 - Download Images..."+  putStrLn "# STAGE 3/5 - Inspect Articles for Images..."   imgs <- wiki4e_listArticlesImages config arts+  putStrLn "# STAGE 4/5 - Download Images..."   wiki4e_fetchImages config imgs-  putStrLn "# STAGE 4/4 - Constructing EPUB..."+  putStrLn "# STAGE 5/5 - Constructing EPUB..."   wiki4e_createEpub config bookName arts imgs   putStrLn "Done." 

@@ -25,6 +25,7 @@ subtree2epub cs l = do   let u = fromJust $ importURL cs   config <- wiki4e_initConfig+  putStrLn $ "# INFO - download depth is hardcoded value = " ++ (show defaultDepth)   putStrLn $ "# STAGE 1/4 - Fetch starting article: " ++ (exportURL u)   arts <- wiki4e_crawlArticlesLinks config [u] l   putStrLn $ "# STAGE 2/4 - Fetch children articles: " ++ (show (length arts))

@@ -1,5 +1,5 @@ Name:            wikipedia4epub-Version:         0.0.5+Version:         0.0.6 License:         BSD3 License-File:    LICENSE Homepage:        http://rampa.sk/static/wikipedia4epub.html