archiver 0.3 → 0.3.1

raw patch · 3 files changed

+17/−11 lines, 3 filesdep +containersPVP ok

version bump matches the API change (PVP)

Dependencies added: containers

API changes (from Hackage documentation)

Files

• Network/URL/Archiver.hs +2/−2
• archiver.cabal +6/−3
• archiver.hs +9/−6

@@ -8,7 +8,7 @@   -- | Error check the URL and then archive it using 'webciteArchive' and 'alexaArchive'-checkArchive :: String -- ^ email for WebCite to send status to +checkArchive :: String -- ^ email for WebCite to send status to                 -> String -- ^ URL to archive                 -> IO () checkArchive email url = when (isURI url) (webciteArchive email url >> alexaArchive url)@@ -16,7 +16,7 @@ {- | Request <http://www.webcitation.org> to copy a supplied URL; WebCite does on-demand archiving, unlike Alexa/Internet Archive,    and so in practice this is the most useful function. This function throws away any return status from WebCite (which may be changed    in the future), so it is suggested that one test with a valid email address.- +    /Warning!/ WebCite has throttling mechanisms; if you request more than 100 URLs per hour, your IP may be banned! It is    suggested that one sleep for \~30 seconds between each URL request. -} webciteArchive :: String -> String -> IO ()

@@ -1,5 +1,5 @@ name:                archiver-version:             0.3+version:             0.3.1  license:             BSD3 license-file:        LICENSE@@ -16,7 +16,10 @@                      Because the interface is a simple text file, this can be combined                      with other scripts; for example, a script using Sqlite to extract                      visited URLs from Firefox, or a program extracting URLs from Pandoc-                     documents.+                     documents. (See <http://www.gwern.net/Archiving%20URLs.html>.)+                     .+                     For explanation of the derivation of the code in `Network.URL.Archiver`,+                     see <http://www.gwern.net/haskell/Wikipedia%20Archive%20Bot.html>.  build-type:          Simple Cabal-Version:       >= 1.6@@ -34,4 +37,4 @@  Executable archiver            main-is:       archiver.hs-           build-depends: base>=4 && < 5, bytestring, random+           build-depends: base>=4 && < 5, containers, bytestring, random

@@ -2,9 +2,11 @@ import Control.Concurrent (threadDelay) import qualified Control.Exception as CE (catch, IOException) import Control.Monad (liftM, when)-import Data.List (delete, nub, sort)+import Data.List (delete)+import qualified Data.Set as S (fromList, toList) import Data.Maybe (fromMaybe) import Network.HTTP (getRequest, simpleHTTP)+import Network.URI (isURI) import System.Environment (getArgs) import qualified Data.ByteString.Char8 as B (count, intercalate, readFile, singleton, split, unpack, writeFile, ByteString) import System.Random@@ -26,17 +28,18 @@                              Right _ -> do -- we have access to the WWW, it seems. proceeding with mission!                                           contents <- B.readFile file                                           (url,rest) <- splitRandom contents-                                          checkArchive email (B.unpack url)-                                          print url-                                          -- banned >=100 requests/hour; choke it-                                          threadDelay 26000000 -- ~26 seconds+                                          let url' = B.unpack url+                                          when (isURI url') $ do checkArchive email url'+                                                                 print url'+                                                                 -- banned >=100 requests/hour; choke it+                                                                 threadDelay 26000000 -- ~26 seconds                                           when (length rest /= 0) (writePages file url >> archivePage usr file) -- drop to get rid of leading \n                                               where email = fromMaybe "nobody@mailinator.com" usr  -- re-reads a possibly modified 'file' from disk, removes the archived URL from it, and writes it back out for 'archivePage' to read immediately writePages :: FilePath -> B.ByteString -> IO () writePages file done = do original <- liftM (B.split '\n') $ B.readFile file-                          let new = nub $ sort original+                          let new =  S.toList $ S.fromList original                           let final = B.intercalate (B.singleton '\n') $ filter (not . (== done)) new                           B.writeFile file final