archiver 0.2 → 0.3

raw patch · 2 files changed

+18/−8 lines, 2 filesdep +randomPVP ok

version bump matches the API change (PVP)

Dependencies added: random

API changes (from Hackage documentation)

Files

• archiver.cabal +3/−3
• archiver.hs +15/−5

@@ -1,5 +1,5 @@ name:                archiver-version:             0.2+version:             0.3  license:             BSD3 license-file:        LICENSE@@ -9,7 +9,7 @@ category:            Documentation, Network synopsis:            Archive supplied URLs in WebCite & Internet Archive description:         archiver is a daemon which will process a specified text file,-                     each line of which is a URL, and will one by one request that+                     each line of which is a URL, and will (randomly) one by one request that                      the URLs be archived or spidered by <http://www.webcitation.org> and                      <http://www.archive.org> for future reference.                      .@@ -34,4 +34,4 @@  Executable archiver            main-is:       archiver.hs-           build-depends: base>=4 && < 5, bytestring+           build-depends: base>=4 && < 5, bytestring, random

@@ -2,11 +2,12 @@ import Control.Concurrent (threadDelay) import qualified Control.Exception as CE (catch, IOException) import Control.Monad (liftM, when)-import Data.List (nub, sort)+import Data.List (delete, nub, sort) import Data.Maybe (fromMaybe) import Network.HTTP (getRequest, simpleHTTP) import System.Environment (getArgs)-import qualified Data.ByteString.Char8 as B (break, intercalate, length, readFile, singleton, split, unpack, writeFile, ByteString)+import qualified Data.ByteString.Char8 as B (count, intercalate, readFile, singleton, split, unpack, writeFile, ByteString)+import System.Random  import Network.URL.Archiver (checkArchive) @@ -24,12 +25,12 @@                                         threadDelay 90000000 >> archivePage usr file                              Right _ -> do -- we have access to the WWW, it seems. proceeding with mission!                                           contents <- B.readFile file-                                          let (url,rest) = B.break (=='\n') contents+                                          (url,rest) <- splitRandom contents                                           checkArchive email (B.unpack url)                                           print url                                           -- banned >=100 requests/hour; choke it-                                          threadDelay 20000000 -- ~20 seconds-                                          when (B.length rest /= 0) (writePages file url >> archivePage usr file) -- drop to get rid of leading \n+                                          threadDelay 26000000 -- ~26 seconds+                                          when (length rest /= 0) (writePages file url >> archivePage usr file) -- drop to get rid of leading \n                                               where email = fromMaybe "nobody@mailinator.com" usr  -- re-reads a possibly modified 'file' from disk, removes the archived URL from it, and writes it back out for 'archivePage' to read immediately@@ -38,3 +39,12 @@                           let new = nub $ sort original                           let final = B.intercalate (B.singleton '\n') $ filter (not . (== done)) new                           B.writeFile file final++-- pick a random entry in the list+splitRandom :: B.ByteString -> IO (B.ByteString, [B.ByteString])+splitRandom s = do let ss = B.split '\n' s+                   let l  = B.count '\n' s+                   i <- getStdRandom (randomR (0,l))+                   let randpick = ss !! i+                   let removed = Data.List.delete randpick ss+                   return (randpick, removed)