archiver 0.6.2 → 0.6.2.1
raw patch · 2 files changed
+22/−12 lines, 2 files
Files
- Network/URL/Archiver.hs +17/−7
- archiver.cabal +5/−5
Network/URL/Archiver.hs view
@@ -3,8 +3,10 @@ import Control.Monad (when, unless, void) import Data.Char (isAlphaNum, isAscii) import Data.List (isPrefixOf)-import Network.HTTP (getRequest, simpleHTTP)-import Network.URI (isURI)+import Data.Maybe (fromJust)+import Network.Browser (browse, formToRequest, request, Form(..))+import Network.HTTP (getRequest, simpleHTTP, RequestMethod(POST))+import Network.URI (isURI, parseURI) import System.Random (getStdGen, randomR) import Text.Printf (printf) @@ -16,7 +18,7 @@ checkArchive :: String -- ^ email for WebCite to send status to -> String -- ^ URL to archive -> IO ()-checkArchive email url = when (isURI url) (alexaToolbar url >> webciteArchive email url >> internetArchiveLive url >> wikiwixArchive url >> googleSearch url)+checkArchive email url = when (isURI url) (alexaToolbar url >> webciteArchive email url >> internetArchiveLive url >> wikiwixArchive url >> googleSearch url >> archiveisArchive url) {- | Request <http://www.webcitation.org> to copy a supplied URL; WebCite does on-demand archiving, unlike Alexa/Internet Archive, and so in practice this is the most useful function. This function throws away any return status from WebCite (which may be changed@@ -29,11 +31,14 @@ webciteArchive email url = unless ("http://www.webcitation.org" `isPrefixOf` url) $ pingURL ("http://www.webcitation.org/archive?url="++url++ "&email="++email) --- | Request a URL through Internet Archive's live Internet mirror; this is completely speculative and may result in no archiving.--- This method is a guess based on my use of their mirror and a banner that is sometimes inserted;--- see <http://www.archive.org/post/380853/virus-operating-in-internet-archive>+-- | Request a URL through Internet Archive's on-demand archiving URL.+--+-- This also does a backup archive attempt through the live Internet mirror;+-- this is completely speculative and may result in no archiving.+-- This method is a guess based on my use of their mirror and a banner that is sometimes inserted;+-- see <http://www.archive.org/post/380853/virus-operating-in-internet-archive> internetArchiveLive :: String -> IO ()-internetArchiveLive url = pingURL ("http://liveweb.archive.org/"++url)+internetArchiveLive url = pingURL("http://web.archive.org/save/"++url) >> pingURL ("http://liveweb.archive.org/"++url) -- | Ping Alexa's servers like the Toolbar does; this may or may not result in any archiving. alexaToolbar :: String -> IO ()@@ -45,6 +50,11 @@ wikiwixArchive :: String -> IO () wikiwixArchive url = pingURL ("http://archive.wikiwix.com/cache/?url="++url)++-- | <http://blog.archive.is/post/45031162768/can-you-recommend-the-best-method-script-so-i-may-batch>+archiveisArchive :: String -> IO ()+archiveisArchive url = do let archiveform = Form POST (fromJust $ parseURI "http://archive.is/submit/") [("url", url), ("submit", "")]+ void $ browse $ request $ formToRequest archiveform -- can't hurt to let Google know it exists googleSearch :: String -> IO ()
archiver.cabal view
@@ -1,14 +1,14 @@ name: archiver-version: 0.6.2+version: 0.6.2.1 license: BSD3 license-file: LICENSE author: Gwern-maintainer: Gwern <gwern0@gmail.com>+maintainer: Gwern <gwern@gwern.net> category: Documentation, Network synopsis: Archive supplied URLs in WebCite & Internet Archive-description: archiver is a daemon which will process a specified text file,+description: `archiver` is a daemon which will process a specified text file, each line of which is a URL, and will (randomly) one by one request that the URLs be archived or spidered by <http://www.webcitation.org>, <http://www.archive.org>, and <http://www.wikiwix.com> for future reference.@@ -28,8 +28,8 @@ tested-with: GHC==6.12.1 source-repository head- type: darcs- location: http://community.haskell.org/~gwern/archiver/+ type: git+ location: git://github.com/gwern/archiver-bot.git Library exposed-modules: Network.URL.Archiver