packages feed

hs-hath-1.3.0: src/Hath/Gallery/Downloader.hs

{-# LANGUAGE DataKinds #-}

-- | The gallery downloader: works through the galleries queued for this client
-- on the H\@H site, into the download directory, as the official
-- GalleryDownloader does.
module Hath.Gallery.Downloader ( runGalleryDownloader, downloadQueue, anotherRound ) where

import           Colog                   ( Message )
import           Colog.Polysemy          ( Log )

import           Control.Concurrent      ( threadDelay )

import qualified Data.ByteString.Char8   as BS
import           Data.String.Interpolate ( i )

import           Hath.AtomicFile         ( writeFileAtomic )
import           Hath.Config             ( ClientConfig(..) )
import           Hath.Download           ( fetchFile )
import           Hath.Error              ( HathError )
import           Hath.Gallery            ( GalleryFile(..)
                                         , GalleryMetadata(..)
                                         , fallbackDirectoryName
                                         , galleryDirectoryName
                                         )
import           Hath.Log                ( logInfo, logWarning )
import           Hath.Protocol           ( sha1Hex )
import           Hath.RPC                ( RPC
                                         , galleryFileSource
                                         , nextGallery
                                         , reportDownloadFailures
                                         , runRPCIO
                                         )
import           Hath.Settings           ( HathSettings(..) )
import           Hath.Stats              ( StatsEnv, addDlBytes, incDlFile, incDlTask )

import qualified Network.HTTP.Client     as HC

import           Polysemy
import           Polysemy.Error          ( Error, catch )
import           Polysemy.Reader         ( Reader, ask, runReader )

import           Relude                  hiding ( Reader, ask, runReader )

import           System.Directory        ( createDirectoryIfMissing, removeFile )
import           System.FilePath         ( (<.>), (</>) )
import           System.IO               ( hPutStrLn )

import           UnliftIO                ( tryAny )

-- | Work through the download queue each time the server asks
-- (@start_downloader@ fills @requested@). A request that comes while a run is
-- under way is dropped, as the official client ignores it
-- (HentaiAtHomeClient.startDownloader): the run goes on until the queue is
-- empty anyway.
runGalleryDownloader :: ClientConfig -> StatsEnv -> TVar HathSettings -> MVar () -> IO Void
runGalleryDownloader config stats settings requested = infinitely $ do
  takeMVar requested
  tryAny (downloadQueue config stats settings) >>= \case
    Left e           -> failed (show e)
    Right (Left e)   -> failed (show e)
    Right (Right ()) -> pass
  void $ tryTakeMVar requested
  where
    failed e = hPutStrLn stderr $ "Gallery downloader error: " <> e

-- | Download the queued galleries one after another (GalleryDownloader.java:58-126).
-- Each fetchqueue after the first marks the gallery before it done, whether or
-- not all of its files came, and brings the next one. The run ends when the
-- queue is empty, when the server refuses the request, or when we refuse its list.
downloadQueue :: ClientConfig -> StatsEnv -> TVar HathSettings -> IO (Either HathError ())
downloadQueue config stats settings = runRPCIO config $ runReader stats $ go Nothing
  where
    go done = nextGallery done >>= \case
      Left "NO_PENDING_DOWNLOADS" -> logInfo "No more galleries to download"
      Left reason -> logWarning [i|The gallery downloader stops: #{reason}|]
      Right gallery -> do
        incDlTask
        current <- embed @IO (readTVarIO settings)
        galleryDirectory (maxFilenameLength current) gallery >>= \case
          Nothing        -> logWarning
            [i|Unexpected download location for gallery #{galleryID gallery}; the gallery downloader stops|]
          Just directory -> do
            downloadGallery (maxAllowedFileSize current) directory gallery
            go (Just gallery)

-- | Make the gallery's directory and give its path. If the file system will
-- not take the name, the official client suspects it cannot store Unicode and
-- falls back on one without the title (GalleryDownloader.java:287-298).
-- Nothing if the name would leave the download directory.
galleryDirectory :: Members '[ Reader ClientConfig, Log Message, Embed IO ] r
                 => Int
                 -> GalleryMetadata
                 -> Sem r (Maybe FilePath)
galleryDirectory maxLength gallery = do
  downloads <- toString . downloadDir <$> ask
  forM (galleryDirectoryName maxLength gallery) $ \name -> do
    let path = downloads </> toString name
    embed (tryAny (createDirectoryIfMissing True path)) >>= \case
      Right () -> pure path
      Left e   -> do
        logWarning
          [i|Could not create the gallery directory "#{name}" (#{e}); using one without the title|]
        let fallback = downloads </> toString (fallbackDirectoryName gallery)
        embed $ createDirectoryIfMissing True fallback
        pure fallback

-- | Download a gallery's files, write its @galleryinfo.txt@ if all of them came
-- (GalleryDownloader.java:132-150), and report the sources that failed.
downloadGallery :: Members '[ RPC, Reader StatsEnv, Log Message, Error HathError, Embed IO ] r
                => Int
                -> FilePath
                -> GalleryMetadata
                -> Sem r ()
downloadGallery maxSize directory gallery = do
  logInfo [i|Starting download of gallery: #{galleryTitle gallery}|]
  ( complete, failures ) <- downloadFiles maxSize directory gallery
  if complete
    then do
      logInfo [i|Finished download of gallery: #{galleryTitle gallery}|]
      whenLeftM_
        (embed
         $ tryAny
         $ writeFileAtomic (directory </> "galleryinfo.txt") (galleryInformation gallery))
        $ \e -> logWarning [i|Could not write galleryinfo.txt: #{e}|]
    else logWarning [i|Permanently failed downloading gallery: #{galleryTitle gallery}|]
  -- the official client goes on whether or not the report gets through
  reportDownloadFailures failures `catch` \(e :: HathError)
    -> logWarning [i|Could not report download failures: #{e}|]

-- | Fetch the gallery's files into @directory@, going through those still
-- missing again while 'anotherRound' allows (GalleryDownloader.java:71-118).
-- True if all of them are there in the end; and the sources that failed, as
-- @host-fileindex-xres@.
downloadFiles :: Members '[ RPC, Reader StatsEnv, Log Message, Error HathError, Embed IO ] r
              => Int
              -> FilePath
              -> GalleryMetadata
              -> Sem r ( Bool, [ Text ] )
downloadFiles maxSize directory gallery = go 0 0 [] [ ( file, 1 ) | file <- files ]
  where
    files = galleryFileList gallery

    -- the files still missing, each with the number its next attempt gets
    go rounds failed failures missing
      | null missing = pure ( True, failures )
      | not (anotherRound (length files) rounds failed) = pure ( False, failures )
      | otherwise = do
        outcomes <- mapM (uncurry (ensureFile maxSize directory gallery)) missing
        let stillMissing
              = [ ( file, attempt + 1 ) | ( ( file, attempt ), Failed _ ) <- zip missing outcomes ]
        go
          (rounds + 1)
          (failed + length stillMissing)
          (failures <> [ source | Failed (Just source) <- outcomes ])
          stillMissing

-- | Whether to go through a gallery's missing files once more, after this many
-- rounds and failed downloads (GalleryDownloader.java:74).
anotherRound :: Int -> Int -> Int -> Bool
anotherRound fileCount rounds failed = rounds < 9 && failed < fileCount * 2

data Outcome
  = Present
    -- ^ it was there already, intact
  | Downloaded
  | Failed !(Maybe Text)
-- ^ with the source that failed, if the server named one

-- | Get one file into the directory unless it is there already, intact
-- (GalleryDownloader.java:392-474): ask the server where to download it from,
-- this being attempt @attempt@ at it, and check what comes. The official
-- client then waits 1 s after a download, 5 s after a failure.
ensureFile :: Members '[ RPC, Reader StatsEnv, Log Message, Error HathError, Embed IO ] r
           => Int
           -> FilePath
           -> GalleryMetadata
           -> GalleryFile
           -> Int
           -> Sem r Outcome
ensureFile maxSize directory gallery file attempt = embed (intactAt path file) >>= \case
  True  -> pure Present
  False -> do
    outcome <- download
    embed $ threadDelay $ case outcome of
      Downloaded -> 1000000
      _          -> 5000000
    pure outcome
  where
    name = BS.unpack (galleryFileName file) <.> BS.unpack (galleryFileExt file)

    path = directory </> name

    download = do
      source <- galleryFileSource gallery file attempt `catch` \(e :: HathError)
        -> Nothing <$ logWarning [i|dlfetch failed for #{name}: #{e}|]
      case HC.parseRequest . BS.unpack =<< source of
        Nothing      -> do
          logWarning [i|No source to download #{name} from|]
          pure (Failed Nothing)
        Just request -> embed (fetchBytes maxSize request) >>= \case
          Right bytes
            | hasExpectedHash file bytes -> do
              embed $ writeFileAtomic path bytes
              incDlFile
              addDlBytes (BS.length bytes)
              logInfo [i|Downloaded #{name} of gallery #{galleryID gallery}|]
              pure Downloaded
          Right _     -> failedFrom request "its SHA-1 is not the one listed"
          Left why    -> failedFrom request why

    failedFrom request (why :: Text) = do
      logWarning [i|Failed to download #{name} (attempt #{attempt}): #{why}|]
      pure $ Failed $ Just [i|#{HC.host request}-#{galleryFileIndex file}-#{galleryFileXRes file}|]

-- | Whether the file is there, not empty, and has the expected hash if one is
-- known. One that is not is removed, as by the official client.
intactAt :: FilePath -> GalleryFile -> IO Bool
intactAt path file = do
  content <- fromRight "" <$> tryAny (readFileBS path)
  let intact = not (BS.null content) && hasExpectedHash file content
  unless intact $ void $ tryAny $ removeFile path
  pure intact

-- | A file generated on demand has no known hash; like the official client, accept it.
hasExpectedHash :: GalleryFile -> ByteString -> Bool
hasExpectedHash file content = maybe True (== sha1Hex content) (galleryFileHash file)

fetchBytes :: Int -> HC.Request -> IO (Either Text ByteString)
fetchBytes maxSize request = do
  chunks <- newIORef []
  -- a chunk at offset 0 begins another attempt, so what came before is dropped
  let keep 0 chunk = writeIORef chunks [ chunk ]
      keep _ chunk = modifyIORef' chunks (chunk :)
  fetched <- fetchFile maxSize request keep
  forM fetched $ \_ -> BS.concat . reverse <$> readIORef chunks