packages feed

hs-hath 1.2.0 → 1.3.0

raw patch · 113 files changed

+7475/−9939 lines, 113 filesdep +QuickCheckdep −asyncdep −co-log-coredep −constraintsdep ~aesondep ~basedep ~base64setup-changedPVP ok

version bump matches the API change (PVP)

Dependencies added: QuickCheck

Dependencies removed: async, co-log-core, constraints, contravariant, exceptions, extra, ghc-compact, ghc-prim, hex-text, http-media, mason, monad-control, monad-loops, mtl, random, reflection, safe-exceptions, servant, servant-client, servant-client-core, servant-server, streaming, suspend, timers

Dependency ranges changed: aeson, base, base64, bytestring, containers, crypton-connection, directory, filepath, http-conduit, network, optparse-applicative, recv, resourcet, stm, temporary, text, tls, tls-session-manager, transformers, unordered-containers, warp

API changes (from Hackage documentation)

- CLI: Options :: !FilePath -> !Maybe Text -> !Maybe Text -> !Bool -> !Bool -> !Bool -> !Bool -> !Bool -> Options
- CLI: [optCachePath] :: Options -> !Maybe Text
- CLI: [optConfigPath] :: Options -> !FilePath
- CLI: [optDisableRateLimit] :: Options -> !Bool
- CLI: [optDownloadDir] :: Options -> !Maybe Text
- CLI: [optShowVersion] :: Options -> !Bool
- CLI: [optSkipStartupVerify] :: Options -> !Bool
- CLI: [optTrustProxyHeaders] :: Options -> !Bool
- CLI: [optValidateConfig] :: Options -> !Bool
- CLI: applyOptionsToConfig :: Options -> ClientConfig -> ClientConfig
- CLI: data Options
- CLI: instance GHC.Show.Show CLI.Options
- CLI: parseOptions :: IO Options
- FileVerification: VerificationCorrupted :: {-# UNPACK #-} !Text -> VerificationResult
- FileVerification: VerificationMissing :: {-# UNPACK #-} !Text -> VerificationResult
- FileVerification: VerificationOK :: VerificationResult
- FileVerification: VerificationStats :: {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !NominalDiffTime -> VerificationStats
- FileVerification: [GetLastVerificationTime] :: forall {k} (m :: k). FileVerification m (Maybe UTCTime)
- FileVerification: [SetLastVerificationTime] :: forall {k} (m :: k). UTCTime -> FileVerification m ()
- FileVerification: [VerifyAllFilesE] :: forall {k} (m :: k). FileVerification m VerificationStats
- FileVerification: [VerifyRandomFileE] :: forall {k} (m :: k). FileVerification m (Maybe VerificationResult)
- FileVerification: [corruptedCount] :: VerificationStats -> {-# UNPACK #-} !Int
- FileVerification: [deletedCount] :: VerificationStats -> {-# UNPACK #-} !Int
- FileVerification: [missingCount] :: VerificationStats -> {-# UNPACK #-} !Int
- FileVerification: [verificationTime] :: VerificationStats -> {-# UNPACK #-} !NominalDiffTime
- FileVerification: [verifiedCount] :: VerificationStats -> {-# UNPACK #-} !Int
- FileVerification: data FileVerification (m :: k) a
- FileVerification: data VerificationResult
- FileVerification: data VerificationStats
- FileVerification: instance GHC.Base.Monoid FileVerification.VerificationStats
- FileVerification: instance GHC.Base.Semigroup FileVerification.VerificationStats
- FileVerification: instance GHC.Classes.Eq FileVerification.VerificationResult
- FileVerification: instance GHC.Classes.Eq FileVerification.VerificationStats
- FileVerification: instance GHC.Generics.Generic FileVerification.VerificationResult
- FileVerification: instance GHC.Generics.Generic FileVerification.VerificationStats
- FileVerification: instance GHC.Show.Show FileVerification.VerificationResult
- FileVerification: instance GHC.Show.Show FileVerification.VerificationStats
- FileVerification: minVerificationIntervalSeconds :: NominalDiffTime
- FileVerification: runFileVerification :: forall (r :: EffectRow) a. Member (Embed IO) r => Connection -> TVar (Maybe UTCTime) -> (((FileVerification :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- FileVerification: runFileVerificationIO :: Connection -> TVar (Maybe UTCTime) -> ('[FileVerification :: (Type -> Type) -> Type -> Type, Embed IO, Final IO] @> a) -> IO a
- FileVerification: shouldVerifyFile :: UTCTime -> UTCTime -> Bool
- FileVerification: verificationCooldownSeconds :: Int64
- FileVerification: verifyAllFiles :: Connection -> IO VerificationStats
- FileVerification: verifyFile :: FileRecord -> VerificationResult
- FileVerification: verifyRandomFile :: Connection -> TVar (Maybe UTCTime) -> IO (Maybe VerificationResult)
- HathNetwork.Genesis: [FetchCertificate] :: forall {k} (m :: k). Genesis m (CertificateChain, PrivKey)
- HathNetwork.Genesis: [FetchSettings] :: forall {k} (m :: k). Genesis m HathSettings
- HathNetwork.Genesis: data Genesis (m :: k) a
- HathNetwork.Genesis: fetchCertificate :: forall (r :: EffectRow). Member (Genesis :: (Type -> Type) -> Type -> Type) r => Sem r (CertificateChain, PrivKey)
- HathNetwork.Genesis: fetchSettings :: forall (r :: EffectRow). Member (Genesis :: (Type -> Type) -> Type -> Type) r => Sem r HathSettings
- HathNetwork.Genesis: runGenesis :: forall (r :: EffectRow) a. Members '[Embed IO, Error RPCError :: (Type -> Type) -> Type -> Type, Reader ClientConfig, EHentaiAPI :: (Type -> Type) -> Type -> Type] r => (((Genesis :: (Type -> Type) -> Type -> Type) ': r) @> a) -> Sem r a
- HathNetwork.Genesis: runGenesisIO :: ClientConfig -> ('[Genesis :: (Type -> Type) -> Type -> Type, EHentaiAPI :: (Type -> Type) -> Type -> Type, Reader ClientConfig, Error ClientError :: (Type -> Type) -> Type -> Type, Error RPCError :: (Type -> Type) -> Type -> Type, Embed IO, Final IO] @> a) -> IO (Either RPCError (Either ClientError a))
- HathNetwork.RPC: RPCResponse :: {-# UNPACK #-} !ByteString -> ![ByteString] -> RPCResponse
- HathNetwork.RPC: [CheckGalleryTask] :: forall {k} (m :: k). RPC m (Maybe GalleryMetadata)
- HathNetwork.RPC: [ClientLogin] :: forall {k} (m :: k). RPC m HathSettings
- HathNetwork.RPC: [ClientStart] :: forall {k} (m :: k). RPC m Bool
- HathNetwork.RPC: [ClientStop] :: forall {k} (m :: k). RPC m Bool
- HathNetwork.RPC: [FetchGalleryFile] :: forall {k} (m :: k). (GalleryMetadata, [GalleryFile]) -> RPC m [(GalleryFile, ByteString)]
- HathNetwork.RPC: [NotifyGalleryCompletion] :: forall {k} (m :: k). GalleryMetadata -> RPC m ()
- HathNetwork.RPC: [ServerStat] :: forall {k} (m :: k). RPC m Bool
- HathNetwork.RPC: [StillAlive] :: forall {k} (m :: k). RPC m ()
- HathNetwork.RPC: [payload] :: RPCResponse -> ![ByteString]
- HathNetwork.RPC: [statusCode] :: RPCResponse -> {-# UNPACK #-} !ByteString
- HathNetwork.RPC: checkGalleryTask :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => Sem r (Maybe GalleryMetadata)
- HathNetwork.RPC: clientLogin :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => Sem r HathSettings
- HathNetwork.RPC: clientStart :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => Sem r Bool
- HathNetwork.RPC: clientStop :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => Sem r Bool
- HathNetwork.RPC: data RPC (m :: k) a
- HathNetwork.RPC: data RPCResponse
- HathNetwork.RPC: fetchGalleryFile :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => (GalleryMetadata, [GalleryFile]) -> Sem r [(GalleryFile, ByteString)]
- HathNetwork.RPC: getPayload :: RPCResponse -> Either RPCError [ByteString]
- HathNetwork.RPC: notifyGalleryCompletion :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => GalleryMetadata -> Sem r ()
- HathNetwork.RPC: parseRPCResponse :: ByteString -> Either RPCError RPCResponse
- HathNetwork.RPC: runRPC :: forall a (r :: EffectRow). Members '[Embed IO, Error RPCError :: (Type -> Type) -> Type -> Type, Reader ClientConfig, EHentaiAPI :: (Type -> Type) -> Type -> Type, Log Message] r => (((RPC :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- HathNetwork.RPC: runRPCIO :: ClientConfig -> ('[RPC :: (Type -> Type) -> Type -> Type, EHentaiAPI :: (Type -> Type) -> Type -> Type, Log Message, Reader ClientConfig, Embed IO, Error ClientError :: (Type -> Type) -> Type -> Type, Error RPCError :: (Type -> Type) -> Type -> Type, Final IO] @> a) -> IO (Either RPCError (Either ClientError a))
- HathNetwork.RPC: serverStat :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => Sem r Bool
- HathNetwork.RPC: stillAlive :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => Sem r ()
- Interface.API: RPCParams :: {-# UNPACK #-} !Maybe Text -> {-# UNPACK #-} !Maybe Text -> {-# UNPACK #-} !Maybe Text -> {-# UNPACK #-} !Maybe Text -> {-# UNPACK #-} !Maybe Text -> {-# UNPACK #-} !Maybe Text -> RPCParams
- Interface.API: RefreshCerts :: ServerCommand
- Interface.API: RefreshSettings :: ServerCommand
- Interface.API: SpeedTest :: ServerCommand
- Interface.API: StartDownloader :: ServerCommand
- Interface.API: StillAlive :: ServerCommand
- Interface.API: ThreadedProxyTest :: ServerCommand
- Interface.API: [EhGallery] :: forall {k} (m :: k). RPCParams -> EHentaiAPI m ByteString
- Interface.API: [EhRPC] :: forall {k} (m :: k). RPCParams -> EHentaiAPI m ByteString
- Interface.API: [act] :: RPCParams -> {-# UNPACK #-} !Maybe Text
- Interface.API: [actkey] :: RPCParams -> {-# UNPACK #-} !Maybe Text
- Interface.API: [acttime] :: RPCParams -> {-# UNPACK #-} !Maybe Text
- Interface.API: [add] :: RPCParams -> {-# UNPACK #-} !Maybe Text
- Interface.API: [cid] :: RPCParams -> {-# UNPACK #-} !Maybe Text
- Interface.API: [clientbuild] :: RPCParams -> {-# UNPACK #-} !Maybe Text
- Interface.API: api :: Proxy API
- Interface.API: checkServerStatus :: '[EHentaiAPI :: (Type -> Type) -> Type -> Type, Error RPCError :: (Type -> Type) -> Type -> Type] >@> Bool
- Interface.API: completeGalleryTask :: GalleryMetadata -> (EHentaiAPI :: (Type -> Type) -> Type -> Type) -@> ()
- Interface.API: data EHentaiAPI (m :: k) a
- Interface.API: data RPCParams
- Interface.API: data ServerCommand
- Interface.API: downloadCertificates :: '[EHentaiAPI :: (Type -> Type) -> Type -> Type, Error RPCError :: (Type -> Type) -> Type -> Type, Reader ClientConfig] >@> (CertificateChain, PrivKey)
- Interface.API: downloadGalleryFile :: GalleryMetadata -> GalleryFile -> '[EHentaiAPI :: (Type -> Type) -> Type -> Type, Reader ClientConfig, Error RPCError :: (Type -> Type) -> Type -> Type, Embed IO] >@> Maybe ByteString
- Interface.API: ehRPC :: forall (r :: EffectRow). Member (EHentaiAPI :: (Type -> Type) -> Type -> Type) r => RPCParams -> Sem r ByteString
- Interface.API: emptyRPCParams :: RPCParams
- Interface.API: fetchBlacklist :: Int -> '[EHentaiAPI :: (Type -> Type) -> Type -> Type, Error RPCError :: (Type -> Type) -> Type -> Type] >@> [Text]
- Interface.API: getSettings :: '[EHentaiAPI :: (Type -> Type) -> Type -> Type, Error RPCError :: (Type -> Type) -> Type -> Type] >@> HathSettings
- Interface.API: heartbeat :: (EHentaiAPI :: (Type -> Type) -> Type -> Type) -@> ()
- Interface.API: instance GHC.Classes.Eq Interface.API.ServerCommand
- Interface.API: instance GHC.Show.Show Interface.API.ServerCommand
- Interface.API: instance Servant.API.ContentTypes.Accept Interface.API.SpeedTest
- Interface.API: instance Servant.API.ContentTypes.MimeRender Interface.API.SpeedTest Data.ByteString.Internal.Type.ByteString
- Interface.API: instance Servant.API.ContentTypes.MimeUnrender Servant.API.ContentTypes.PlainText Data.ByteString.Internal.Type.ByteString
- Interface.API: instance Web.Internal.HttpApiData.FromHttpApiData Interface.API.ServerCommand
- Interface.API: login :: '[EHentaiAPI :: (Type -> Type) -> Type -> Type, Error RPCError :: (Type -> Type) -> Type -> Type] >@> HathSettings
- Interface.API: nextGalleryTask :: '[EHentaiAPI :: (Type -> Type) -> Type -> Type, Reader ClientConfig] >@> Either Text GalleryMetadata
- Interface.API: openResource :: FileURI -> (ByteString, ByteString) -> '[EHentaiAPI :: (Type -> Type) -> Type -> Type, Reader ClientConfig, Error RPCError :: (Type -> Type) -> Type -> Type, Embed IO] >@> Maybe (Response BodyReader)
- Interface.API: renderServerCommand :: ServerCommand -> Text
- Interface.API: reportFailures :: [Text] -> (EHentaiAPI :: (Type -> Type) -> Type -> Type) -@> ()
- Interface.API: runEHentaiAPI :: forall a (r :: EffectRow). Members '[Embed IO, Error ClientError :: (Type -> Type) -> Type -> Type, Reader ClientConfig] r => (((EHentaiAPI :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- Interface.API: runEHentaiAPIIO :: ClientConfig -> ('[EHentaiAPI :: (Type -> Type) -> Type -> Type, Reader ClientConfig, Error ClientError :: (Type -> Type) -> Type -> Type, Embed IO, Final IO] @> a) -> IO (Either ClientError a)
- Interface.API: startListening :: (EHentaiAPI :: (Type -> Type) -> Type -> Type) -@> Bool
- Interface.API: stopListening :: (EHentaiAPI :: (Type -> Type) -> Type -> Type) -@> Bool
- Interface.API: type API = "favicon.ico" :> Get '[PlainText] NoContent :<|> "robots.txt" :> Get '[PlainText] Text :<|> "h" :> Capture "info" Text :> Capture "opts" Text :> Capture "filename" Text :> RawM :<|> "metrics" :> Get '[PlainText] Text :<|> "servercmd" :> Capture "command" ServerCommand :> Capture "additional" Text :> Capture "time" Int :> Capture "key" Text :> StreamGet NoFraming OctetStream Headers '[Header "Content-Length" Int] SourceIO ByteString :<|> "t" :> Capture "testsize" Int :> Capture "testtime" Int64 :> Capture "testkey" Text :> Capture "nothing" Text :> StreamGet NoFraming SpeedTest Headers '[Header "Content-Length" Int] SourceIO ByteString :<|> "admin" :> "settings" :> Get '[PlainText] Text :<|> Raw
- LegacyCiphers: cipher_ECDHE_ECDSA_AES128CBC_SHA :: Cipher
- LegacyCiphers: cipher_ECDHE_RSA_AES128CBC_SHA :: Cipher
- LegacyCiphers: cipher_ECDHE_RSA_AES256CBC_SHA :: Cipher
- Metrics.Counter: add :: Int -> Counter -> IO ()
- Metrics.Counter: inc :: Counter -> IO ()
- Metrics.Counter: read :: Counter -> IO Int64
- Metrics.Gauge: add :: Int -> Gauge -> IO ()
- Metrics.Gauge: dec :: Gauge -> IO ()
- Metrics.Gauge: inc :: Gauge -> IO ()
- Metrics.Gauge: read :: Gauge -> IO Int64
- Metrics.Gauge: set :: Int64 -> Gauge -> IO ()
- Metrics.Gauge: sub :: Int -> Gauge -> IO ()
- Metrics.Prometheus: render :: [MetricSnapshot] -> ByteString
- Metrics.Registry: newRegistry :: IO Registry
- Metrics.Registry: registerCounter :: Text -> Text -> Registry -> IO Counter
- Metrics.Registry: registerCounterWith :: Text -> Text -> Labels -> Registry -> IO Counter
- Metrics.Registry: registerGauge :: Text -> Text -> Registry -> IO Gauge
- Metrics.Registry: registerGaugeWith :: Text -> Text -> Labels -> Registry -> IO Gauge
- Metrics.Snapshot: generateSnapshot :: Registry -> IO ByteString
- Metrics.Snapshot: getCachedSnapshot :: Registry -> IO ByteString
- Metrics.Snapshot: startSnapshotWorker :: Registry -> Int -> IO ()
- Metrics.Types: Counter :: !MetricId -> !Text -> !IORef Int64 -> Counter
- Metrics.Types: CounterSnapshot :: !MetricId -> !Text -> !Int64 -> MetricSnapshot
- Metrics.Types: Gauge :: !MetricId -> !Text -> !IORef Int64 -> Gauge
- Metrics.Types: GaugeSnapshot :: !MetricId -> !Text -> !Int64 -> MetricSnapshot
- Metrics.Types: MetricId :: !Text -> !Labels -> MetricId
- Metrics.Types: Registry :: !IORef [Counter] -> !IORef [Gauge] -> !TVar ByteString -> Registry
- Metrics.Types: [counterHelp] :: Counter -> !Text
- Metrics.Types: [counterMetricId] :: Counter -> !MetricId
- Metrics.Types: [counterRef] :: Counter -> !IORef Int64
- Metrics.Types: [gaugeHelp] :: Gauge -> !Text
- Metrics.Types: [gaugeMetricId] :: Gauge -> !MetricId
- Metrics.Types: [gaugeRef] :: Gauge -> !IORef Int64
- Metrics.Types: [metricLabels] :: MetricId -> !Labels
- Metrics.Types: [metricName] :: MetricId -> !Text
- Metrics.Types: [registryCachedSnapshot] :: Registry -> !TVar ByteString
- Metrics.Types: [registryCounters] :: Registry -> !IORef [Counter]
- Metrics.Types: [registryGauges] :: Registry -> !IORef [Gauge]
- Metrics.Types: data Counter
- Metrics.Types: data Gauge
- Metrics.Types: data MetricId
- Metrics.Types: data MetricSnapshot
- Metrics.Types: data Registry
- Metrics.Types: instance GHC.Classes.Eq Metrics.Types.MetricId
- Metrics.Types: instance GHC.Classes.Ord Metrics.Types.MetricId
- Metrics.Types: instance GHC.Generics.Generic Metrics.Types.MetricId
- Metrics.Types: instance GHC.Show.Show Metrics.Types.MetricId
- Metrics.Types: instance GHC.Show.Show Metrics.Types.MetricSnapshot
- Metrics.Types: type Labels = Map Text Text
- Middleware: IPRecord :: !Int -> !Double -> !Double -> IPRecord
- Middleware: IPv4 :: {-# UNPACK #-} !HostAddress -> IP
- Middleware: IPv6 :: {-# UNPACK #-} !HostAddress6 -> IP
- Middleware: [bannedUntil] :: IPRecord -> !Double
- Middleware: [hitCount] :: IPRecord -> !Int
- Middleware: [lastHit] :: IPRecord -> !Double
- Middleware: checkRateLimit :: IP -> Double -> IO Bool
- Middleware: data IP
- Middleware: data IPRecord
- Middleware: instance Data.Hashable.Class.Hashable Middleware.IP
- Middleware: instance GHC.Classes.Eq Middleware.IP
- Middleware: instance GHC.Classes.Ord Middleware.IP
- Middleware: instance GHC.Generics.Generic Middleware.IP
- Middleware: instance GHC.Show.Show Middleware.IP
- Middleware: instance GHC.Show.Show Middleware.IPRecord
- Middleware: isLocalIP :: IP -> Bool
- Middleware: normalizeAcceptMiddleware :: Middleware
- Middleware: parseIPv4 :: ByteString -> Maybe IP
- Middleware: rateLimitMiddleware :: (IP -> IO Bool) -> Middleware
- Middleware: timeWindow :: Double
- Middleware: tracingConnections :: StatsEnv -> Middleware
- Middleware: tracingTimeUsage :: StatsEnv -> Middleware
- Middleware: type IPMap = HashMap IP IPRecord
- Middleware: type KeystampMap = HashMap ByteString (Int, Double)
- Middleware: withHentaiHeaders :: Middleware
- Server: CacheRunner :: (forall (r :: EffectRow) a. Members '[Embed IO, Log Message, Reader ClientConfig, Error RPCError :: (Type -> Type) -> Type -> Type] r => Sem ((KVStore FileURI StorageResult :: (Type -> Type) -> Type -> Type) ': r) a -> Sem r a) -> CacheRunner
- Server: Cert :: ServerAction
- Server: GalleryTask :: GalleryTask
- Server: GracefulShutdown :: ServerAction
- Server: Reload :: ServerAction
- Server: [runCacheWith] :: CacheRunner -> forall (r :: EffectRow) a. Members '[Embed IO, Log Message, Reader ClientConfig, Error RPCError :: (Type -> Type) -> Type -> Type] r => Sem ((KVStore FileURI StorageResult :: (Type -> Type) -> Type -> Type) ': r) a -> Sem r a
- Server: data GalleryTask
- Server: data ServerAction
- Server: makeApplication :: ServerLoopContext -> Application
- Server: mkServerLoopContext :: ClientConfig -> TVar HathSettings -> TChan ServerAction -> TChan GalleryTask -> Bool -> Bool -> CacheRunner -> StatsEnv -> ServerLoopContext
- Server: newtype CacheRunner
- Server: startServer :: ClientConfig -> HathSettings -> (CertificateChain, PrivKey) -> TChan ServerAction -> Bool -> Bool -> IO ()
- Server: type IPMap = HashMap IP IPRecord
- Server: type KeystampMap = HashMap ByteString (Int, Double)
- SettingM: [GetSettings] :: forall {k} (m :: k). SettingM m HathSettings
- SettingM: [UpdateSettings] :: forall {k} (m :: k). HathSettings -> SettingM m ()
- SettingM: data SettingM (m :: k) a
- SettingM: getSettings :: forall (r :: EffectRow). Member (SettingM :: (Type -> Type) -> Type -> Type) r => Sem r HathSettings
- SettingM: runSettingM :: forall (r :: EffectRow) a. Member (Embed IO) r => TVar HathSettings -> (((SettingM :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- SettingM: updateSettings :: forall (r :: EffectRow). Member (SettingM :: (Type -> Type) -> Type -> Type) r => HathSettings -> Sem r ()
- Stats: StatsEnv :: !Double -> !Registry -> !Counter -> !Counter -> !Counter -> !Counter -> !Gauge -> !Gauge -> !Counter -> !Counter -> !Counter -> !Counter -> !Gauge -> StatsEnv
- Stats: [AddDlBytes] :: forall {k} (m :: k). Int -> Stats m ()
- Stats: [AddDownload] :: forall {k} (m :: k). Int -> Stats m ()
- Stats: [AddUpload] :: forall {k} (m :: k). Int -> Stats m ()
- Stats: [DecActiveConnections] :: forall {k} (m :: k). Stats m ()
- Stats: [IncActiveConnections] :: forall {k} (m :: k). Stats m ()
- Stats: [IncDlFile] :: forall {k} (m :: k). Stats m ()
- Stats: [IncDlTask] :: forall {k} (m :: k). Stats m ()
- Stats: [IncError] :: forall {k} (m :: k). Text -> Stats m ()
- Stats: [IncFetched] :: forall {k} (m :: k). Stats m ()
- Stats: [IncServed] :: forall {k} (m :: k). Stats m ()
- Stats: [ReadPrometheus] :: forall {k} (m :: k). Stats m Text
- Stats: [UpdateUptime] :: forall {k} (m :: k). Stats m ()
- Stats: [statsActiveConnectionsGauge] :: StatsEnv -> !Gauge
- Stats: [statsDlBytesCounter] :: StatsEnv -> !Counter
- Stats: [statsDlFileCounter] :: StatsEnv -> !Counter
- Stats: [statsDlTaskCounter] :: StatsEnv -> !Counter
- Stats: [statsDownloadBytesCounter] :: StatsEnv -> !Counter
- Stats: [statsErrorCounter] :: StatsEnv -> !Counter
- Stats: [statsFetchedCounter] :: StatsEnv -> !Counter
- Stats: [statsLatency] :: StatsEnv -> !Gauge
- Stats: [statsRegistry] :: StatsEnv -> !Registry
- Stats: [statsServedCounter] :: StatsEnv -> !Counter
- Stats: [statsStart] :: StatsEnv -> !Double
- Stats: [statsUploadBytesCounter] :: StatsEnv -> !Counter
- Stats: [statsUptimeGauge] :: StatsEnv -> !Gauge
- Stats: addDlBytes :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Int -> Sem r ()
- Stats: addDownload :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Int -> Sem r ()
- Stats: addUpload :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Int -> Sem r ()
- Stats: data Stats (m :: k) a
- Stats: data StatsEnv
- Stats: decActiveConnections :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Sem r ()
- Stats: incActiveConnections :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Sem r ()
- Stats: incDlFile :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Sem r ()
- Stats: incDlTask :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Sem r ()
- Stats: incFetched :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Sem r ()
- Stats: incServed :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Sem r ()
- Stats: newStatsEnv :: IO StatsEnv
- Stats: readPrometheus :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Sem r Text
- Stats: runStats :: forall (r :: EffectRow) a. Members '[Embed IO, Reader StatsEnv] r => (((Stats :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- Stats: updateUptime :: forall (r :: EffectRow). Member (Stats :: (Type -> Type) -> Type -> Type) r => Sem r ()
- Storage.Database: FileRecord :: {-# UNPACK #-} !Int64 -> {-# UNPACK #-} !Text -> {-# UNPACK #-} !Text -> {-# UNPACK #-} !Maybe Text -> !ByteString -> FileRecord
- Storage.Database: [fileRecordBytes] :: FileRecord -> !ByteString
- Storage.Database: [fileRecordFileId] :: FileRecord -> {-# UNPACK #-} !Text
- Storage.Database: [fileRecordFileName] :: FileRecord -> {-# UNPACK #-} !Maybe Text
- Storage.Database: [fileRecordLRUCounter] :: FileRecord -> {-# UNPACK #-} !Int64
- Storage.Database: [fileRecordS4] :: FileRecord -> {-# UNPACK #-} !Text
- Storage.Database: data FileRecord
- Storage.Database: initializeDB :: Connection -> IO ()
- Storage.Database: runCache :: forall (r :: EffectRow) a. Members '[Embed IO] r => Connection -> (((KVStore FileURI StorageResult :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- Storage.Database: runCachePure :: forall (r :: [(Type -> Type) -> Type -> Type]) a. Map FileURI FileRecord -> (((KVStore FileURI FileRecord :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- Storage.Filesystem: runCacheFilesystem :: forall (r :: EffectRow) a. Members '[Embed IO, Log Message, Reader ClientConfig, Error RPCError :: (Type -> Type) -> Type -> Type] r => FilePath -> (((KVStore FileURI StorageResult :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- Storage.Locate: LocateURI :: !ByteString -> !FileURI -> !Map ByteString ByteString -> LocateURI
- Storage.Locate: Stored :: !StorageResult -> Located
- Storage.Locate: Streamed :: !(ByteString -> IO ()) -> IO () -> Located
- Storage.Locate: [LocateResource] :: forall {k} (m :: k). LocateURI -> Locate m (Maybe Located)
- Storage.Locate: [locateURIFilename] :: LocateURI -> !ByteString
- Storage.Locate: [locateURIOptions] :: LocateURI -> !Map ByteString ByteString
- Storage.Locate: [locateURI] :: LocateURI -> !FileURI
- Storage.Locate: data Locate (m :: k) a
- Storage.Locate: data LocateURI
- Storage.Locate: data Located
- Storage.Locate: locateResource :: forall (r :: EffectRow). Member (Locate :: (Type -> Type) -> Type -> Type) r => LocateURI -> Sem r (Maybe Located)
- Storage.Locate: runLocate :: forall (r :: EffectRow) a. Members '[KVStore FileURI StorageResult :: (Type -> Type) -> Type -> Type, SettingM :: (Type -> Type) -> Type -> Type, Error RPCError :: (Type -> Type) -> Type -> Type, Reader ClientConfig, EHentaiAPI :: (Type -> Type) -> Type -> Type, Embed IO, Final IO, Stats :: (Type -> Type) -> Type -> Type, Log Message] r => (((Locate :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- Storage.Locate: streamUpstream :: FileURI -> Response BodyReader -> (ByteString -> IO ()) -> (ByteString -> IO ()) -> IO ()
- Storage.R2: R2Connection :: !MinioConn -> !Text -> !AtomicLRU FileURI (Double, ByteString) -> R2Connection
- Storage.R2: [r2ConnBucket] :: R2Connection -> !Text
- Storage.R2: [r2MinioConn] :: R2Connection -> !MinioConn
- Storage.R2: [r2UrlCache] :: R2Connection -> !AtomicLRU FileURI (Double, ByteString)
- Storage.R2: data R2Connection
- Storage.R2: mkR2Connection :: R2Config -> IO (Either Text R2Connection)
- Storage.R2: runCacheR2 :: forall (r :: EffectRow) a. Members '[Embed IO, Log Message, Error RPCError :: (Type -> Type) -> Type -> Type] r => R2Connection -> (((KVStore FileURI StorageResult :: (Type -> Type) -> Type -> Type) ': r) @> a) -> r @> a
- Types: CacheBackendFilesystem :: CacheBackend
- Types: CacheBackendR2 :: CacheBackend
- Types: CacheBackendSQLite :: CacheBackend
- Types: CertificateFailure :: {-# UNPACK #-} !Text -> RPCError
- Types: ClientConfig :: {-# UNPACK #-} !Text -> {-# UNPACK #-} !Text -> {-# UNPACK #-} !Text -> {-# UNPACK #-} !Maybe ClientProxy -> {-# UNPACK #-} !Text -> {-# UNPACK #-} !Text -> !CacheBackend -> !Maybe R2Config -> {-# UNPACK #-} !Int64 -> ClientConfig
- Types: ClientProxy :: {-# UNPACK #-} !Text -> {-# UNPACK #-} !Integer -> {-# UNPACK #-} !Maybe (Text, Text) -> ClientProxy
- Types: EmptyResponse :: RPCError
- Types: FileRecord :: {-# UNPACK #-} !Int64 -> {-# UNPACK #-} !Text -> {-# UNPACK #-} !Text -> {-# UNPACK #-} !Maybe Text -> !ByteString -> FileRecord
- Types: FileURI :: {-# UNPACK #-} !ByteString -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !ByteString -> FileURI
- Types: GalleryFile :: {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !ByteString -> {-# UNPACK #-} !ByteString -> !Maybe ByteString -> {-# UNPACK #-} !ByteString -> GalleryFile
- Types: GalleryMetadata :: {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !ByteString -> {-# UNPACK #-} !ByteString -> {-# UNPACK #-} ![GalleryFile] -> GalleryMetadata
- Types: HathSettings :: {-# UNPACK #-} !ShortByteString -> {-# UNPACK #-} !ShortByteString -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int64 -> {-# UNPACK #-} !Int64 -> {-# UNPACK #-} !Int64 -> {-# UNPACK #-} !Bool -> {-# UNPACK #-} !Bool -> {-# UNPACK #-} !Bool -> {-# UNPACK #-} !Bool -> {-# UNPACK #-} !Bool -> {-# UNPACK #-} !HashSet ShortByteString -> ![ByteString] -> HathSettings
- Types: R2Config :: {-# UNPACK #-} !Text -> {-# UNPACK #-} !Text -> R2Config
- Types: RPCResponse :: {-# UNPACK #-} !ByteString -> ![ByteString] -> RPCResponse
- Types: Record :: FileRecord -> StorageResult
- Types: Redirect :: ByteString -> StorageResult
- Types: RequestFailure :: {-# UNPACK #-} !Text -> RPCError
- Types: StorageError :: {-# UNPACK #-} !Text -> RPCError
- Types: [auth] :: ClientProxy -> {-# UNPACK #-} !Maybe (Text, Text)
- Types: [cacheBackend] :: ClientConfig -> !CacheBackend
- Types: [cacheNeedsRescan] :: HathSettings -> {-# UNPACK #-} !Bool
- Types: [cacheNeedsVerify] :: HathSettings -> {-# UNPACK #-} !Bool
- Types: [cachePath] :: ClientConfig -> {-# UNPACK #-} !Text
- Types: [checkIPOrigin] :: HathSettings -> {-# UNPACK #-} !Bool
- Types: [clientHost] :: HathSettings -> {-# UNPACK #-} !ShortByteString
- Types: [clientId] :: ClientConfig -> {-# UNPACK #-} !Text
- Types: [clientPort] :: HathSettings -> {-# UNPACK #-} !Int
- Types: [diskLimitBytes] :: HathSettings -> {-# UNPACK #-} !Int64
- Types: [diskRemainingBytes] :: HathSettings -> {-# UNPACK #-} !Int64
- Types: [downloadDir] :: ClientConfig -> {-# UNPACK #-} !Text
- Types: [fileExt] :: FileURI -> {-# UNPACK #-} !ByteString
- Types: [fileHash] :: FileURI -> {-# UNPACK #-} !ByteString
- Types: [fileRecordBytes] :: FileRecord -> !ByteString
- Types: [fileRecordFileId] :: FileRecord -> {-# UNPACK #-} !Text
- Types: [fileRecordFileName] :: FileRecord -> {-# UNPACK #-} !Maybe Text
- Types: [fileRecordLRUCounter] :: FileRecord -> {-# UNPACK #-} !Int64
- Types: [fileRecordS4] :: FileRecord -> {-# UNPACK #-} !Text
- Types: [fileSize] :: FileURI -> {-# UNPACK #-} !Int
- Types: [fileXRes] :: FileURI -> {-# UNPACK #-} !Int
- Types: [fileYRes] :: FileURI -> {-# UNPACK #-} !Int
- Types: [floodControl] :: HathSettings -> {-# UNPACK #-} !Bool
- Types: [galleryFileCount] :: GalleryMetadata -> {-# UNPACK #-} !Int
- Types: [galleryFileExt] :: GalleryFile -> {-# UNPACK #-} !ByteString
- Types: [galleryFileHash] :: GalleryFile -> !Maybe ByteString
- Types: [galleryFileIndex] :: GalleryFile -> {-# UNPACK #-} !Int
- Types: [galleryFileList] :: GalleryMetadata -> {-# UNPACK #-} ![GalleryFile]
- Types: [galleryFileName] :: GalleryFile -> {-# UNPACK #-} !ByteString
- Types: [galleryFilePage] :: GalleryFile -> {-# UNPACK #-} !Int
- Types: [galleryFileXRes] :: GalleryFile -> {-# UNPACK #-} !ByteString
- Types: [galleryID] :: GalleryMetadata -> {-# UNPACK #-} !Int
- Types: [galleryMinXRes] :: GalleryMetadata -> {-# UNPACK #-} !ByteString
- Types: [galleryTitle] :: GalleryMetadata -> {-# UNPACK #-} !ByteString
- Types: [host] :: ClientProxy -> {-# UNPACK #-} !Text
- Types: [key] :: ClientConfig -> {-# UNPACK #-} !Text
- Types: [lruCacheSize] :: ClientConfig -> {-# UNPACK #-} !Int64
- Types: [payload] :: RPCResponse -> ![ByteString]
- Types: [port] :: ClientProxy -> {-# UNPACK #-} !Integer
- Types: [proxy] :: ClientConfig -> {-# UNPACK #-} !Maybe ClientProxy
- Types: [r2Bucket] :: R2Config -> {-# UNPACK #-} !Text
- Types: [r2Config] :: ClientConfig -> !Maybe R2Config
- Types: [r2Endpoint] :: R2Config -> {-# UNPACK #-} !Text
- Types: [rpcBaseURL] :: HathSettings -> {-# UNPACK #-} !ShortByteString
- Types: [rpcServerIPs] :: HathSettings -> ![ByteString]
- Types: [staticRanges] :: HathSettings -> {-# UNPACK #-} !HashSet ShortByteString
- Types: [statusCode] :: RPCResponse -> {-# UNPACK #-} !ByteString
- Types: [throttleBytes] :: HathSettings -> {-# UNPACK #-} !Int64
- Types: [useLessMemory] :: HathSettings -> {-# UNPACK #-} !Bool
- Types: [version] :: ClientConfig -> {-# UNPACK #-} !Text
- Types: clientBuild :: Text
- Types: data CacheBackend
- Types: data ClientConfig
- Types: data ClientProxy
- Types: data FileRecord
- Types: data FileURI
- Types: data GalleryFile
- Types: data GalleryMetadata
- Types: data HathSettings
- Types: data R2Config
- Types: data RPCError
- Types: data RPCResponse
- Types: data StorageResult
- Types: defaultClientConfig :: ClientConfig
- Types: defaultHathSettings :: HathSettings
- Types: emptyFileURI :: FileURI
- Types: getPayload :: RPCResponse -> Either RPCError [ByteString]
- Types: hentaiHeader :: IsString a => [(HeaderName, a)]
- Types: instance Database.SQLite.Simple.FromRow.FromRow Types.FileRecord
- Types: instance Database.SQLite.Simple.ToRow.ToRow Types.FileRecord
- Types: instance Dhall.Marshal.Decode.FromDhall Types.CacheBackend
- Types: instance Dhall.Marshal.Decode.FromDhall Types.ClientConfig
- Types: instance Dhall.Marshal.Decode.FromDhall Types.ClientProxy
- Types: instance Dhall.Marshal.Decode.FromDhall Types.R2Config
- Types: instance Dhall.Marshal.Encode.ToDhall Types.CacheBackend
- Types: instance Dhall.Marshal.Encode.ToDhall Types.ClientConfig
- Types: instance Dhall.Marshal.Encode.ToDhall Types.ClientProxy
- Types: instance Dhall.Marshal.Encode.ToDhall Types.R2Config
- Types: instance GHC.Classes.Eq Types.CacheBackend
- Types: instance GHC.Classes.Eq Types.FileRecord
- Types: instance GHC.Classes.Eq Types.FileURI
- Types: instance GHC.Classes.Eq Types.GalleryFile
- Types: instance GHC.Classes.Eq Types.GalleryMetadata
- Types: instance GHC.Classes.Eq Types.RPCError
- Types: instance GHC.Classes.Eq Types.RPCResponse
- Types: instance GHC.Classes.Ord Types.FileURI
- Types: instance GHC.Exception.Type.Exception Types.RPCError
- Types: instance GHC.Generics.Generic Types.CacheBackend
- Types: instance GHC.Generics.Generic Types.ClientConfig
- Types: instance GHC.Generics.Generic Types.ClientProxy
- Types: instance GHC.Generics.Generic Types.FileRecord
- Types: instance GHC.Generics.Generic Types.R2Config
- Types: instance GHC.Generics.Generic Types.RPCError
- Types: instance GHC.Generics.Generic Types.RPCResponse
- Types: instance GHC.Show.Show Types.CacheBackend
- Types: instance GHC.Show.Show Types.ClientConfig
- Types: instance GHC.Show.Show Types.ClientProxy
- Types: instance GHC.Show.Show Types.FileRecord
- Types: instance GHC.Show.Show Types.FileURI
- Types: instance GHC.Show.Show Types.HathSettings
- Types: instance GHC.Show.Show Types.R2Config
- Types: instance GHC.Show.Show Types.RPCError
- Types: instance GHC.Show.Show Types.RPCResponse
- Types: parseFileURI :: ByteString -> FileURI
- Types: parseMetadata :: ByteString -> ByteString -> Either Text GalleryMetadata
- Types: parseRPCResponse :: ByteString -> Either RPCError RPCResponse
- Types: parseRPCResponse' :: forall (r :: EffectRow). Member (Error RPCError :: (Type -> Type) -> Type -> Type) r => ByteString -> Sem r [ByteString]
- Types: parseSettings :: [ByteString] -> HathSettings
- Types: readClientConfig :: Text -> IO ClientConfig
- Types: reconstructRecord :: FileURI -> ByteString -> FileRecord
- Types: renderFileURIBytes :: FileURI -> ByteString
- Types: renderFileURIText :: FileURI -> Text
- Types: verifyRPCResponse :: ByteString -> ByteString -> ByteString
- Utils: bufferSending :: forall (m :: Type -> Type). Int -> StepT m ByteString
- Utils: class HathHash a
- Utils: firstJustM :: Monad m => (a -> m (Maybe b)) -> [a] -> m (Maybe b)
- Utils: hasParam :: ByteString -> URLParams -> Bool
- Utils: hash :: HathHash a => ByteString -> a
- Utils: instance Utils.HathHash Data.ByteString.Internal.Type.ByteString
- Utils: instance Utils.HathHash Data.Text.Internal.Text
- Utils: log :: Severity -> Text -> Log Message -@> ()
- Utils: lookupParam :: ByteString -> URLParams -> Maybe ByteString
- Utils: parseURLParams :: ByteString -> URLParams
- Utils: tooManyRequestsResponse :: Response
- Utils: type URLParams = Map ByteString ByteString
- Version: versionInfo :: Text
+ Hath.AtomicFile: writeFileAtomic :: FilePath -> ByteString -> IO ()
+ Hath.CLI: Options :: !FilePath -> !Maybe Text -> !Maybe Text -> !Bool -> !Bool -> !Bool -> !Bool -> Options
+ Hath.CLI: [optCachePath] :: Options -> !Maybe Text
+ Hath.CLI: [optConfigPath] :: Options -> !FilePath
+ Hath.CLI: [optDisableRateLimit] :: Options -> !Bool
+ Hath.CLI: [optDownloadDir] :: Options -> !Maybe Text
+ Hath.CLI: [optSkipStartupVerify] :: Options -> !Bool
+ Hath.CLI: [optTrustProxyHeaders] :: Options -> !Bool
+ Hath.CLI: [optValidateConfig] :: Options -> !Bool
+ Hath.CLI: applyOptionsToConfig :: Options -> ClientConfig -> ClientConfig
+ Hath.CLI: data Options
+ Hath.CLI: instance GHC.Show.Show Hath.CLI.Options
+ Hath.CLI: parseOptions :: IO Options
+ Hath.Cache: CacheRunner :: (forall (r :: EffectRow) a. Members '[Embed IO, Log Message, Error HathError :: (Type -> Type) -> Type -> Type] r => Sem ((Cache :: (Type -> Type) -> Type -> Type) ': r) a -> Sem r a) -> CacheRunner
+ Hath.Cache: InMemory :: !ByteString -> CacheHit
+ Hath.Cache: LocalFile :: !FilePath -> CacheHit
+ Hath.Cache: Redirect :: !ByteString -> CacheHit
+ Hath.Cache: [DeleteFile] :: forall {k} (m :: k). FileURI -> Cache m ()
+ Hath.Cache: [LookupFile] :: forall {k} (m :: k). FileURI -> Cache m (Maybe CacheHit)
+ Hath.Cache: [StoreFile] :: forall {k} (m :: k). FileURI -> ByteString -> Cache m ()
+ Hath.Cache: [runCacheWith] :: CacheRunner -> forall (r :: EffectRow) a. Members '[Embed IO, Log Message, Error HathError :: (Type -> Type) -> Type -> Type] r => Sem ((Cache :: (Type -> Type) -> Type -> Type) ': r) a -> Sem r a
+ Hath.Cache: data Cache (m :: k) a
+ Hath.Cache: data CacheHit
+ Hath.Cache: deleteFile :: forall (r :: EffectRow). Member (Cache :: (Type -> Type) -> Type -> Type) r => FileURI -> Sem r ()
+ Hath.Cache: instance GHC.Classes.Eq Hath.Cache.CacheHit
+ Hath.Cache: instance GHC.Show.Show Hath.Cache.CacheHit
+ Hath.Cache: lookupFile :: forall (r :: EffectRow). Member (Cache :: (Type -> Type) -> Type -> Type) r => FileURI -> Sem r (Maybe CacheHit)
+ Hath.Cache: newtype CacheRunner
+ Hath.Cache: storeFile :: forall (r :: EffectRow). Member (Cache :: (Type -> Type) -> Type -> Type) r => FileURI -> ByteString -> Sem r ()
+ Hath.Cache.Filesystem: data FsCache
+ Hath.Cache.Filesystem: newFsCache :: FilePath -> IO FsCache
+ Hath.Cache.Filesystem: runCacheFilesystem :: forall (r :: EffectRow) a. Members '[Embed IO, Log Message, Error HathError :: (Type -> Type) -> Type -> Type] r => FsCache -> Sem ((Cache :: (Type -> Type) -> Type -> Type) ': r) a -> Sem r a
+ Hath.Cache.Filesystem: runEvictor :: FsCache -> IO Int64 -> IO Void
+ Hath.Cache.Filesystem: sweep :: FsCache -> Int64 -> IO (Int, Int64)
+ Hath.Cache.R2: R2Connection :: !MinioConn -> !Text -> !AtomicLRU FileURI (Double, ByteString) -> R2Connection
+ Hath.Cache.R2: [r2ConnBucket] :: R2Connection -> !Text
+ Hath.Cache.R2: [r2MinioConn] :: R2Connection -> !MinioConn
+ Hath.Cache.R2: [r2UrlCache] :: R2Connection -> !AtomicLRU FileURI (Double, ByteString)
+ Hath.Cache.R2: data R2Connection
+ Hath.Cache.R2: fileURIToKey :: FileURI -> Text
+ Hath.Cache.R2: mkR2Connection :: R2Config -> IO (Either Text R2Connection)
+ Hath.Cache.R2: putR2Object :: R2Connection -> FileURI -> ByteString -> IO (Either Text ())
+ Hath.Cache.R2: runCacheR2 :: forall (r :: EffectRow) a. Members '[Embed IO, Log Message, Error HathError :: (Type -> Type) -> Type -> Type] r => R2Connection -> Sem ((Cache :: (Type -> Type) -> Type -> Type) ': r) a -> Sem r a
+ Hath.Cache.R2: runR2 :: R2Connection -> Minio a -> IO (Either Text a)
+ Hath.Cache.SQLite: VerificationStats :: !Int -> !Int -> !NominalDiffTime -> VerificationStats
+ Hath.Cache.SQLite: [corruptedCount] :: VerificationStats -> !Int
+ Hath.Cache.SQLite: [verificationTime] :: VerificationStats -> !NominalDiffTime
+ Hath.Cache.SQLite: [verifiedCount] :: VerificationStats -> !Int
+ Hath.Cache.SQLite: data VerificationStats
+ Hath.Cache.SQLite: initializeDB :: Connection -> IO ()
+ Hath.Cache.SQLite: instance GHC.Classes.Eq Hath.Cache.SQLite.VerificationStats
+ Hath.Cache.SQLite: instance GHC.Show.Show Hath.Cache.SQLite.VerificationStats
+ Hath.Cache.SQLite: runCacheSQLite :: forall (r :: EffectRow) a. Member (Embed IO) r => Connection -> Sem ((Cache :: (Type -> Type) -> Type -> Type) ': r) a -> Sem r a
+ Hath.Cache.SQLite: verifyAllFiles :: Connection -> IO VerificationStats
+ Hath.Config: CacheBackendFilesystem :: CacheBackend
+ Hath.Config: CacheBackendR2 :: CacheBackend
+ Hath.Config: CacheBackendSQLite :: CacheBackend
+ Hath.Config: ClientConfig :: !Text -> !Text -> !Text -> !Maybe ClientProxy -> !Text -> !Text -> !CacheBackend -> !Maybe R2Config -> !Int64 -> ClientConfig
+ Hath.Config: ClientProxy :: !Text -> !Integer -> !Maybe (Text, Text) -> ClientProxy
+ Hath.Config: R2Config :: !Text -> !Text -> R2Config
+ Hath.Config: [auth] :: ClientProxy -> !Maybe (Text, Text)
+ Hath.Config: [cacheBackend] :: ClientConfig -> !CacheBackend
+ Hath.Config: [cachePath] :: ClientConfig -> !Text
+ Hath.Config: [clientId] :: ClientConfig -> !Text
+ Hath.Config: [downloadDir] :: ClientConfig -> !Text
+ Hath.Config: [host] :: ClientProxy -> !Text
+ Hath.Config: [key] :: ClientConfig -> !Text
+ Hath.Config: [lruCacheSize] :: ClientConfig -> !Int64
+ Hath.Config: [port] :: ClientProxy -> !Integer
+ Hath.Config: [proxy] :: ClientConfig -> !Maybe ClientProxy
+ Hath.Config: [r2Bucket] :: R2Config -> !Text
+ Hath.Config: [r2Config] :: ClientConfig -> !Maybe R2Config
+ Hath.Config: [r2Endpoint] :: R2Config -> !Text
+ Hath.Config: [version] :: ClientConfig -> !Text
+ Hath.Config: data CacheBackend
+ Hath.Config: data ClientConfig
+ Hath.Config: data ClientProxy
+ Hath.Config: data R2Config
+ Hath.Config: defaultClientConfig :: ClientConfig
+ Hath.Config: instance Dhall.Marshal.Decode.FromDhall Hath.Config.CacheBackend
+ Hath.Config: instance Dhall.Marshal.Decode.FromDhall Hath.Config.ClientConfig
+ Hath.Config: instance Dhall.Marshal.Decode.FromDhall Hath.Config.ClientProxy
+ Hath.Config: instance Dhall.Marshal.Decode.FromDhall Hath.Config.R2Config
+ Hath.Config: instance Dhall.Marshal.Encode.ToDhall Hath.Config.CacheBackend
+ Hath.Config: instance Dhall.Marshal.Encode.ToDhall Hath.Config.ClientConfig
+ Hath.Config: instance Dhall.Marshal.Encode.ToDhall Hath.Config.ClientProxy
+ Hath.Config: instance Dhall.Marshal.Encode.ToDhall Hath.Config.R2Config
+ Hath.Config: instance GHC.Classes.Eq Hath.Config.CacheBackend
+ Hath.Config: instance GHC.Generics.Generic Hath.Config.CacheBackend
+ Hath.Config: instance GHC.Generics.Generic Hath.Config.ClientConfig
+ Hath.Config: instance GHC.Generics.Generic Hath.Config.ClientProxy
+ Hath.Config: instance GHC.Generics.Generic Hath.Config.R2Config
+ Hath.Config: instance GHC.Show.Show Hath.Config.CacheBackend
+ Hath.Config: instance GHC.Show.Show Hath.Config.ClientConfig
+ Hath.Config: instance GHC.Show.Show Hath.Config.ClientProxy
+ Hath.Config: instance GHC.Show.Show Hath.Config.R2Config
+ Hath.Config: readClientConfig :: Text -> IO ClientConfig
+ Hath.Download: fetchFile :: Int -> Request -> (Int -> ByteString -> IO ()) -> IO (Either Text Int)
+ Hath.Download: receive :: Int -> Int -> BodyReader -> (Int -> ByteString -> IO ()) -> IO (Either Text ())
+ Hath.Error: CertificateFailure :: !Text -> HathError
+ Hath.Error: EmptyResponse :: HathError
+ Hath.Error: RequestFailure :: !Text -> HathError
+ Hath.Error: StorageError :: !Text -> HathError
+ Hath.Error: TransportError :: !Text -> HathError
+ Hath.Error: data HathError
+ Hath.Error: instance GHC.Classes.Eq Hath.Error.HathError
+ Hath.Error: instance GHC.Exception.Type.Exception Hath.Error.HathError
+ Hath.Error: instance GHC.Show.Show Hath.Error.HathError
+ Hath.FileURI: FileURI :: !ByteString -> !Int -> !Int -> !Int -> !ByteString -> FileURI
+ Hath.FileURI: [fileExt] :: FileURI -> !ByteString
+ Hath.FileURI: [fileHash] :: FileURI -> !ByteString
+ Hath.FileURI: [fileSize] :: FileURI -> !Int
+ Hath.FileURI: [fileXRes] :: FileURI -> !Int
+ Hath.FileURI: [fileYRes] :: FileURI -> !Int
+ Hath.FileURI: data FileURI
+ Hath.FileURI: instance GHC.Classes.Eq Hath.FileURI.FileURI
+ Hath.FileURI: instance GHC.Classes.Ord Hath.FileURI.FileURI
+ Hath.FileURI: instance GHC.Show.Show Hath.FileURI.FileURI
+ Hath.FileURI: mimeType :: FileURI -> ByteString
+ Hath.FileURI: parseFileURI :: ByteString -> Maybe FileURI
+ Hath.FileURI: renderFileURIBytes :: FileURI -> ByteString
+ Hath.FileURI: renderFileURIText :: FileURI -> Text
+ Hath.Gallery: GalleryFile :: !Int -> !Int -> !ByteString -> !ByteString -> !Maybe ByteString -> !ByteString -> GalleryFile
+ Hath.Gallery: GalleryMetadata :: !Int -> !ByteString -> !Text -> !ByteString -> ![GalleryFile] -> GalleryMetadata
+ Hath.Gallery: [galleryFileExt] :: GalleryFile -> !ByteString
+ Hath.Gallery: [galleryFileHash] :: GalleryFile -> !Maybe ByteString
+ Hath.Gallery: [galleryFileIndex] :: GalleryFile -> !Int
+ Hath.Gallery: [galleryFileList] :: GalleryMetadata -> ![GalleryFile]
+ Hath.Gallery: [galleryFileName] :: GalleryFile -> !ByteString
+ Hath.Gallery: [galleryFilePage] :: GalleryFile -> !Int
+ Hath.Gallery: [galleryFileXRes] :: GalleryFile -> !ByteString
+ Hath.Gallery: [galleryID] :: GalleryMetadata -> !Int
+ Hath.Gallery: [galleryInformation] :: GalleryMetadata -> !ByteString
+ Hath.Gallery: [galleryMinXRes] :: GalleryMetadata -> !ByteString
+ Hath.Gallery: [galleryTitle] :: GalleryMetadata -> !Text
+ Hath.Gallery: data GalleryFile
+ Hath.Gallery: data GalleryMetadata
+ Hath.Gallery: fallbackDirectoryName :: GalleryMetadata -> Text
+ Hath.Gallery: galleryDirectoryName :: Int -> GalleryMetadata -> Maybe Text
+ Hath.Gallery: instance GHC.Classes.Eq Hath.Gallery.GalleryFile
+ Hath.Gallery: instance GHC.Classes.Eq Hath.Gallery.GalleryMetadata
+ Hath.Gallery: parseMetadata :: ByteString -> ByteString -> Either Text GalleryMetadata
+ Hath.Gallery.Downloader: anotherRound :: Int -> Int -> Int -> Bool
+ Hath.Gallery.Downloader: downloadQueue :: ClientConfig -> StatsEnv -> TVar HathSettings -> IO (Either HathError ())
+ Hath.Gallery.Downloader: runGalleryDownloader :: ClientConfig -> StatsEnv -> TVar HathSettings -> MVar () -> IO Void
+ Hath.Locate: Stored :: !CacheHit -> Located
+ Hath.Locate: Streamed :: !(ByteString -> IO ()) -> IO () -> Located
+ Hath.Locate: data Located
+ Hath.Locate: heldBytes :: IO Int
+ Hath.Locate: locateResource :: forall (r :: EffectRow). Members '[Cache :: (Type -> Type) -> Type -> Type, RPC :: (Type -> Type) -> Type -> Type, Reader StatsEnv, Log Message, Reader ClientConfig, Error HathError :: (Type -> Type) -> Type -> Type, Embed IO, Final IO] r => Int -> ByteString -> FileURI -> (ByteString, ByteString) -> Sem r (Either Status Located)
+ Hath.Locate: usableReply :: Int -> FileURI -> Int -> [Header] -> Bool
+ Hath.Log: logError :: forall (r :: EffectRow). Member (Log Message) r => Text -> Sem r ()
+ Hath.Log: logInfo :: forall (r :: EffectRow). Member (Log Message) r => Text -> Sem r ()
+ Hath.Log: logWarning :: forall (r :: EffectRow). Member (Log Message) r => Text -> Sem r ()
+ Hath.Protocol: clientBuild :: Int64
+ Hath.Protocol: commandTimeValid :: Int64 -> Int64 -> Bool
+ Hath.Protocol: fetchParams :: Map ByteString ByteString -> Maybe (ByteString, ByteString)
+ Hath.Protocol: hathRequestToken :: ClientConfig -> ByteString -> ByteString
+ Hath.Protocol: keystampHash :: ClientConfig -> Int64 -> ByteString -> ByteString
+ Hath.Protocol: keystampValid :: ClientConfig -> Int64 -> ByteString -> ByteString -> Bool
+ Hath.Protocol: outgoingHeaders :: [Header] -> [Header]
+ Hath.Protocol: parseURLParams :: ByteString -> Map ByteString ByteString
+ Hath.Protocol: responseHeaders :: ByteString -> Int -> [Header]
+ Hath.Protocol: retryWithServerTime :: Text -> ByteString -> Bool
+ Hath.Protocol: rpcActionKey :: ClientConfig -> Text -> Text -> Int64 -> Text
+ Hath.Protocol: rpcPayload :: ByteString -> Either HathError [ByteString]
+ Hath.Protocol: rpcQuery :: ClientConfig -> Text -> Text -> Int64 -> ByteString
+ Hath.Protocol: sameSecret :: ByteString -> ByteString -> Bool
+ Hath.Protocol: serverCommandKey :: ClientConfig -> Text -> Text -> Int -> Text
+ Hath.Protocol: sha1Hex :: ByteString -> ByteString
+ Hath.Protocol: sha1HexText :: ByteString -> Text
+ Hath.Protocol: speedTestKey :: ClientConfig -> Int -> Int64 -> ByteString
+ Hath.Protocol: verifyRPCResponse :: ByteString -> ByteString -> ByteString
+ Hath.RPC: Refused :: !Text -> StartFailure
+ Hath.RPC: Unannounced :: !Text -> StartFailure
+ Hath.RPC: [CallDownloader] :: forall {k} (m :: k). Text -> Text -> RPC m ByteString
+ Hath.RPC: [CallRPC] :: forall {k} (m :: k). Text -> Text -> RPC m ByteString
+ Hath.RPC: clientLogin :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type, Log Message, Embed IO] r => Sem r HathSettings
+ Hath.RPC: clientSettings :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type, Embed IO] r => HathSettings -> Sem r HathSettings
+ Hath.RPC: clientStart :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type] r => Sem r ()
+ Hath.RPC: clientStop :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type] r => Sem r ()
+ Hath.RPC: connectMicros :: Int
+ Hath.RPC: data RPC (m :: k) a
+ Hath.RPC: data StartFailure
+ Hath.RPC: failureReport :: [Text] -> Maybe Text
+ Hath.RPC: fileSources :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type] r => ByteString -> ByteString -> ByteString -> Sem r [ByteString]
+ Hath.RPC: galleryFileSource :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type] r => GalleryMetadata -> GalleryFile -> Int -> Sem r (Maybe ByteString)
+ Hath.RPC: getBlacklist :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type] r => Int -> Sem r [FileURI]
+ Hath.RPC: getCertificate :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type, Reader ClientConfig] r => Sem r (CertificateChain, PrivKey)
+ Hath.RPC: httpManager :: Manager
+ Hath.RPC: instance GHC.Classes.Eq Hath.RPC.StartFailure
+ Hath.RPC: instance GHC.Show.Show Hath.RPC.StartFailure
+ Hath.RPC: nextGallery :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Reader ClientConfig] r => Maybe GalleryMetadata -> Sem r (Either Text GalleryMetadata)
+ Hath.RPC: overload :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type] r => Sem r ()
+ Hath.RPC: reportDownloadFailures :: forall (r :: EffectRow). Member (RPC :: (Type -> Type) -> Type -> Type) r => [Text] -> Sem r ()
+ Hath.RPC: runRPC :: forall (r :: EffectRow) a. Members '[Embed IO, Error HathError :: (Type -> Type) -> Type -> Type, Reader ClientConfig] r => Sem ((RPC :: (Type -> Type) -> Type -> Type) ': r) a -> Sem r a
+ Hath.RPC: runRPCIO :: ClientConfig -> Sem '[RPC :: (Type -> Type) -> Type -> Type, Log Message, Reader ClientConfig, Error HathError :: (Type -> Type) -> Type -> Type, Embed IO, Final IO] a -> IO (Either HathError a)
+ Hath.RPC: serverStat :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type, Log Message, Embed IO] r => Sem r ()
+ Hath.RPC: startFailure :: Int -> HathError -> StartFailure
+ Hath.RPC: stillAlive :: forall (r :: EffectRow). Members '[RPC :: (Type -> Type) -> Type -> Type, Error HathError :: (Type -> Type) -> Type -> Type] r => Sem r ()
+ Hath.RPC: toldToStop :: HathError -> Bool
+ Hath.RPC: worthRetrying :: Maybe Int -> Bool
+ Hath.Server: Cert :: ServerAction
+ Hath.Server: GracefulShutdown :: ServerAction
+ Hath.Server: ServerEnv :: !ClientConfig -> !TVar HathSettings -> !TChan ServerAction -> !MVar () -> !StatsEnv -> !CacheRunner -> !Maybe FloodControl -> !Bool -> !Maybe ByteString -> ServerEnv
+ Hath.Server: ServerOptions :: !Bool -> !Bool -> ServerOptions
+ Hath.Server: [disableRateLimit] :: ServerOptions -> !Bool
+ Hath.Server: [envActions] :: ServerEnv -> !TChan ServerAction
+ Hath.Server: [envCache] :: ServerEnv -> !CacheRunner
+ Hath.Server: [envConfig] :: ServerEnv -> !ClientConfig
+ Hath.Server: [envDownloadRequest] :: ServerEnv -> !MVar ()
+ Hath.Server: [envFloodControl] :: ServerEnv -> !Maybe FloodControl
+ Hath.Server: [envMetricsToken] :: ServerEnv -> !Maybe ByteString
+ Hath.Server: [envSettings] :: ServerEnv -> !TVar HathSettings
+ Hath.Server: [envStats] :: ServerEnv -> !StatsEnv
+ Hath.Server: [envTrustProxyHeaders] :: ServerEnv -> !Bool
+ Hath.Server: [trustProxyHeaders] :: ServerOptions -> !Bool
+ Hath.Server: data ServerAction
+ Hath.Server: data ServerEnv
+ Hath.Server: data ServerOptions
+ Hath.Server: makeApplication :: ServerEnv -> Application
+ Hath.Server: needsRenewal :: Int64 -> CertificateChain -> Bool
+ Hath.Server: startServer :: ClientConfig -> ServerOptions -> HathSettings -> (CertificateChain, PrivKey) -> TChan ServerAction -> IO ()
+ Hath.Server.API: AdminSettings :: Route
+ Hath.Server.API: Favicon :: Route
+ Hath.Server.API: File :: !ByteString -> !ByteString -> Route
+ Hath.Server.API: Metrics :: Route
+ Hath.Server.API: ProxyTest :: !Int -> !Int64 -> !String -> ProxyTest
+ Hath.Server.API: RefreshCerts :: ServerCommand
+ Hath.Server.API: RefreshSettings :: ServerCommand
+ Hath.Server.API: Refuse :: !Status -> Route
+ Hath.Server.API: Robots :: Route
+ Hath.Server.API: ServerCmd :: !Text -> !Text -> !Int -> !Text -> Route
+ Hath.Server.API: SpeedTest :: !Int -> ServerCommand
+ Hath.Server.API: StartDownloader :: ServerCommand
+ Hath.Server.API: StillAlive :: ServerCommand
+ Hath.Server.API: Test :: !Int -> !Int64 -> !ByteString -> Route
+ Hath.Server.API: ThreadedProxyTest :: !ProxyTest -> ServerCommand
+ Hath.Server.API: [proxyTestCount] :: ProxyTest -> !Int
+ Hath.Server.API: [proxyTestSize] :: ProxyTest -> !Int64
+ Hath.Server.API: [proxyTestURL] :: ProxyTest -> !String
+ Hath.Server.API: data ProxyTest
+ Hath.Server.API: data Route
+ Hath.Server.API: data ServerCommand
+ Hath.Server.API: instance GHC.Classes.Eq Hath.Server.API.ProxyTest
+ Hath.Server.API: instance GHC.Classes.Eq Hath.Server.API.ServerCommand
+ Hath.Server.API: instance GHC.Show.Show Hath.Server.API.ProxyTest
+ Hath.Server.API: instance GHC.Show.Show Hath.Server.API.ServerCommand
+ Hath.Server.API: parseServerCommand :: Text -> Map ByteString ByteString -> Maybe ServerCommand
+ Hath.Server.API: route :: ByteString -> Route
+ Hath.Server.App: Cert :: ServerAction
+ Hath.Server.App: GracefulShutdown :: ServerAction
+ Hath.Server.App: ServerEnv :: !ClientConfig -> !TVar HathSettings -> !TChan ServerAction -> !MVar () -> !StatsEnv -> !CacheRunner -> !Maybe FloodControl -> !Bool -> !Maybe ByteString -> ServerEnv
+ Hath.Server.App: [envActions] :: ServerEnv -> !TChan ServerAction
+ Hath.Server.App: [envCache] :: ServerEnv -> !CacheRunner
+ Hath.Server.App: [envConfig] :: ServerEnv -> !ClientConfig
+ Hath.Server.App: [envDownloadRequest] :: ServerEnv -> !MVar ()
+ Hath.Server.App: [envFloodControl] :: ServerEnv -> !Maybe FloodControl
+ Hath.Server.App: [envMetricsToken] :: ServerEnv -> !Maybe ByteString
+ Hath.Server.App: [envSettings] :: ServerEnv -> !TVar HathSettings
+ Hath.Server.App: [envStats] :: ServerEnv -> !StatsEnv
+ Hath.Server.App: [envTrustProxyHeaders] :: ServerEnv -> !Bool
+ Hath.Server.App: data ServerAction
+ Hath.Server.App: data ServerEnv
+ Hath.Server.App: runApp :: ServerEnv -> App a -> IO (Either Status a)
+ Hath.Server.App: type App = Sem '[Cache :: Type -> Type -> Type -> Type, RPC :: Type -> Type -> Type -> Type, Log Message, Reader StatsEnv, Reader ClientConfig, Error HathError :: Type -> Type -> Type -> Type, Error Status :: Type -> Type -> Type -> Type, Embed IO, Final IO]
+ Hath.Server.ConnectionLimit: Admitted :: Admission
+ Hath.Server.ConnectionLimit: AdmittedNearLimit :: Admission
+ Hath.Server.ConnectionLimit: Refused :: Admission
+ Hath.Server.ConnectionLimit: admission :: Int -> Int -> Admission
+ Hath.Server.ConnectionLimit: connectionLimit :: TVar HathSettings -> IO () -> IO (SockAddr -> IO Bool, SockAddr -> IO ())
+ Hath.Server.ConnectionLimit: data Admission
+ Hath.Server.ConnectionLimit: instance GHC.Classes.Eq Hath.Server.ConnectionLimit.Admission
+ Hath.Server.ConnectionLimit: instance GHC.Show.Show Hath.Server.ConnectionLimit.Admission
+ Hath.Server.ConnectionLimit: overloadDue :: Double -> Maybe Double -> Bool
+ Hath.Server.FloodControl: IPv4 :: !HostAddress -> IP
+ Hath.Server.FloodControl: IPv6 :: !HostAddress6 -> IP
+ Hath.Server.FloodControl: checkRateLimit :: FloodControl -> IP -> Double -> IO Bool
+ Hath.Server.FloodControl: data FloodControl
+ Hath.Server.FloodControl: data IP
+ Hath.Server.FloodControl: data IPRecord
+ Hath.Server.FloodControl: instance GHC.Classes.Eq Hath.Server.FloodControl.IP
+ Hath.Server.FloodControl: instance GHC.Classes.Eq Hath.Server.FloodControl.IPRecord
+ Hath.Server.FloodControl: instance GHC.Classes.Ord Hath.Server.FloodControl.IP
+ Hath.Server.FloodControl: instance GHC.Show.Show Hath.Server.FloodControl.IP
+ Hath.Server.FloodControl: instance GHC.Show.Show Hath.Server.FloodControl.IPRecord
+ Hath.Server.FloodControl: isLocalIP :: IP -> Bool
+ Hath.Server.FloodControl: isRPCServer :: [ByteString] -> IP -> Bool
+ Hath.Server.FloodControl: newFloodControl :: IO FloodControl
+ Hath.Server.FloodControl: parseIPv4 :: ByteString -> Maybe IP
+ Hath.Server.FloodControl: registerHit :: Double -> IP -> IPMap -> (IPMap, Bool)
+ Hath.Server.FloodControl: sockAddrIP :: SockAddr -> Maybe IP
+ Hath.Server.FloodControl: type IPMap = Map IP IPRecord
+ Hath.Server.Handlers: application :: ServerEnv -> Application
+ Hath.Server.LegacyCiphers: cipher_ECDHE_ECDSA_AES128CBC_SHA :: Cipher
+ Hath.Server.LegacyCiphers: cipher_ECDHE_RSA_AES128CBC_SHA :: Cipher
+ Hath.Server.LegacyCiphers: cipher_ECDHE_RSA_AES256CBC_SHA :: Cipher
+ Hath.Server.Middleware: rateLimitMiddleware :: FloodControl -> (IP -> IO Bool) -> Middleware
+ Hath.Server.Middleware: tracingConnections :: StatsEnv -> Middleware
+ Hath.Server.Middleware: tracingTimeUsage :: StatsEnv -> Middleware
+ Hath.ServerTime: serverNow :: IO Int64
+ Hath.ServerTime: setServerTime :: Int64 -> IO ()
+ Hath.Settings: HathSettings :: !ByteString -> !Int -> !Int64 -> !Int64 -> !Int64 -> ![ByteString] -> !Int -> !Int -> !Int -> HathSettings
+ Hath.Settings: [clientHost] :: HathSettings -> !ByteString
+ Hath.Settings: [clientPort] :: HathSettings -> !Int
+ Hath.Settings: [diskLimitBytes] :: HathSettings -> !Int64
+ Hath.Settings: [diskRemainingBytes] :: HathSettings -> !Int64
+ Hath.Settings: [maxAllowedFileSize] :: HathSettings -> !Int
+ Hath.Settings: [maxConnectionsOverride] :: HathSettings -> !Int
+ Hath.Settings: [maxFilenameLength] :: HathSettings -> !Int
+ Hath.Settings: [rpcServerIPs] :: HathSettings -> ![ByteString]
+ Hath.Settings: [throttleBytes] :: HathSettings -> !Int64
+ Hath.Settings: data HathSettings
+ Hath.Settings: defaultHathSettings :: HathSettings
+ Hath.Settings: instance GHC.Show.Show Hath.Settings.HathSettings
+ Hath.Settings: maxConnections :: HathSettings -> Int
+ Hath.Settings: parseSettings :: HathSettings -> [ByteString] -> HathSettings
+ Hath.Settings: settingNumber :: ByteString -> [ByteString] -> Maybe Int64
+ Hath.Stats: addDlBytes :: forall (r :: EffectRow). Members '[Reader StatsEnv, Embed IO] r => Int -> Sem r ()
+ Hath.Stats: addDownload :: forall (r :: EffectRow). Members '[Reader StatsEnv, Embed IO] r => Int -> Sem r ()
+ Hath.Stats: addUpload :: forall (r :: EffectRow). Members '[Reader StatsEnv, Embed IO] r => Int -> Sem r ()
+ Hath.Stats: data StatsEnv
+ Hath.Stats: incDlFile :: forall (r :: EffectRow). Members '[Reader StatsEnv, Embed IO] r => Sem r ()
+ Hath.Stats: incDlTask :: forall (r :: EffectRow). Members '[Reader StatsEnv, Embed IO] r => Sem r ()
+ Hath.Stats: incFetched :: forall (r :: EffectRow). Members '[Reader StatsEnv, Embed IO] r => Sem r ()
+ Hath.Stats: incServed :: forall (r :: EffectRow). Members '[Reader StatsEnv, Embed IO] r => Sem r ()
+ Hath.Stats: newStatsEnv :: IO StatsEnv
+ Hath.Stats: observeDuration :: StatsEnv -> Double -> IO ()
+ Hath.Stats: readMetrics :: forall (r :: EffectRow). Members '[Reader StatsEnv, Embed IO] r => Sem r Text
+ Hath.Stats: renderMetrics :: StatsEnv -> Int64 -> IO Text
+ Hath.Stats: trackConnection :: StatsEnv -> IO a -> IO a
+ Hath.Version: versionInfo :: Text

Files

CHANGELOG.md view
@@ -1,10 +1,97 @@ # Changelog for `hs-hath` -All notable changes to this project will be documented in this file.- The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the [Haskell Package Versioning Policy](https://pvp.haskell.org/).++## [1.3.0] - 2026-11-18++### Upgrade notes+- **`/metrics` and `/admin/settings`** answer only a request that sends `Authorization: Bearer <token>`, the token being the value of `HATH_METRICS_TOKEN` in the environment. Without the variable both are gone (404); a Prometheus scrape needs `authorization: { credentials: <token> }` (see Security)+- **Reverse proxy**: behind a reverse proxy, run hs-hath with `--trust-proxy-headers`. Server commands are now accepted only from the addresses `rpc_server_ip` lists and from local ones (loopback, private), taken from the peer as flood control sees it. A proxy or tunnel on a local address that does not pass the reader's address on therefore still works: its commands are judged by their key alone, as every command was before+- **`client_start` failures**: when `client_start` fails for a reason other than `FAIL_OTHER_CLIENT_CONNECTED` (another client connected) or `FAIL_CID_IN_USE` (client id in use), for example `FAIL_CONNECT_TEST` or no reply, the process stays up without announcing itself, as the official client does, instead of retrying every second. A supervisor will not restart it+- **Blacklist**: blacklisted files are now deleted from the cache (SQLite, Filesystem and R2); before, the list was only printed+- **Certificate**: a certificate with less than 24 h left is refused at start+- **Gallery layout**: galleries are downloaded into a new directory layout, `<downloadDir>/<title> [<gid>]` (see Changed); files under the old `<downloadDir>/<gid>/` are not recognised, so those galleries are fetched again+- **`/h/` requests** need valid `fileindex` and `xres` options even for cached files (404 otherwise), as with the official client+- **Library API**: modules and exports were removed, and signatures changed (see Changed). Removed modules: `Hath.Metrics.Counter`, `Hath.Metrics.Gauge`, `Hath.Metrics.Histogram`, `Hath.Metrics.Prometheus`, `Hath.Metrics.Registry`, `Hath.Metrics.Snapshot` and `Hath.Metrics.Types` (`Hath.Stats` holds the counters and renders them). Removed exports: from `Hath.Stats` the `Stats` effect (`Stats(..)`, `runStats`, `readPrometheus`) and the fields of `StatsEnv`; `Hath.Cache.SQLite.FileRecord`; `Hath.Locate.streamUpstream`; `Hath.Gallery.Downloader.downloadNextGallery`; the re-exports of `CacheRunner(..)` and `GalleryTask(..)` from `Hath.Server`; the constructor and fields of `Hath.Server.FloodControl.IPRecord` and its `Hashable IP` instance; the `galleryFileCount` field of `Hath.Gallery.GalleryMetadata`; the `optShowVersion` field of `Hath.CLI.Options` (the option parser answers `--version` itself); `Hath.Server.Middleware.normalizeAcceptMiddleware` and `withHathHeaders`++### Security+- **Dependencies with published advisories**: the snapshot is lts-24.67 (was lts-24.27), and `stack.yaml` and `stack-release.yaml` take `warp-3.4.21` and `http2-5.4.14` as extra dependencies, with `http-semantics-0.4.2` and `time-manager-0.5.2`, which they need. warp before 3.4.17 could be made to read one request as two, which lets a request past a reverse proxy in front of it ([HSEC-2026-0013](https://osv.dev/vulnerability/HSEC-2026-0013)); http2 before 5.4.14 took a CONNECT naming two hosts ([HSEC-2026-0015](https://osv.dev/vulnerability/HSEC-2026-0015)); the snapshot brings aeson 2.2.5.1 ([HSEC-2026-0007](https://osv.dev/vulnerability/HSEC-2026-0007)) and scientific 0.3.10.0 ([HSEC-2026-0011](https://osv.dev/vulnerability/HSEC-2026-0011)). `file-io` is held at the compiler's 0.1.5. Still open: crypton-x509 1.7 does not enforce X.509 Name Constraints ([HSEC-2026-0008](https://osv.dev/vulnerability/HSEC-2026-0008)); the fix is in 1.9.1, which `tls` 2.1 cannot be built with+- **`/metrics` and `/admin/settings`** answered anyone who reached the port, and the second lists the RPC servers' addresses. They now need a token (see Upgrade notes), compared in constant time; a wrong or missing one gets 403+- **Flood control** counts a request to any path, as the official client counts every connection; it only counted `/h/`, so `/t/`, `/servercmd/` and the rest could be asked for without limit+- **Keys and signatures** (keystamps, the keys of server commands and speed tests, the signature of an RPC reply, a gallery list's `DLHASH`) are compared in constant time, so the time a refusal takes says nothing about how close a guess was+- **Cache misses**: a file being fetched is held in memory until it is cached, and every request for it meanwhile fetched it again, so one valid link asked for often enough could take all the memory. The downloads under way may now hold 512 MiB between them; past that a miss is answered 503 until some have finished (a single larger file is still fetched when nothing else is under way)+- **RPC replies** longer than 10 MiB are refused, as by the official client. The RPC server is reached over plain HTTP, and a reply without end from whoever sits in between was read into memory until none was left+- **CI**: the workflows' token is read-only except in the job that creates the release, and the actions that are not GitHub's own are pinned to a commit (the release image to a digest) instead of a tag that can be moved+- `.gitignore` lists the files that hold secrets (`client-login`, `.env`, `*.p12`, `service/client.toml`, `r2/wrangler.toml`), so that they cannot be committed by accident++### Added+- **Connection limit**: as in the official client, a new connection is closed without a reply while more than `max_connections` others are open (the server's `max_connections`, or else 20 plus one per 10,000 bytes/s of `throttle_bytes`, up to 500). Local addresses and the RPC servers are never refused, but count. With more than 80 % taken, the RPC server is told (`act=overload`), at most every 30 s and without holding up the connection. Warp only knows the peer's address, so behind a reverse proxy on a local address nothing is limited+- `max_allowed_filesize` from the server's settings (1 GiB unless it says otherwise) is enforced: a cache miss, a gallery file or a `threaded_proxy_test` download larger than it is not fetched+- **`galleryinfo.txt`**: once all of a gallery's files are there, the list's INFORMATION section is written to it+- Library modules `Hath.ServerTime` (`serverNow`, `setServerTime`), `Hath.Download` (`fetchFile`, and `receive`, moved from `Hath.Locate` and given the time a read may wait) and `Hath.Server.ConnectionLimit`++### Changed+- **Gallery layout** as in the official client: a gallery goes into `<downloadDir>/<title> [<gid>]`, or `[<gid>-<xres>x]` for a resized one, its title cleaned up the same way and cut short with `...` to `max_filename_length` (from the server's settings, 125 by default); it used to be `<downloadDir>/<gid>/`. If the file system will not take the name, the directory is `<gid>` or `<gid>-<xres>x`. A file already there and intact is kept, a damaged one deleted+- **Certificate refresh** swaps the certificate in place for new handshakes instead of restarting the listener, so transfers in progress are no longer cut off and `client_start` is not re-sent+- **Filesystem backend**: hits are sent from the file instead of being read into memory, and the least recently served files are evicted to stay under the disk limit from the H@H settings page (100 MiB below it, as the official client). Its per-request in-memory LRU, which never survived past one request, is gone+- **Metrics**: `hath_latency_seconds` (a gauge holding the last request's latency in microseconds) is replaced by the histogram `hath_request_duration_seconds`; the always-zero `hath_errors_total` is removed. `/metrics` is rendered when it is read, not every 5 seconds by a background thread, so it shows current values and `hath_uptime_seconds`, which was always 0, counts; the other names, their order and the format are unchanged+- `client-login.example` uses the Filesystem backend+- **Logs**: failed TLS handshakes and connections that close before sending anything (scanners probing SSL3/TLS 1.0/1.1 and cipher suites, a few thousand a day on a public node) are no longer printed. The TLS 1.0/1.1 entries are dropped from the allowed versions; `tls` 2.x rejected them anyway+- **TLS session resumption** (session IDs on TLS 1.2, tickets on TLS 1.3): a returning visitor skips the certificate and its RSA signature, which cuts the server's CPU per handshake by about 45 % on TLS 1.3 and 75 % on TLS 1.2 (several ms on a small VM)+- **Internals**: the modules live under `Hath.*`, split by subject+- `/admin/settings` lists only the settings the server assigns+- The SQLite backend no longer records the file name a file was first requested under, and a hit no longer writes to the database (the `lru_counter` column, which nothing read, stays as it is)+- **R2 backend**: a file just uploaded after a cache miss is redirected at once on its next request, instead of first asking the bucket whether it is there (a 40–350 ms round trip)+- `hath-bench` writes the hs-hath config through `ClientConfig`'s `ToDhall` instance instead of a hand-written copy of the record+- **RPC requests** are sent as the official client sends them: the query carries `clientbuild`, `act`, `add`, `cid`, `acttime`, `actkey` in that order, `add=` even when empty, with the `;` inside `add` unescaped. Requests to the RPC server, to image servers (cache misses) and for gallery files carry `User-Agent: Hentai@Home 1.6.6` and, as in the official client, `Connection: Close` (RPC, gallery files) or `Hath-Request` (image servers), and nothing else: no longer our response headers (`Cache-Control`, `Server`, `X-Content-Type-Options`) or `Accept-Encoding: gzip`+- **RPC transport**: a request whose exchange fails (no connection, a broken reply, an HTTP error other than 404/410) is sent again at once, three times in all, and a connection attempt gives up after 5 s, as in the official client. The reply may still take up to 60 s+- **Responses** carry the official client's headers: `Server`, `Connection: close`, `Content-Type` and `Content-Length`, and `Cache-Control` only with a body; no more `User-Agent` or `X-Content-Type-Options`. An empty body says `Content-Length: 0` where the official client says nothing and closes. An error is the official page, `An error has occurred. (<status>)` as `text/html; charset=ISO-8859-1`; so are the replies of server commands, and `robots.txt` is `text/plain; charset=ISO-8859-1`. A request other than GET or HEAD gets 400. As in the official client, the path is not percent-decoded except for `%3d`. Flood control's 429, and Range and If-Modified-Since for files from disk, are as they were+- **Library API** (RPC): `CallRPC` and `CallDownloader` take `add` as `Text`, empty when there is none; `HathError`'s `TransportError` carries a `Text`; `serverStat` returns `()` and fails unless the server answers OK; `clientLogin` and `clientSettings` need `Embed IO`; `Hath.Protocol.clientBuild` is an `Int64`. New: `Hath.Settings.settingNumber`, `Hath.Protocol`'s `rpcQuery`, `outgoingHeaders` and `retryWithServerTime`, `Hath.RPC.worthRetrying`+- **Library API** (stats, R2): `Hath.Stats` counts with plain functions over `Reader StatsEnv` (no effect to interpret) and adds `readMetrics`, `renderMetrics`, `trackConnection` and `observeDuration`; `App` no longer carries `Reader ServerEnv` or `AtomicState HathSettings`; `Hath.Cache.R2.putR2Object` uploads a file (the cache and `migrate-to-r2` share it)+- **Library API** (lifecycle): `Hath.Cache.Cache` has `DeleteFile` (`deleteFile`); `Hath.RPC.clientStart` returns `()`, and it, `clientStop` and `stillAlive` fail with the server's status unless it answers OK; `getBlacklist` returns `FileURI`s; new `Hath.RPC.StartFailure`, `startFailure` and `toldToStop`, and `Hath.Server.needsRenewal`; `startServer` returns after a shutdown instead of exiting. The library depends on `hourglass` (crypton-x509's date type)+- **Library API** (HTTP): `Hath.Server.API` reads paths (`Route`, `route`, `ServerCommand`, `ProxyTest`, `parseServerCommand`) instead of declaring a servant API; `Hath.Server.Handlers` exports the WAI `application` instead of `server`, which took the `ServerEnv`; `App` ends a request by throwing a `Status` instead of a `ServerError`, and `runApp` returns `IO (Either Status a)`. `Hath.Protocol`: `responseHeaders`, `keystampValid`, `commandTimeValid` and `fetchParams` replace `hathHeaders`, `parseKeystamp`, `maxKeystampDrift` and `maxCommandDrift`; `hathRequestToken` takes the file id as requested, as does `Hath.RPC.fileSources` (now `fileSources fileIndex xres fileId`) and `Hath.Locate.locateResource`. `Hath.FileURI.parseFileURI` takes only the ids the official client takes, and `renderFileURIBytes` writes an id without a resolution in three parts. New: `Hath.Server.FloodControl.isRPCServer`+- **Library API** (cache misses, connection limit): `Hath.Locate.locateResource` takes `max_allowed_filesize` and returns `Either Status Located` (the status for the reader) instead of `Maybe Located`; new are `Hath.Locate.usableReply`, `Hath.RPC.httpManager` (one manager for RPC, proxy tests, gallery files and cache misses, whose connections give up after 5 s), `connectMicros` and `overload`, `Hath.Settings.maxConnections`, and the `HathSettings` fields `maxConnectionsOverride` and `maxAllowedFileSize`+- **Library API** (gallery downloader): `runGalleryDownloader` takes the settings and an `MVar ()` that `start_downloader` fills (`ServerEnv`'s `envDownloadRequest`, replacing `envGalleryTasks`; `GalleryTask` is gone); `downloadQueue` (one run) and `anotherRound` are exported, `galleryFilePath` is not. `GalleryMetadata` has `galleryTitle` (cleaned up) and `galleryInformation`; new `Hath.Gallery.galleryDirectoryName` and `fallbackDirectoryName`, and `HathSettings`' `maxFilenameLength`. `Hath.RPC.nextGallery` takes the gallery to mark done, `galleryFileSource` replaces `galleryFileSources`, `failureReport` is new and `completeGallery` is gone+- **TLS 1.3 early data (0-RTT)** is no longer accepted: a request sent in early data can be replayed. The vendored warp-tls is 3.4.18 (was 3.4.13, which took up to 2018 bytes of it), still with the one added line that lets TLS 1.2 clients without the Extended Main Secret extension connect. It needs `network` 3.2.12 or later, which the snapshot has (see Security)++### Fixed+- **Settings refresh** (`refresh_settings`) reset every setting the reply did not repeat to its default; it now updates only what it carries+- **Filesystem backend** could not start (the startup cache check opened `cachePath` as an SQLite file); the check now only runs for the SQLite backend+- Files are written through a temp file and renamed, so a crash no longer leaves a truncated file that later gets served (gallery downloads and the Filesystem cache). Truncated files left by older versions are dropped and refetched+- **`downloadDir`** and `--download-dir` had no effect: the gallery downloader always wrote to `./download`. It now writes to the configured directory+- The CI workflow now runs the test suite+- The release build (`stack-release.yaml`) pointed at directories that no longer exist+- **Clock skew**: keystamps, server commands, speed tests and the signatures of our RPC requests are now checked against the RPC server's clock, taken from the `server_time` of the `server_stat`, `client_login` and `client_settings` replies, as the official client does. A node whose clock is off no longer fails to log in or refuses readers with "time is out of sync". A request without arguments that gets `KEY_EXPIRED` takes the time from `server_stat` and is sent once more+- **`server_stat`**: a reply other than a signed OK (`FAIL`, a forged reply) no longer counts as the server being available. A server that requires a newer client (`min_client_build` above 181) stops hs-hath at startup with a message saying so; a newer client being out (`cur_client_build`) is logged as a warning+- **Start order**: the port is bound before `client_start` is sent, so the server's connectivity test no longer races the listener. `client_start` is sent once, not every second until it succeeds; `FAIL_OTHER_CLIENT_CONNECTED` and `FAIL_CID_IN_USE` stop hs-hath with the official client's explanation (any other failure: see Upgrade notes). `client_settings` is fetched once after it, and `still_alive`, the blacklist and the certificate check only start once it went through+- **Shutdown** no longer cuts transfers off: as in the official client, `client_stop` is sent once, and only if `client_start` went through; then hs-hath serves 5 s more, stops accepting connections and waits up to 25 s for open ones. It used to stop serving first, then send `client_stop` up to three times+- **`still_alive`** is sent every 110 s instead of every minute, as in the official client; `TERM_BAD_NETWORK` shuts the client down as above, and other failures are logged instead of dropped+- **Blacklist** as in the official client: fetched once after `client_start` (the last three days), then every 6 hours (the last 12 hours), instead of before `client_start` and every 36 minutes; only the number of files is logged+- **Certificate expiry**: hs-hath looks at the certificate every 5 minutes; with less than a day left it fetches a new one and swaps it in, and shuts down as above if none comes or the new one expires within a day too (also after `refresh_certs`)+- **Keystamps** are accepted while less than 15 minutes from the server's time either way, as in the official client (it was 10 minutes). Speed tests (`/t/`) allow 5 minutes, as server commands do (it was 10), and work without the part after the key+- **HEAD** goes through the same checks as GET and gets the same status and headers (for a file its own Content-Type and Content-Length), without the body. A HEAD for a file not in the cache fetches it into the cache, as in the official client. Any HEAD used to get 200 unchecked+- **Server commands** are only taken from the RPC servers' addresses, as in the official client, or from a local one (see Upgrade notes). A command with a part missing gets 403; the command name is read in any case; an unknown command, or one whose parameters cannot be read, is answered `INVALID_COMMAND` with 200 (it was 404 or 403); `refresh_settings` answers 200 and nothing whether or not the refresh worked (a failed one was 500)+- **File ids** are checked as the official client checks them, and a malformed one gets 404 without the RPC server being asked: a lower-case SHA-1, the five-part form or the three-part one without a resolution (which used to get 404), a size that fits a Java int, one of the eight types. `fileindex` must be a number and `xres` a number or `org`, also for a file in the cache. `srfetch` and `Hath-Request` carry the file id as the reader wrote it, not as we would write it. `/h/<file id>/<options>` is served without the file name after it, as by the official client+- **`threaded_proxy_test`** downloads go through the same code as gallery files (`Hath.Download.fetchFile`), as in the official client: a connection gets 5 s and every read 10 s, a download three attempts (none after a 404 or 410), `User-Agent` and `Connection: Close` are sent, and a reply larger than `max_allowed_filesize` fails. The time is taken from the download's first byte to its end (it counted from before connecting), and a download counts if its body had the announced length and at least the test size. All of a test's attempts stay within 60 s+- **Cache misses** are downloaded on a thread of their own, as by the official client: a reader who goes away no longer aborts the download, and the file is still checked (length, SHA-1) and cached. It is held in memory once, in one buffer that the reader is relayed from and the cache receives; it used to be kept as chunks and then copied once more+- **Upstream timeouts on a cache miss**: a source gets 5 s to connect and 30 s for its headers, every read 30 s, and the whole body 300 s, as in the official client; a stalled image server used to hold the reader and the upstream connection indefinitely. When the download breaks off (upstream closes early, stalls or runs out of time), the reader's connection is dropped at once+- **Cache miss statuses** as in the official client: 404 when `srfetch` fails or names no source (a failed `srfetch` gave 500), 502 if a source answered with something other than the file, 500 if none could be reached (both gave 404)+- **Gallery downloads longer than five minutes**: every request of one download carried the time the download began, so the server came to refuse them as `KEY_EXPIRED`. Each request now carries its own time+- Errors that end a gallery download are logged; they used to be dropped+- An option value containing `=` (in `/h/` and `/servercmd/` URLs) no longer fails the request+- **Gallery downloader queue**: one `start_downloader` works through the whole queue, as in the official client. Each `fetchqueue` after the first marks the gallery before it done, whether or not all its files came, and brings the next one, until the server answers `NO_PENDING_DOWNLOADS`. It used to download one gallery per `start_downloader`, throw away the next gallery the server had sent along, and never mark a gallery with a failed file done, so the server sent that one again and again. A failed `dlfetch` is one failed attempt at that file; it used to end the gallery. A `start_downloader` that comes during a run is ignored, as by the official client (the run picks up what was queued meanwhile)+- **Gallery retries and failure reports** as in the official client: the files still missing are gone through again, for at most nine rounds and while fewer downloads have failed than twice the gallery's number of files, with a pause of 1 s after each download and 5 s after each failure. `dlfetch` numbers the attempts at a file from 1 (it sent 0 to 3), and only the first source it names is tried. Every failure, a broken download as well as a wrong SHA-1, is recorded as `host-fileindex-xres`; the distinct ones are reported in one `dlfails` per gallery, and only if there are 1 to 50 of them. An empty `dlfails` used to follow every failed file+- **Gallery files** are checked as the official FileDownloader checks them: a status below 400, a Content-Length that is present, at most `max_allowed_filesize` and matched by the body, up to three attempts (none after a 404 or 410), 5 s to connect and 10 s for every read. An error page or a cut-off body used to be saved as the image when the list gives no SHA-1 for it+- `hath_download_task_count_total` counts the galleries taken from the queue; it counted every look at the queue, empty ones included++### Removed+- The library API removals are listed under Upgrade notes+- The disabled static-range check: like the official client, any file the server routes here is served+- The dependencies `servant`, `servant-server`, `servant-client`, `servant-client-core` and `http-media`: requests are routed by their path, as the official client routes them, and the RPC client uses `http-client` directly+- The dependencies `extra`, `http-conduit` and `unordered-containers` of hs-hath itself (the vendored `minio-hs` keeps its own), `mason`, `timers`, `suspend`, a dozen more that nothing imported, and the pinned `random`; `crypton-connection` is only a dependency of `hath-bench` (and of the vendored `minio-hs`)+- The vendored `polysemy-zoo`, the internal library `hs-hath-placeholder` (vendored `placeholder`), the parts of the vendored `minio-hs` that are not built, and `Setup.hs`  ## [1.2.0] - 2026-11-08 
README.md view
@@ -1,80 +1,77 @@-# hs-hath - A Haskell Implementation of the Hentai@Home Client--[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)--A high-performance Haskell implementation of the Hentai@Home client. The client caches and serves gallery files, communicates with the H@H server via TLS-secured RPC, and supports S3-compatible storage backends including Cloudflare R2.+# hs-hath -## Features+A [Hentai@Home](https://ehwiki.org/wiki/Hentai@Home) client written in Haskell, also on [Hackage](https://hackage.haskell.org/package/hs-hath). -Resource caching uses an LRU cache with SQLite-backed or R2-backed storage. RPC communication to the H@H server uses TLS with automatic certificate refresh every 24 hours. Rate limiting includes both IP-based throttling and keystamp-based fallback for NAT environments. Gallery downloads support parallel file fetching. Prometheus metrics are available at the `/metrics` endpoint. The client tracks build information via the `--version` flag.+It caches files on the local disk, in an SQLite database, or in S3-compatible storage (such as Cloudflare R2), uses Dhall for configuration, and exposes Prometheus metrics at `/metrics`. -## Installation+## Getting Started  ### Prerequisites-- Stack (Haskell toolchain)-- SQLite 3-- OpenSSL libraries -### Building-```bash-git clone https://github.com/pe200012/hs-hath.git-cd hs-hath-stack build-```+* [Stack](https://docs.haskellstack.org/)+* SQLite 3+* The `openssl` command, only to run the test suite and `hath-bench`, which create certificates with it -## Configuration+### Installation -Create a `client-login` file with your credentials:-```dhall-{ clientId = "your-client-id"-, key = "your-api-key" -, version = "1.0.0"-, proxy = None ClientProxy-, downloadDir = "./downloads"-, cachePath = "./cache.db"-}-```+1. Build the project:+   ```sh+   stack build+   ```+2. Copy `client-login.example` to `client-login` and fill in your credentials and paths:+   ```sh+   cp client-login.example client-login+   ```+   ```dhall+   in  ClientConfig::{+       , clientId = "<YOURID>"+       , key = "<YOURKEY>"+       , version = "176"+       , downloadDir = "./download"+       , cachePath = "./cache"+       , cacheBackend = CacheBackend.CacheBackendFilesystem+       , r2Config = None R2Config+       }+   ```+   The example defines the record's types and defaults, so the file only names what differs. `version`, `proxy` and `lruCacheSize` are not read. `cacheBackend` is one of: +   * `CacheBackendFilesystem`: files under `cachePath` (a directory) as `xxxx/<file id>`, sent straight from disk. The least recently served files are evicted to stay under the disk limit set on the H@H settings page.+   * `CacheBackendSQLite`: files stored in the database file at `cachePath`. Nothing is evicted, and every cached file is checked against its hash at start unless `--skip-startup-verify` is given.+   * `CacheBackendR2`: `r2Config = Some { r2Endpoint = "https://<account id>.r2.cloudflarestorage.com", r2Bucket = "<bucket>" }`, with `R2_ACCESS_KEY` and `R2_SECRET_KEY` in the environment. A hit is answered with a redirect to a presigned link; nothing is evicted.+ ## Usage -Start the client:-```bash-stack exec hs-hath-```+Run the client: -View version information:-```bash-stack exec hs-hath -- --version+```sh+stack exec hs-hath ``` -The client responds to `SIGINT` and `SIGTERM` for graceful shutdown. Settings reload automatically via RPC command. The Prometheus metrics endpoint is available at `/metrics` for monitoring.--## Architecture--The RPC client handles communication with the H@H server, including certificate management and settings synchronization. The resource cache stores files using SQLite with LRU eviction. An HTTP server built on Warp serves cached resources. Rate limiting middleware provides both IP-based and keystamp-based request throttling. The storage backend supports local filesystem and S3-compatible services like Cloudflare R2.--## Metrics--The client exports Prometheus metrics in text format at the `/metrics` endpoint. Metrics include request counters, cache statistics, and operational counters for monitoring and alerting.--## Performance--Has not been optimized yet. Currently, it will take ~200MiB RAM on average.+The configuration is read from `./client-login`; `-c FILE` names another file. `stack exec hs-hath -- --help` lists the other options. -## Troubleshooting+Prometheus metrics are served at `/metrics`, and the settings the server assigned at `/admin/settings`, once `HATH_METRICS_TOKEN` is set in the environment (any long random string, for example from `openssl rand -hex 32`). A request must send it as `Authorization: Bearer <token>`; in Prometheus: -### Certificate errors-Verify system clock synchronization and certificate file permissions. Certificate refresh occurs automatically every 24 hours.+```yaml+scrape_configs:+  - job_name: hath+    scheme: https+    authorization:+      credentials: <token>+    static_configs:+      - targets: ["<host>:<port>"]+``` -### Connection issues-Check proxy settings if configured. Ensure firewall allows outbound TLS connections on the required ports.+Without the variable both paths answer 404, like any path the official client does not have. -### Storage backend errors-For S3-compatible backends like Cloudflare R2, verify that the storage server returns required headers: `ETag`, `Content-Length`, and `Last-Modified`. Missing `ETag` headers cause write failures.+## Notes & Caveats -### Rate limiting-The client implements strict request throttling. Banned IPs and rate limit violations appear in console logs. Verify client configuration if legitimate requests are being rejected.+* **Memory usage**: Not optimized yet — expect ~200 MiB RAM on average.+* **S3 / Cloudflare R2**: The object store must return `ETag`, `Content-Length`, and `Last-Modified` headers; missing `ETag` will fail cache writes.+* **Reverse proxy**: Run with `--trust-proxy-headers`, which takes the reader's address from `X-Forwarded-For` when the proxy is on a loopback or private address. Flood control then counts readers, and server commands (`/servercmd/`) are taken from the RPC servers' addresses only. The connection limit sees only the peer's address and never refuses a local one, so it does not apply to a proxy on a local address.+* **Tunnels** (rathole, frp, a plain port forward that rewrites the source): hs-hath sees every connection come from a local address and cannot tell readers apart, so neither flood control nor the connection limit applies; limit connections where the tunnel meets the internet. A server command from a local address is accepted on its key alone.+* **Start-up**: If the server refuses `client_start` because another client is connected or the client id is in use, hs-hath exits. After any other failure (for example `FAIL_CONNECT_TEST`) it stays up without announcing itself, as the official client does, so that the port can be tested; restart it once the cause is fixed. A certificate with less than 24 h left is refused at start.+* **Galleries** are saved as `<downloadDir>/<title> [<gid>]`, or `[<gid>-<xres>x]` for a resized one, and `galleryinfo.txt` is written once all of a gallery's files are there.  ## License -This project is licensed under the GNU General Public License v3.0 - see the [LICENSE](LICENSE) file for details.+Distributed under the GPL-3.0 License. See [`LICENSE`](LICENSE).
− Setup.hs
@@ -1,2 +0,0 @@-import Distribution.Simple-main = defaultMain
app/Bench.hs view
@@ -1,5 +1,3 @@-{-# LANGUAGE BangPatterns #-}- -- | hath-bench: plays the H@H RPC server for a client running on this machine -- (hs-hath or the official Java client), replays the measurements the real -- server makes, and prints a scorecard.@@ -11,18 +9,26 @@ --   * readers pulling /h/ files, cold (srfetch + upstream) and warm module Bench ( main ) where +import           Control.Concurrent       ( forkIO, threadDelay )+import           Control.Exception        ( try )+ import qualified Data.ByteString          as BS import qualified Data.ByteString.Builder  as BB import qualified Data.ByteString.Char8    as BSC-import qualified Data.ByteString.Lazy     as LBS import qualified Data.List                as List import qualified Data.Map.Strict          as Map import           Data.String.Interpolate  ( i ) import qualified Data.Text                as T import           Data.Time.Clock.System   ( getSystemTime, systemSeconds ) +import           Dhall                    ( embed, inject )+import qualified Dhall.Core               as Dhall+ import           GHC.Clock                ( getMonotonicTime ) +import           Hath.Config              ( ClientConfig(..), defaultClientConfig )+import           Hath.Protocol            ( sha1HexText )+ import           Network.Connection       ( TLSSettings(TLSSettingsSimple) ) import qualified Network.HTTP.Client      as HC import           Network.HTTP.Client.TLS  ( mkManagerSettings )@@ -42,81 +48,97 @@  import           Text.Printf              ( printf ) -import           Control.Concurrent       ( forkIO, threadDelay )--import           Control.Exception        ( try )- import           UnliftIO                 ( pooledMapConcurrentlyN, timeout ) -import           Utils                    ( hash )- data Opts-  = Opts-  { oClientId   :: !Text-  , oClientKey  :: !Text-  , oRpcPort    :: !Int-  , oClientPort :: !Int-  , oFiles      :: !Int-  , oFileSize   :: !Int-  , oUpDelayMs  :: !Int-  , oUpKBps     :: !Int-  , oWorkDir    :: !FilePath-  , oClientHost :: !Text -- ^ where the bench reaches the client-  , oPublicHost :: !Text -- ^ where the client reaches the bench (also its rpc_server_ip)-  }+  = Opts { oClientId   :: !Text+         , oClientKey  :: !Text+         , oRpcPort    :: !Int+         , oClientPort :: !Int+         , oFiles      :: !Int+         , oFileSize   :: !Int+         , oUpDelayMs  :: !Int+         , oUpKBps     :: !Int+         , oWorkDir    :: !FilePath+         , oClientHost :: !Text -- ^ where the bench reaches the client+         , oPublicHost :: !Text -- ^ where the client reaches the bench (also its rpc_server_ip)+         }  optsP :: Parser Opts optsP   = Opts-  <$> strOption (long "client-id" <> value "12345" <> showDefault <> help "Client ID of the client under test")+  <$> strOption+    (long "client-id" <> value "12345" <> showDefault <> help "Client ID of the client under test")   <*> strOption     (long "client-key"      <> value "benchbenchbenchbench"      <> showDefault      <> help "Client key (the Java client wants 20 alphanumerics)")-  <*> option auto+  <*> option+    auto     (long "rpc-port"      <> value 19800      <> showDefault      <> help "Port of the fake RPC server, image server and speed-test target")-  <*> option auto (long "client-port" <> value 18443 <> showDefault <> help "Port the client is told to listen on")-  <*> option auto (long "files" <> value 30 <> showDefault <> help "Gallery size for the /h/ phases")-  <*> option auto (long "file-size" <> value 300000 <> showDefault <> help "Bytes per gallery file")-  <*> option auto+  <*> option+    auto+    (long "client-port"+     <> value 18443+     <> showDefault+     <> help "Port the client is told to listen on")+  <*> option+    auto+    (long "files" <> value 30 <> showDefault <> help "Gallery size for the /h/ phases")+  <*> option+    auto+    (long "file-size" <> value 300000 <> showDefault <> help "Bytes per gallery file")+  <*> option+    auto     (long "upstream-delay-ms"      <> value 300      <> showDefault      <> help "Latency the fake image server adds before its first byte")-  <*> option auto+  <*> option+    auto     (long "upstream-kbps"      <> value 1000      <> showDefault      <> help "Fake image server bandwidth in KB/s (0 = unlimited)")-  <*> strOption (long "work-dir" <> value "bench-work" <> showDefault <> help "Where certs and client configs go")-  <*> strOption (long "client-host" <> value "127.0.0.1" <> showDefault <> help "Address of the client under test")   <*> strOption+    (long "work-dir"+     <> value "bench-work"+     <> showDefault+     <> help "Where certs and client configs go")+  <*> strOption+    (long "client-host"+     <> value "127.0.0.1"+     <> showDefault+     <> help "Address of the client under test")+  <*> strOption     (long "public-host"      <> value "127.0.0.1"      <> showDefault-     <> help "Address the client uses to reach this bench; anything but 127.0.0.1 listens on all IPv4 interfaces")+     <> help+       "Address the client uses to reach this bench; anything but 127.0.0.1 listens on all IPv4 interfaces")  data GFile = GFile { gIndex :: !Int, gId :: !Text, gBytes :: !ByteString }  data Bench-  = Bench-  { bOpts     :: !Opts-  , bP12      :: !ByteString-  , bFiles    :: !(Map Text GFile)-  , bRpcCalls :: !(IORef (Map Text Int))-  , bSrfetch  :: !(IORef Int)-  , bStarted  :: !(MVar ())-  }+  = Bench { bOpts     :: !Opts+          , bP12      :: !ByteString+          , bFiles    :: !(Map Text GFile)+          , bRpcCalls :: !(IORef (Map Text Int))+          , bSrfetch  :: !(IORef Int)+          , bStarted  :: !(MVar ())+          }  main :: IO () main = do   hSetBuffering stdout LineBuffering   o <- execParser-    (info (optsP <**> helper) (fullDesc <> progDesc "Fake H@H RPC server that measures a local client"))+    (info+       (optsP <**> helper)+       (fullDesc <> progDesc "Fake H@H RPC server that measures a local client"))   when (oFileSize o < 64) $ die "--file-size must be at least 64"   wd <- makeAbsolute (oWorkDir o)   createDirectoryIfMissing True (wd </> "java" </> "data")@@ -125,14 +147,16 @@   -- a fresh nonce per run gives new file ids, so the cold phase is really cold   nonce <- systemSeconds <$> getSystemTime   let files = mkGallery nonce (oFiles o) (oFileSize o)-  b <- Bench o p12 (Map.fromList [ ( gId f, f ) | f <- files ])-    <$> newIORef mempty+  b <- Bench o p12 (Map.fromList [ ( gId f, f ) | f <- files ]) <$> newIORef mempty     <*> newIORef 0     <*> newEmptyMVar   void     $ forkIO     $ Warp.runSettings-      (Warp.setHost (if oPublicHost o == "127.0.0.1" then "127.0.0.1" else "*4")+      (Warp.setHost+         (if oPublicHost o == "127.0.0.1"+            then "127.0.0.1"+            else "*4")        $ Warp.setPort (oRpcPort o) Warp.defaultSettings)       (app b)   putTextLn (instructions o wd)@@ -152,38 +176,55 @@ makeCert :: Opts -> FilePath -> IO ByteString makeCert o wd = do   environment <- getEnvironment-  let openssl args = void+  let openssl args+        = void         $ readCreateProcess           (proc "openssl" args)           { env = Just (( "HATH_BENCH_PASS", toString (oClientKey o) ) : environment) }           ""   openssl-    [ "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "7", "-subj", "/CN=127.0.0.1"-    , "-keyout", wd </> "key.pem", "-out", wd </> "cert.pem" ]+    [ "req"+    , "-x509"+    , "-newkey"+    , "rsa:2048"+    , "-nodes"+    , "-days"+    , "7"+    , "-subj"+    , "/CN=127.0.0.1"+    , "-keyout"+    , wd </> "key.pem"+    , "-out"+    , wd </> "cert.pem"+    ]   openssl-    [ "pkcs12", "-export", "-inkey", wd </> "key.pem", "-in", wd </> "cert.pem"-    , "-out", wd </> "cert.p12", "-passout", "env:HATH_BENCH_PASS"-    , "-name", "hath.network" ] -- the alias the Java client looks up+    [ "pkcs12"+    , "-export"+    , "-inkey"+    , wd </> "key.pem"+    , "-in"+    , wd </> "cert.pem"+    , "-out"+    , wd </> "cert.p12"+    , "-passout"+    , "env:HATH_BENCH_PASS"+    , "-name"+    , "hath.network"+    ] -- the alias the Java client looks up   readFileBS (wd </> "cert.p12")  writeClientConfigs :: Opts -> FilePath -> IO () writeClientConfigs o wd = do   writeFileText (wd </> "java" </> "data" </> "client_login") [i|#{oClientId o}-#{oClientKey o}|]-  writeFileText-    (wd </> "hs-hath.dhall")-    [i|let CacheBackend = < CacheBackendSQLite | CacheBackendR2 | CacheBackendFilesystem >-let ClientProxy = { host : Text, port : Integer, auth : Optional { _1 : Text, _2 : Text } }-in  { clientId = "#{oClientId o}"-    , key = "#{oClientKey o}"-    , version = "176"-    , proxy = None ClientProxy-    , downloadDir = "#{wd}/hs-download"-    , cachePath = "#{wd}/hs-cache.db"-    , cacheBackend = CacheBackend.CacheBackendSQLite-    , r2Config = None { r2Endpoint : Text, r2Bucket : Text }-    , lruCacheSize = +104857600-    }-|]+  writeFileText (wd </> "hs-hath.dhall")+    $ Dhall.pretty+    $ embed+      inject+      defaultClientConfig { clientId    = oClientId o+                          , key         = oClientKey o+                          , downloadDir = toText (wd </> "hs-download")+                          , cachePath   = toText (wd </> "hs-cache.db")+                          }  instructions :: Opts -> FilePath -> Text instructions o wd@@ -197,13 +238,16 @@ Waiting for client_start...|]  mkGallery :: Int64 -> Int -> Int -> [ GFile ]-mkGallery nonce n size-  = [ GFile k fileId body-    | k <- [ 1 .. n ]-    , let prefix = encodeUtf8 ([i|hath-bench #{nonce} #{k} |] :: Text)-          body   = prefix <> BS.replicate (size - BS.length prefix) 46-          fileId = [i|#{hash @Text body}-#{size}-1280-1707-jpg|] ]+mkGallery nonce n size = file <$> [ 1 .. n ]+  where+    file k = GFile k fileId body+      where+        prefix = encodeUtf8 ([i|hath-bench #{nonce} #{k} |] :: Text) +        body   = prefix <> BS.replicate (size - BS.length prefix) 46++        fileId = [i|#{sha1HexText body}-#{size}-1280-1707-jpg|]+ -- | RPC endpoint, fake image server (/up) and speed-test target (/t) on one port. app :: Bench -> Wai.Application app b req respond = case Wai.pathInfo req of@@ -223,21 +267,24 @@       $ find ((k <> "=") `BS.isPrefixOf`)       $ BSC.split '&' (BS.dropWhile (== 63) (Wai.rawQueryString req)) -    -- hs-hath's servant client insists on the real server's content type,-    -- the Java client on a Content-Length+    -- the Java client insists on a Content-Length     rpcResponse body       = respond       $ Wai.responseLBS         H.status200-        [ ( H.hContentType, "text/plain;charset=UTF-8" ), ( H.hContentLength, show (BS.length body) ) ]-        (LBS.fromStrict body)+        [ ( H.hContentLength, show (BS.length body) ) ]+        (fromStrict body)      -- 1.6.6 signs replies: OK-<t>-<sha1("<t>-<key>" ++ concatMap ("\n" ++) lines)>     okWith signed ls = do       now <- systemSeconds <$> getSystemTime-      let sig = hash @Text (encodeUtf8 (T.concat ([i|#{now}-#{oClientKey o}|] : map ("\n" <>) ls)))-          status = if signed then [i|OK-#{now}-#{sig}|] else "OK"-      rpcResponse (encodeUtf8 (T.unlines (status : ls)))+      let sig+            = sha1HexText (encodeUtf8 (T.concat ([i|#{now}-#{oClientKey o}|] : map ("\n" <>) ls)))+          status+            = if signed+              then [i|OK-#{now}-#{sig}|]+              else "OK"+      rpcResponse (encodeUtf8 (unlines (status : ls)))      settings       = [ [i|rpc_server_ip=#{oPublicHost o}|]@@ -247,7 +294,8 @@         , "throttle_bytes=640000000"         , "disable_bwm=true"         , "disklimit_bytes=1073741824"-        , "diskremaining_bytes=1073741824" ]+        , "diskremaining_bytes=1073741824"+        ]      rpc signed = do       let ok = okWith signed@@ -255,7 +303,8 @@       atomicModifyIORef' (bRpcCalls b) (\m -> ( Map.insertWith (+) act 1 m, () ))       now <- systemSeconds <$> getSystemTime       case act of-        "server_stat" -> ok [ "min_client_build=1", "cur_client_build=999", [i|server_time=#{now}|] ]+        "server_stat"+          -> ok [ "min_client_build=1", "cur_client_build=999", [i|server_time=#{now}|] ]         "client_login" -> ok settings         "client_settings" -> ok settings         "get_cert" -> rpcResponse (bP12 b)@@ -281,31 +330,32 @@             when (oUpKBps o > 0) $ threadDelay (BS.length c * 1000 `div` oUpKBps o)      testTarget sizeText testKey = case readMaybe @Int (toString sizeText) of-      Nothing -> respond $ Wai.responseLBS H.status400 [] ""+      Nothing   -> respond $ Wai.responseLBS H.status400 [] ""       Just size         | testKey == "broken403" -> respond $ Wai.responseLBS H.status403 [] ""         -- no Content-Length and only half the bytes: the tester sees a clean, early EOF         | testKey == "brokenshort" -> zeros [] (size `div` 2)         | otherwise -> zeros [ ( H.hContentLength, show size ) ] size -    zeros headers n-      = respond-      $ Wai.responseStream H.status200 headers-      $ \write _ -> forM_ (chunksOf 65536 (BS.replicate n 48)) (write . BB.byteString)+    zeros headers n = respond $ Wai.responseStream H.status200 headers $ \write _+      -> forM_ (chunksOf 65536 (BS.replicate n 48)) (write . BB.byteString)  chunksOf :: Int -> ByteString -> [ ByteString ] chunksOf n bs   | BS.null bs = []-  | otherwise = let ( a, rest ) = BS.splitAt n bs in a : chunksOf n rest+  | otherwise+    = let+        ( a, rest ) = BS.splitAt n bs+      in +        a : chunksOf n rest  data Sample-  = Sample-  { sStatus :: !(Maybe Int) -- ^ Nothing: connection error or timeout-  , sTtfb   :: !Double      -- ^ seconds until response headers-  , sTotal  :: !Double      -- ^ seconds until the body is drained-  , sBytes  :: !Int-  , sHead   :: !ByteString  -- ^ first bytes of the body-  }+  = Sample { sStatus :: !(Maybe Int) -- ^ Nothing: connection error or timeout+           , sTtfb   :: !Double      -- ^ seconds until response headers+           , sTotal  :: !Double      -- ^ seconds until the body is drained+           , sBytes  :: !Int+           , sHead   :: !ByteString  -- ^ first bytes of the body+           }  -- | GET with redirects followed, timing headers and body separately. fetch :: HC.Manager -> String -> IO Sample@@ -317,7 +367,11 @@       t1 <- getMonotonicTime       let drain !n acc = HC.brRead (HC.responseBody resp) >>= \c -> if BS.null c             then pure ( n, acc )-            else drain (n + BS.length c) (if BS.length acc < 256 then acc <> BS.take 256 c else acc)+            else drain+              (n + BS.length c)+              (if BS.length acc < 256+                 then acc <> BS.take 256 c+                 else acc)       ( n, headBytes ) <- drain 0 ""       t2 <- getMonotonicTime       pure $ Sample (Just (H.statusCode (HC.responseStatus resp))) (t1 - t0) (t2 - t0) n headBytes@@ -327,25 +381,25 @@     _ -> Sample Nothing (t3 - t0) (t3 - t0) 0 ""  data Row-  = Row-  { rowName    :: !Text-  , rowSamples :: ![ Sample ]-  , rowWall    :: !Double-  , rowPassed  :: !Bool-  , rowNote    :: !Text-  }+  = Row { rowName    :: !Text+        , rowSamples :: ![ Sample ]+        , rowWall    :: !Double+        , rowPassed  :: !Bool+        , rowNote    :: !Text+        }  runPhases :: Bench -> HC.Manager -> [ GFile ] -> IO [ Row ] runPhases b mgr files = do   now <- systemSeconds <$> getSystemTime   let o = bOpts b       base = [i|https://#{oClientHost o}:#{oClientPort o}|] :: String-      sha1 = hash @Text . encodeUtf8 @Text+      sha1 = sha1HexText . encodeUtf8 @Text        servercmd :: Text -> Text -> String       servercmd cmd add         = let-            sig = sha1 [i|hentai@home-servercmd-#{cmd}-#{add}-#{oClientId o}-#{now}-#{oClientKey o}|]+            sig+              = sha1 [i|hentai@home-servercmd-#{cmd}-#{add}-#{oClientId o}-#{now}-#{oClientKey o}|]           in              [i|#{base}/servercmd/#{cmd}/#{add}/#{now}/#{sig}|] @@ -371,29 +425,42 @@         let ( passed, note ) = check ss         pure $ Row name ss (t1 - t0) passed note -      allFull size ss = all (\s -> sStatus s == Just 200 && sBytes s == size) ss+      allFull size = all (\s -> sStatus s == Just 200 && sBytes s == size)       bodyText = decodeUtf8 . sHead :: Sample -> Text-      bodyIs p ss = ( all (\s -> sStatus s == Just 200 && p (bodyText s)) ss, T.intercalate " " (bodyText <$> ss) )+      bodyIs p ss+        = ( all (\s -> sStatus s == Just 200 && p (bodyText s)) ss+          , T.intercalate " " (bodyText <$> ss)+          )        galleryPhase name expectFetches = do         before <- readIORef (bSrfetch b)         row <- phase name 4 (fileUrl <$> files) (\ss -> ( allFull (oFileSize o) ss, "" ))         fetches <- subtract before <$> readIORef (bSrfetch b)-        pure row { rowPassed = rowPassed row && fetches == expectFetches-                 , rowNote = [i|srfetch=#{fetches} (want #{expectFetches})|] }+        pure+          row { rowPassed = rowPassed row && fetches == expectFetches+              , rowNote   = [i|srfetch=#{fetches} (want #{expectFetches})|]+              }    -- for poking the client from a third machine (e.g. a flood-control burst)-  writeFileText (oWorkDir o </> "urls.txt") (T.unlines (toText . fileUrl <$> files))+  writeFileText (oWorkDir o </> "urls.txt") (unlines (toText . fileUrl <$> files))   sequence     [ phase "servercmd still_alive" 1 [ servercmd "still_alive" "" ] (bodyIs (const True))-    , phase "servercmd speed_test" 2 (replicate 2 (servercmd "speed_test" "testsize=2500000"))+    , phase+        "servercmd speed_test"+        2+        (replicate 2 (servercmd "speed_test" "testsize=2500000"))         (\ss -> ( allFull 2500000 ss, "" ))-    , phase "servercmd unsigned" 1 [ [i|#{base}/servercmd/still_alive//#{now}/0|] ]+    , phase+        "servercmd unsigned"+        1+        [ [i|#{base}/servercmd/still_alive//#{now}/0|] ]         (\ss -> ( map sStatus ss == [ Just 403 ], "want 403" ))     , phase "proxy_test healthy" 1 [ proxyTest "ok" ] (bodyIs ("OK:5-" `T.isPrefixOf`))     , phase "proxy_test target 403" 1 [ proxyTest "broken403" ] (bodyIs (== "OK:0-0"))     , phase "proxy_test short body" 1 [ proxyTest "brokenshort" ] (bodyIs (== "OK:0-0"))-    , phase "/t/ 5x5MB concurrent" 5+    , phase+        "/t/ 5x5MB concurrent"+        5         [ [i|#{base}/t/5000000/#{now}/#{speedKey}/#{k}|] | k <- [ 1 .. 5 :: Int ] ]         (\ss -> ( allFull 5000000 ss, "" ))     , galleryPhase "/h/ cold (cache miss)" (length files)@@ -403,21 +470,37 @@ report :: Opts -> [ Row ] -> Map Text Int -> IO () report o rows calls = do   printf "\nhath-bench results (client on %s:%d)\n\n" (toString (oClientHost o)) (oClientPort o)-  printf "%-24s %-14s %15s %15s %8s  %s\n"-    ("phase" :: String) ("status" :: String) ("ttfb p50/p95" :: String)-    ("total p50/p95" :: String) ("MB/s" :: String) ("check" :: String)+  printf+    "%-24s %-14s %15s %15s %8s  %s\n"+    ("phase" :: String)+    ("status" :: String)+    ("ttfb p50/p95" :: String)+    ("total p50/p95" :: String)+    ("MB/s" :: String)+    ("check" :: String)   forM_ rows $ \r -> do-    let ss = rowSamples r-        statuses = T.unwords-          [ [i|#{maybe "err" (show @Text) s}x#{c}|] :: Text-          | ( s, c ) <- Map.toList (Map.fromListWith (+) [ ( sStatus x, 1 :: Int ) | x <- ss ]) ]+    let ss       = rowSamples r+        statuses+          = unwords+            [ [i|#{maybe "err" (show @Text) s}x#{c}|] :: Text+            | ( s, c ) <- Map.toList (Map.fromListWith (+) [ ( sStatus x, 1 :: Int ) | x <- ss ])+            ]         ms f = [i|#{msAt 0.5 f ss}/#{msAt 0.95 f ss}|] :: Text-        mbps = fromIntegral (sum (sBytes <$> ss)) / max 1e-9 (rowWall r) / 1e6 :: Double-    printf "%-24s %-14s %15s %15s %8.1f  %s %s\n"-      (toString (rowName r)) (toString statuses) (toString (ms sTtfb)) (toString (ms sTotal)) mbps-      (if rowPassed r then "ok" else "FAIL" :: String) (toString (rowNote r))-  printf "\nRPC calls seen: %s\n"-    (toString (T.unwords [ [i|#{a}x#{n}|] | ( a, n ) <- Map.toList calls ]))+        mbps     = fromIntegral (sum (sBytes <$> ss)) / max 1e-9 (rowWall r) / 1e6 :: Double+    printf+      "%-24s %-14s %15s %15s %8.1f  %s %s\n"+      (toString (rowName r))+      (toString statuses)+      (toString (ms sTtfb))+      (toString (ms sTotal))+      mbps+      (if rowPassed r+         then "ok"+         else "FAIL" :: String)+      (toString (rowNote r))+  printf+    "\nRPC calls seen: %s\n"+    (toString (unwords [ [i|#{a}x#{n}|] | ( a, n ) <- Map.toList calls ]))   where     msAt :: Double -> (Sample -> Double) -> [ Sample ] -> Int     msAt p f ss = case sort (f <$> ss) of
app/Main.hs view
@@ -1,86 +1,72 @@ module Main ( main ) where -import           CLI                    ( Options(..), applyOptionsToConfig, parseOptions )--import qualified Data.Text              as T- import           Database.SQLite.Simple ( withConnection ) -import           FileVerification       ( VerificationStats(..), verifyAllFiles )--import           HathNetwork.Genesis    ( fetchCertificate, runGenesisIO )-import           HathNetwork.RPC--import           Relude                 hiding ( runReader )--import           Server                 ( ServerAction(GracefulShutdown), startServer )+import           Hath.CLI               ( Options(..), applyOptionsToConfig, parseOptions )+import           Hath.Cache.SQLite      ( VerificationStats(..), initializeDB, verifyAllFiles )+import           Hath.Config            ( CacheBackend(..), ClientConfig(..), readClientConfig )+import           Hath.RPC               ( clientLogin, getCertificate, runRPCIO, serverStat )+import           Hath.Server            ( ServerAction(GracefulShutdown)+                                        , ServerOptions(..)+                                        , needsRenewal+                                        , startServer+                                        )+import           Hath.ServerTime        ( serverNow ) -import           Storage.Database       ( initializeDB )+import           Relude  import           System.Posix           ( Handler(Catch), installHandler, sigINT, sigTERM ) -import           Types- import           UnliftIO               ( newTChanIO, writeTChan ) -import           Version                ( versionInfo )- main :: IO () main = do   options <- parseOptions--  -- Handle --version flag-  when (optShowVersion options) $ do-    putStrLn $ T.unpack versionInfo-    exitSuccess--  baseConfig <- readClientConfig (toText $ optConfigPath options)-  let config = applyOptionsToConfig options baseConfig-+  config <- applyOptionsToConfig options <$> readClientConfig (toText $ optConfigPath options)   when (optValidateConfig options) $ do     putStrLn "Configuration is valid."     exitSuccess -  -- Initialize database and run startup verification unless skipped-  withConnection (T.unpack $ cachePath config) $ \conn -> do-    initializeDB conn+  -- for the other backends cachePath is a directory, or not used at all+  when (cacheBackend config == CacheBackendSQLite) $ prepareSQLiteCache options config -    -- Run full cache verification by default (skip with --skip-startup-verify)-    unless (optSkipStartupVerify options) $ do-      putStrLn "Verifying cache integrity (this may take a while)..."-      stats <- verifyAllFiles conn-      putStrLn $ "Verification complete:"-      putStrLn $ "  Files verified: " <> show (verifiedCount stats)-      putStrLn $ "  Corrupted files removed: " <> show (corruptedCount stats)-      putStrLn $ "  Time elapsed: " <> show (verificationTime stats)-      when (corruptedCount stats > 0)-        $ putStrLn-        $ "WARNING: " <> show (corruptedCount stats) <> " corrupted files were found and removed."+  actions <- newTChanIO+  forM_ [ sigINT, sigTERM ] $ \signal+    -> installHandler signal (Catch $ atomically $ writeTChan actions GracefulShutdown) Nothing -  chan <- newTChanIO-  void $ installHandler sigINT (Catch $ atomically $ writeTChan chan GracefulShutdown) Nothing-  void $ installHandler sigTERM (Catch $ atomically $ writeTChan chan GracefulShutdown) Nothing+  runRPCIO config serverStat >>= either (`giveUp` "RPC is not available, exiting...") pure+  settings <- runRPCIO config clientLogin >>= either (`giveUp` "Unable to login, exiting...") pure+  certs <- runRPCIO config getCertificate+    >>= either (`giveUp` "Unable to fetch certs, exiting...") pure+  now <- serverNow+  when (needsRenewal now (fst certs)) $ do+    putStrLn "The certificate from the server expires within a day; try again later."+    exitFailure+  startServer+    config+    ServerOptions { disableRateLimit  = optDisableRateLimit options+                  , trustProxyHeaders = optTrustProxyHeaders options+                  }+    settings+    certs+    actions -  runRPCIO config serverStat >>= \case-    Right (Right True) -> runRPCIO config clientLogin >>= \case-      Right (Right settings) -> runGenesisIO config fetchCertificate >>= \case-        Right (Right certs) -> startServer-          config-          settings-          certs-          chan-          (optDisableRateLimit options)-          (optTrustProxyHeaders options)-        e -> do-          print e-          putStrLn "Unable to fetch certs, exiting..."-          exitFailure-      e -> do-        print e-        putStrLn "Unable to login, exiting..."-        exitFailure-    e -> do-      print e-      putStrLn "RPC is not available, exiting..."-      exitFailure+prepareSQLiteCache :: Options -> ClientConfig -> IO ()+prepareSQLiteCache options config = withConnection (toString $ cachePath config) $ \conn -> do+  initializeDB conn+  unless (optSkipStartupVerify options) $ do+    putStrLn "Verifying cache integrity (this may take a while)..."+    stats <- verifyAllFiles conn+    putStrLn "Verification complete:"+    putStrLn $ "  Files verified: " <> show (verifiedCount stats)+    putStrLn $ "  Corrupted files removed: " <> show (corruptedCount stats)+    putStrLn $ "  Time elapsed: " <> show (verificationTime stats)+    when (corruptedCount stats > 0)+      $ putStrLn+      $ "WARNING: " <> show (corruptedCount stats) <> " corrupted files were found and removed." +giveUp :: Show failure => failure -> String -> IO a+giveUp failure message = do+  print failure+  putStrLn message+  exitFailure
app/Migrate.hs view
@@ -1,50 +1,27 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeApplications #-}-+-- | migrate-to-r2: upload an SQLite or Filesystem cache to an R2 bucket. module Migrate ( main ) where -import           Colog                  ( LogAction(..), Message, Severity(..), richMessageAction )-import           Colog.Polysemy         ( Log, runLogAction )--import qualified Conduit                as C--import           Control.Exception      ( SomeException, try )-import           Control.Monad          ( when )--import qualified Data.ByteString        as BS-import qualified Data.Text              as T-import qualified Data.Text.IO           as TIO--import           Database.SQLite.Simple ( Connection, Only(..), Query, fold_, open, query_ )+import           Database.SQLite.Simple ( Only(..), fold_, query_, withConnection ) -import           Network.Minio          ( AccessKey(..)-                                        , CredentialValue(..)-                                        , SecretKey(..)-                                        , defaultPutObjectOptions-                                        , runMinio-                                        , setCreds-                                        , setRegion+import           Hath.Cache.R2          ( R2Connection(..)+                                        , fileURIToKey+                                        , mkR2Connection+                                        , putR2Object+                                        , runR2                                         )+import           Hath.Config            ( R2Config(..) )+import           Hath.FileURI           ( FileURI, parseFileURI )++import           Network.Minio          ( defaultPutObjectOptions ) import qualified Network.Minio          as Minio  import           Options.Applicative -import           Polysemy-import           Polysemy.Error         ( Error, runError )-import           Polysemy.KVStore       ( updateKV )--import           Relude                 hiding ( Reader )--import           Storage.Database       ( FileRecord(..), initializeDB )-import           Storage.R2             ( R2Connection(..), mkR2Connection, runCacheR2 )+import           Relude  import           System.Directory       ( doesDirectoryExist, doesFileExist, listDirectory )-import           System.FilePath        ( (</>), takeFileName )--import           Types                  ( FileURI, R2Config(..), RPCError(..), parseFileURI, renderFileURIText )+import           System.FilePath        ( (</>) ) --- | Command line options data MigrateOptions   = MigrateOptions   { optSourceType :: !SourceType@@ -55,12 +32,9 @@   , optBatchSize  :: !Int   , optVerbose    :: !Bool   }-  deriving ( Show )  data SourceType = SQLiteSource | FilesystemSource-  deriving ( Show, Eq ) --- | Parse command line options parseOptions :: Parser MigrateOptions parseOptions   = MigrateOptions@@ -107,173 +81,66 @@ main :: IO () main = do   opts <- execParser optsInfo--  -- Check environment variables-  accessKeyMay <- lookupEnv "R2_ACCESS_KEY"-  secretKeyMay <- lookupEnv "R2_SECRET_KEY"--  when (isNothing accessKeyMay) $ do-    TIO.putStrLn "Error: R2_ACCESS_KEY environment variable not set"-    exitFailure--  when (isNothing secretKeyMay) $ do-    TIO.putStrLn "Error: R2_SECRET_KEY environment variable not set"-    exitFailure--  -- Create R2 connection-  let r2Cfg = R2Config { r2Endpoint = optR2Endpoint opts, r2Bucket = optR2Bucket opts }--  r2Result <- mkR2Connection r2Cfg-  case r2Result of-    Left err     -> do-      TIO.putStrLn $ "Error creating R2 connection: " <> err+  mkR2Connection R2Config { r2Endpoint = optR2Endpoint opts, r2Bucket = optR2Bucket opts } >>= \case+    Left err -> do+      putTextLn $ "Error creating R2 connection: " <> err       exitFailure-    Right r2Conn -> do-      TIO.putStrLn $ "Connected to R2: " <> optR2Endpoint opts <> "/" <> optR2Bucket opts--      case optSourceType opts of-        SQLiteSource     -> migrateSQLite opts r2Conn-        FilesystemSource -> migrateFilesystem opts r2Conn---- | Migrate from SQLite database-migrateSQLite :: MigrateOptions -> R2Connection -> IO ()-migrateSQLite opts r2Conn = do-  TIO.putStrLn $ "Migrating from SQLite: " <> T.pack (optSourcePath opts)--  conn <- open (optSourcePath opts)--  -- Count total files-  [ Only totalCount ] <- query_ conn "SELECT COUNT(*) FROM files" :: IO [ Only Int ]-  TIO.putStrLn $ "Total files to migrate: " <> show totalCount--  when (optDryRun opts) $ do-    TIO.putStrLn "[DRY RUN] Would upload the following files:"--  -- Stream files from SQLite and upload to R2-  migratedCount <- newIORef (0 :: Int)-  errorCount <- newIORef (0 :: Int)--  void $ fold_ conn selectAllQuery ( (), 0 :: Int ) $ \( (), _idx ) record -> do-    let fileId = fileRecordFileId record-        uri    = parseFileURI (encodeUtf8 fileId)--    if optDryRun opts-      then do-        when (optVerbose opts) $ TIO.putStrLn $ "  Would upload: " <> fileId-      else do-        result <- uploadToR2 r2Conn uri record-        case result of-          Left err -> do-            atomicModifyIORef' errorCount (\c -> ( c + 1, () ))-            when (optVerbose opts)-              $ TIO.putStrLn-              $ "  Error uploading " <> fileId <> ": " <> show err-          Right () -> do-            atomicModifyIORef' migratedCount (\c -> ( c + 1, () ))-            when (optVerbose opts) $ TIO.putStrLn $ "  Uploaded: " <> fileId--    -- Progress update every batch-    let idx' = _idx + 1-    when (idx' `mod` optBatchSize opts == 0)-      $ TIO.putStrLn-      $ "  Progress: " <> show idx' <> "/" <> show totalCount--    pure ( (), idx' )+    Right r2 -> do+      putTextLn $ "Connected to R2: " <> optR2Endpoint opts <> "/" <> optR2Bucket opts+      tally <- case optSourceType opts of+        SQLiteSource     -> migrateSQLite opts r2+        FilesystemSource -> migrateFilesystem opts r2+      putTextLn "\nMigration complete:"+      putTextLn $ "  Migrated: " <> show (migrated tally)+      putTextLn $ "  Errors: " <> show (failed tally) -  finalMigrated <- readIORef migratedCount-  finalErrors <- readIORef errorCount+data Tally = Tally { migrated :: !Int, failed :: !Int } -  TIO.putStrLn $ "\nMigration complete:"-  TIO.putStrLn $ "  Migrated: " <> show finalMigrated-  TIO.putStrLn $ "  Errors: " <> show finalErrors+migrateOne :: MigrateOptions -> Text -> (FileURI -> IO (Either Text ())) -> Tally -> IO Tally+migrateOne opts fileId upload tally+  | optDryRun opts = tally <$ say ("  Would upload: " <> fileId)+  | otherwise+    = maybe (pure $ Left "not a file id") upload (parseFileURI (encodeUtf8 fileId)) >>= \case+      Left err -> tally { failed = failed tally + 1 }+        <$ say ("  Error uploading " <> fileId <> ": " <> err)+      Right () -> tally { migrated = migrated tally + 1 } <$ say ("  Uploaded: " <> fileId)   where-    selectAllQuery :: Query-    selectAllQuery = "SELECT lru_counter, s4, file_id, file_name, bytes FROM files"+    say = when (optVerbose opts) . putTextLn --- | Migrate from filesystem cache-migrateFilesystem :: MigrateOptions -> R2Connection -> IO ()-migrateFilesystem opts r2Conn = do-  TIO.putStrLn $ "Migrating from filesystem: " <> T.pack (optSourcePath opts)+-- | Rows are streamed, so the database is never in memory as a whole.+migrateSQLite :: MigrateOptions -> R2Connection -> IO Tally+migrateSQLite opts r2 = withConnection (optSourcePath opts) $ \conn -> do+  putTextLn $ "Migrating from SQLite: " <> toText (optSourcePath opts)+  total <- maybe (0 :: Int) fromOnly . listToMaybe <$> query_ conn "SELECT COUNT(*) FROM files"+  putTextLn $ "Total files to migrate: " <> show total+  when (optDryRun opts) $ putTextLn "[DRY RUN] Would upload the following files:"+  fmap fst+    $ fold_ conn "SELECT file_id, bytes FROM files" ( Tally 0 0, 0 :: Int )+    $ \( tally, done ) ( fileId, bytes ) -> do+      tally' <- migrateOne opts fileId (\uri -> putR2Object r2 uri bytes) tally+      when ((done + 1) `mod` optBatchSize opts == 0)+        $ putTextLn+        $ "  Progress: " <> show (done + 1) <> "/" <> show total+      pure ( tally', done + 1 ) -  exists <- doesDirectoryExist (optSourcePath opts)-  unless exists $ do-    TIO.putStrLn $ "Error: Directory does not exist: " <> T.pack (optSourcePath opts)+-- | The cache is @root\/xxxx\/\<file id>@, xxxx being the first four characters of the id.+migrateFilesystem :: MigrateOptions -> R2Connection -> IO Tally+migrateFilesystem opts r2 = do+  putTextLn $ "Migrating from filesystem: " <> toText root+  unlessM (doesDirectoryExist root) $ do+    putTextLn $ "Error: Directory does not exist: " <> toText root     exitFailure--  -- Filesystem cache structure: optSourcePath/s4/file_id-  -- Where s4 is first 4 chars of hash-  s4Dirs <- listDirectory (optSourcePath opts)--  when (optDryRun opts) $ TIO.putStrLn "[DRY RUN] Would upload the following files:"--  migratedCount <- newIORef (0 :: Int)-  errorCount <- newIORef (0 :: Int)--  forM_ s4Dirs $ \s4Dir -> do-    let s4Path = optSourcePath opts </> s4Dir-    isDir <- doesDirectoryExist s4Path-    when isDir $ do-      files <- listDirectory s4Path-      forM_ files $ \fileName -> do-        let filePath = s4Path </> fileName-            fileId   = T.pack fileName-            uri      = parseFileURI (encodeUtf8 fileId)--        isFile <- doesFileExist filePath-        when isFile $ do-          if optDryRun opts-            then do-              when (optVerbose opts) $ TIO.putStrLn $ "  Would upload: " <> fileId-            else do-              let bucket = r2ConnBucket r2Conn-                  key    = fileURIToKey uri-              result <- try @SomeException-                $ Minio.runMinioWith (r2MinioConn r2Conn)-                $ Minio.fPutObject bucket key filePath defaultPutObjectOptions-              case result of-                Left err          -> do-                  atomicModifyIORef' errorCount (\c -> ( c + 1, () ))-                  when (optVerbose opts)-                    $ TIO.putStrLn-                    $ "  Error uploading " <> fileId <> ": " <> show err-                Right (Left mErr) -> do-                  atomicModifyIORef' errorCount (\c -> ( c + 1, () ))-                  when (optVerbose opts)-                    $ TIO.putStrLn-                    $ "  Error uploading " <> fileId <> ": " <> show mErr-                Right (Right ())  -> do-                  atomicModifyIORef' migratedCount (\c -> ( c + 1, () ))-                  when (optVerbose opts) $ TIO.putStrLn $ "  Uploaded: " <> fileId--  finalMigrated <- readIORef migratedCount-  finalErrors <- readIORef errorCount--  TIO.putStrLn $ "\nMigration complete:"-  TIO.putStrLn $ "  Migrated: " <> show finalMigrated-  TIO.putStrLn $ "  Errors: " <> show finalErrors---- | Upload a file record to R2-uploadToR2 :: R2Connection -> FileURI -> FileRecord -> IO (Either Text ())-uploadToR2 conn uri record = do-  let key     = fileURIToKey uri-      bucket  = r2ConnBucket conn-      content = fileRecordBytes record-      size    = fromIntegral (BS.length content) :: Int64-      src     = C.yield content--  result <- try @SomeException-    $ Minio.runMinioWith (r2MinioConn conn)-    $ Minio.putObject bucket key src (Just size) defaultPutObjectOptions--  case result of-    Left err          -> pure $ Left $ T.pack $ show err-    Right (Left mErr) -> pure $ Left $ T.pack $ show mErr-    Right (Right _)   -> pure $ Right ()---- | Generate R2 object key from FileURI (same as in R2.hs)-fileURIToKey :: FileURI -> Text-fileURIToKey uri = s4 <> "/" <> fileId+  when (optDryRun opts) $ putTextLn "[DRY RUN] Would upload the following files:"+  dirs <- filterM (doesDirectoryExist . (root </>)) =<< listDirectory root+  foldlM migrateDirectory (Tally 0 0) dirs   where-    fileId = renderFileURIText uri+    root = optSourcePath opts -    s4     = T.take 4 fileId+    migrateDirectory tally dir = foldlM (migrateFile dir) tally =<< listDirectory (root </> dir)++    migrateFile dir tally name = do+      let path       = root </> dir </> name+          upload uri+            = runR2 r2+            $ Minio.fPutObject (r2ConnBucket r2) (fileURIToKey uri) path defaultPutObjectOptions+      ifM (doesFileExist path) (migrateOne opts (toText name) upload tally) (pure tally)
hs-hath.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           hs-hath-version:        1.2.0+version:        1.3.0 synopsis:       A Haskell implementation of the Hentai@Home client description:    Please see the README on GitHub at <https://github.com/pe200012/hs-hath#readme> category:       Web@@ -13,7 +13,7 @@ bug-reports:    https://github.com/pe200012/hs-hath/issues author:         pe200012 maintainer:     1326263755@qq.com-copyright:      2024 pe200012+copyright:      2024-2026 pe200012 license:        GPL-3 license-file:   LICENSE build-type:     Simple@@ -21,6 +21,11 @@     GHC == 9.12.2 extra-source-files:     README.md+    vendor/cryptostore-0.3.0.1/LICENSE+    vendor/minio-hs-1.7.0/LICENSE+    vendor/warp-tls-3.4.18/LICENSE+    vendor/warp-tls-3.4.18/certificate.pem+    vendor/warp-tls-3.4.18/key.pem extra-doc-files:     CHANGELOG.md @@ -30,29 +35,34 @@  library   exposed-modules:-      CLI-      FileVerification-      HathNetwork.Genesis-      HathNetwork.RPC-      Interface.API-      LegacyCiphers-      Metrics.Counter-      Metrics.Gauge-      Metrics.Prometheus-      Metrics.Registry-      Metrics.Snapshot-      Metrics.Types-      Middleware-      Server-      SettingM-      Stats-      Storage.Database-      Storage.Filesystem-      Storage.Locate-      Storage.R2-      Types-      Utils-      Version+      Hath.AtomicFile+      Hath.Cache+      Hath.Cache.Filesystem+      Hath.Cache.R2+      Hath.Cache.SQLite+      Hath.CLI+      Hath.Config+      Hath.Download+      Hath.Error+      Hath.FileURI+      Hath.Gallery+      Hath.Gallery.Downloader+      Hath.Locate+      Hath.Log+      Hath.Protocol+      Hath.RPC+      Hath.Server+      Hath.Server.API+      Hath.Server.App+      Hath.Server.ConnectionLimit+      Hath.Server.FloodControl+      Hath.Server.Handlers+      Hath.Server.LegacyCiphers+      Hath.Server.Middleware+      Hath.ServerTime+      Hath.Settings+      Hath.Stats+      Hath.Version   other-modules:       Paths_hs_hath   autogen-modules:@@ -61,77 +71,46 @@       src   default-extensions:       NoImplicitPrelude-      TypeApplications       OverloadedStrings-      ScopedTypeVariables       QuasiQuotes       LambdaCase-      ViewPatterns-      MultiParamTypeClasses       GADTs   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints   build-depends:-      aeson >=2.1 && <2.3-    , async ==2.2.*-    , base >=4.7 && <4.22-    , base64 ==1.0.*+      base >=4.7 && <4.22     , bytestring >=0.11 && <0.13     , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*     , co-log-polysemy >=0.0.1.5 && <0.0.2     , conduit ==1.3.*     , containers >=0.6 && <0.8     , crypton ==1.0.*-    , crypton-connection ==0.4.*     , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*     , dhall ==1.42.*     , directory ==1.3.*-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9     , filepath >=1.4 && <1.6-    , hex-text ==0.1.*+    , hourglass ==0.2.*     , hs-hath-cryptostore     , hs-hath-minio-    , hs-hath-placeholder-    , hs-hath-polysemy-zoo     , hs-hath-warp-tls     , http-client ==0.7.*     , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*     , http-types ==0.12.*     , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3     , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*     , network >=3.1 && <3.3     , optparse-applicative ==0.18.*     , polysemy ==1.9.*     , process ==1.6.*     , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21     , sqlite-simple ==0.4.*-    , stm ==2.5.*     , string-interpolate ==0.3.*-    , suspend ==0.2.*     , template-haskell >=2.19 && <2.24     , text >=2.0 && <2.2     , time >=1.12 && <1.15-    , timers ==0.2.*     , tls >=2.0 && <2.4     , tls-session-manager >=0.0.5 && <0.1-    , transformers ==0.6.*     , unix ==2.8.*     , unliftio ==0.2.*-    , unordered-containers ==0.2.*     , wai ==3.2.*     , wai-extra ==3.1.*     , warp ==3.4.*@@ -177,68 +156,42 @@       vendor/cryptostore-0.3.0.1/src   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -w   build-depends:-      aeson >=2.1 && <2.3-    , asn1-encoding ==0.9.*+      asn1-encoding ==0.9.*     , asn1-types >=0.3.1 && <0.4-    , async ==2.2.*     , base >=4.7 && <4.22-    , base64 ==1.0.*     , basement >=0.0.16 && <0.1     , bytestring >=0.11 && <0.13     , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*     , co-log-polysemy >=0.0.1.5 && <0.0.2     , conduit ==1.3.*     , containers >=0.6 && <0.8     , crypton ==1.0.*-    , crypton-connection ==0.4.*     , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*     , crypton-x509-validation ==1.6.*     , dhall ==1.42.*     , directory ==1.3.*-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9     , filepath >=1.4 && <1.6-    , hex-text ==0.1.*     , hourglass ==0.2.*     , http-client ==0.7.*     , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*     , http-types ==0.12.*     , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3     , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*     , network >=3.1 && <3.3     , optparse-applicative ==0.18.*     , pem >=0.1 && <0.3     , polysemy ==1.9.*     , process ==1.6.*     , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21     , sqlite-simple ==0.4.*-    , stm ==2.5.*     , string-interpolate ==0.3.*-    , suspend ==0.2.*     , template-haskell >=2.19 && <2.24     , text >=2.0 && <2.2     , time >=1.12 && <1.15-    , timers ==0.2.*     , tls >=2.0 && <2.4     , tls-session-manager >=0.0.5 && <0.1-    , transformers ==0.6.*     , unix ==2.8.*     , unliftio ==0.2.*-    , unordered-containers ==0.2.*     , wai ==3.2.*     , wai-extra ==3.1.*     , warp ==3.4.*@@ -292,7 +245,6 @@   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints   build-depends:       aeson >=1 && <3-    , async ==2.2.*     , base >=4.7 && <5     , base64 >=0 && <2     , base64-bytestring ==1.*@@ -300,7 +252,6 @@     , bytestring >=0.11 && <0.13     , case-insensitive ==1.*     , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*     , co-log-polysemy >=0.0.1.5 && <0.0.2     , conduit ==1.*     , conduit-extra ==1.*@@ -308,27 +259,18 @@     , crypton >=0 && <2     , crypton-connection ==0.*     , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*     , data-default-class ==0.*     , dhall ==1.42.*     , digest ==0.*     , directory ==1.*-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9     , filepath >=1.4 && <1.6-    , hex-text ==0.1.*     , http-client ==0.*     , http-client-tls ==0.*     , http-conduit ==2.*-    , http-media ==0.8.*     , http-types ==0.*     , ini ==0.*     , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3     , memory ==0.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*     , network >=3.1 && <3.3     , network-uri ==2.*     , optparse-applicative ==0.18.*@@ -337,20 +279,12 @@     , relude ==1.*     , resourcet ==1.*     , retry ==0.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21     , sqlite-simple ==0.4.*-    , stm ==2.5.*     , string-interpolate ==0.3.*-    , suspend ==0.2.*     , template-haskell >=2.19 && <2.24     , text >=1 && <3     , time ==1.*     , time-units ==1.*-    , timers ==0.2.*     , tls >=2.0 && <2.4     , tls-session-manager >=0.0.5 && <0.1     , transformers ==0.*@@ -369,200 +303,6 @@     , relude    default-language: GHC2021 -library hs-hath-placeholder-  exposed-modules:-      Control.Placeholder-  other-modules:-      Paths_hs_hath-  autogen-modules:-      Paths_hs_hath-  hs-source-dirs:-      vendor/placeholder-0/src-  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints-  build-depends:-      aeson >=2.1 && <2.3-    , async ==2.2.*-    , base >=4.7 && <4.22-    , base64 ==1.0.*-    , bytestring >=0.11 && <0.13-    , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*-    , co-log-polysemy >=0.0.1.5 && <0.0.2-    , conduit ==1.3.*-    , containers >=0.6 && <0.8-    , crypton ==1.0.*-    , crypton-connection ==0.4.*-    , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*-    , dhall ==1.42.*-    , directory ==1.3.*-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9-    , filepath >=1.4 && <1.6-    , ghc-prim >=0 && <10-    , hex-text ==0.1.*-    , http-client ==0.7.*-    , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*-    , http-types ==0.12.*-    , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3-    , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*-    , network >=3.1 && <3.3-    , optparse-applicative ==0.18.*-    , polysemy ==1.9.*-    , process ==1.6.*-    , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21-    , sqlite-simple ==0.4.*-    , stm ==2.5.*-    , string-interpolate ==0.3.*-    , suspend ==0.2.*-    , template-haskell >=2.19 && <2.24-    , text >=2.0 && <2.2-    , time >=1.12 && <1.15-    , timers ==0.2.*-    , tls >=2.0 && <2.4-    , tls-session-manager >=0.0.5 && <0.1-    , transformers ==0.6.*-    , unix ==2.8.*-    , unliftio ==0.2.*-    , unordered-containers ==0.2.*-    , wai ==3.2.*-    , wai-extra ==3.1.*-    , warp ==3.4.*-  default-language: GHC2021--library hs-hath-polysemy-zoo-  exposed-modules:-      Polysemy.Capture-      Polysemy.ConstraintAbsorber-      Polysemy.ConstraintAbsorber.MonadCatch-      Polysemy.ConstraintAbsorber.MonadCont-      Polysemy.ConstraintAbsorber.MonadError-      Polysemy.ConstraintAbsorber.MonadReader-      Polysemy.ConstraintAbsorber.MonadState-      Polysemy.ConstraintAbsorber.MonadWriter-      Polysemy.Cont-      Polysemy.Cont.Internal-      Polysemy.EndState-      Polysemy.Final.IO-      Polysemy.Final.IO.Internal-      Polysemy.Final.More-      Polysemy.Final.MTL-      Polysemy.Final.NonDet-      Polysemy.Floodgate-      Polysemy.Fresh-      Polysemy.Input.Streaming-      Polysemy.KVStore-      Polysemy.Operators-      Polysemy.Random-      Polysemy.Reader.Compact-      Polysemy.Reader.More-      Polysemy.RevState-      Polysemy.SetStore-      Polysemy.Several-      Polysemy.Shift-      Polysemy.Shift.Internal-  other-modules:-      Paths_hs_hath-  autogen-modules:-      Paths_hs_hath-  hs-source-dirs:-      vendor/polysemy-zoo-0.8.2.0/src-  default-extensions:-      DataKinds-      DeriveFunctor-      FlexibleContexts-      GADTs-      LambdaCase-      PolyKinds-      RankNTypes-      ScopedTypeVariables-      StandaloneDeriving-      TypeApplications-      TypeOperators-      TypeFamilies-      UnicodeSyntax-  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -Wall-  build-depends:-      aeson >=2.1 && <2.3-    , async ==2.2.*-    , base >=4.18 && <4.22-    , base64 ==1.0.*-    , bytestring >=0.11 && <0.13-    , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*-    , co-log-polysemy >=0.0.1.5 && <0.0.2-    , conduit ==1.3.*-    , constraints >=0.14 && <0.16-    , containers ==0.7.*-    , contravariant ==1.5.*-    , crypton ==1.0.*-    , crypton-connection ==0.4.*-    , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*-    , dhall ==1.42.*-    , directory ==1.3.*-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9-    , filepath >=1.4 && <1.6-    , ghc-compact ==0.1.*-    , ghc-prim >=0.10 && <0.14-    , hex-text ==0.1.*-    , http-client ==0.7.*-    , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*-    , http-types ==0.12.*-    , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3-    , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*-    , network >=3.1 && <3.3-    , optparse-applicative >=0.18.1 && <0.19-    , polysemy ==1.9.*-    , process ==1.6.*-    , random ==1.3.*-    , reflection ==2.1.*-    , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21-    , sqlite-simple ==0.4.*-    , stm ==2.5.*-    , streaming ==0.2.*-    , string-interpolate ==0.3.*-    , suspend ==0.2.*-    , template-haskell >=2.19 && <2.24-    , text ==2.1.*-    , time >=1.12 && <1.15-    , timers ==0.2.*-    , tls >=2.0 && <2.4-    , tls-session-manager >=0.0.5 && <0.1-    , transformers ==0.6.*-    , unix ==2.8.*-    , unliftio ==0.2.*-    , unordered-containers ==0.2.*-    , wai ==3.2.*-    , wai-extra ==3.1.*-    , warp ==3.4.*-  default-language: GHC2021- library hs-hath-warp-tls   exposed-modules:       Network.Wai.Handler.WarpTLS@@ -572,73 +312,48 @@   autogen-modules:       Paths_hs_hath   hs-source-dirs:-      vendor/warp-tls-3.4.13+      vendor/warp-tls-3.4.18   default-extensions:       Strict       StrictData   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints   build-depends:-      aeson >=2.1 && <2.3-    , async ==2.2.*-    , base >=4.7 && <4.22-    , base64 ==1.0.*-    , bytestring ==0.*+      base >=4.7 && <4.22+    , bytestring >=0.9 && <1     , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*     , co-log-polysemy >=0.0.1.5 && <0.0.2     , conduit ==1.3.*     , containers >=0.6 && <0.8     , crypton ==1.0.*-    , crypton-connection ==0.4.*     , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*     , dhall ==1.42.*     , directory ==1.3.*-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9     , filepath >=1.4 && <1.6-    , hex-text ==0.1.*     , http-client ==0.7.*     , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*     , http-types ==0.12.*     , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3     , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*-    , network ==3.*+    , network >=3.2.12 && <4     , optparse-applicative ==0.18.*     , polysemy ==1.9.*     , process ==1.6.*-    , recv ==0.*+    , recv >=0.1.0 && <0.2     , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21     , sqlite-simple ==0.4.*-    , stm ==2.5.*+    , stm >=2.3 && <3     , streaming-commons ==0.*     , string-interpolate ==0.3.*-    , suspend ==0.2.*     , template-haskell >=2.19 && <2.24     , text >=1 && <3     , time >=1.12 && <1.15-    , timers ==0.2.*-    , tls >=2.1.3 && <2.5-    , tls-session-manager ==0.*-    , transformers ==0.6.*+    , tls >=2.1.3 && <2.6+    , tls-session-manager >=0.0.4 && <1     , unix ==2.8.*     , unliftio ==0.2.*-    , unordered-containers ==0.2.*     , wai ==3.2.*     , wai-extra ==3.1.*-    , warp >=3.3.29 && <3.4.13+    , warp >=3.3.29 && <3.6   default-language: GHC2021  executable hath-bench@@ -647,73 +362,44 @@       app   default-extensions:       NoImplicitPrelude-      TypeApplications       OverloadedStrings-      ScopedTypeVariables       QuasiQuotes       LambdaCase-      ViewPatterns-      MultiParamTypeClasses       GADTs   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N -main-is Bench   build-depends:-      aeson >=2.1 && <2.3-    , async ==2.2.*-    , base >=4.7 && <4.22-    , base64 ==1.0.*+      base >=4.7 && <4.22     , bytestring >=0.11 && <0.13     , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*     , co-log-polysemy >=0.0.1.5 && <0.0.2     , conduit ==1.3.*     , containers >=0.6 && <0.8     , crypton ==1.0.*     , crypton-connection ==0.4.*     , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*     , dhall ==1.42.*     , directory ==1.3.*-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9     , filepath >=1.4 && <1.6-    , hex-text ==0.1.*     , hs-hath     , http-client ==0.7.*     , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*     , http-types ==0.12.*     , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3     , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*     , network >=3.1 && <3.3     , optparse-applicative ==0.18.*     , polysemy ==1.9.*     , process ==1.6.*     , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21     , sqlite-simple ==0.4.*-    , stm ==2.5.*     , string-interpolate ==0.3.*-    , suspend ==0.2.*     , template-haskell >=2.19 && <2.24     , text >=2.0 && <2.2     , time >=1.12 && <1.15-    , timers ==0.2.*     , tls >=2.0 && <2.4     , tls-session-manager >=0.0.5 && <0.1-    , transformers ==0.6.*     , unix ==2.8.*     , unliftio ==0.2.*-    , unordered-containers ==0.2.*     , wai ==3.2.*     , wai-extra ==3.1.*     , warp ==3.4.*@@ -725,73 +411,43 @@       app   default-extensions:       NoImplicitPrelude-      TypeApplications       OverloadedStrings-      ScopedTypeVariables       QuasiQuotes       LambdaCase-      ViewPatterns-      MultiParamTypeClasses       GADTs   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N -fpedantic-bottoms   build-depends:-      aeson >=2.1 && <2.3-    , async ==2.2.*-    , base >=4.7 && <4.22-    , base64 ==1.0.*+      base >=4.7 && <4.22     , bytestring >=0.11 && <0.13     , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*     , co-log-polysemy >=0.0.1.5 && <0.0.2     , conduit ==1.3.*     , containers >=0.6 && <0.8     , crypton ==1.0.*-    , crypton-connection ==0.4.*     , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*     , dhall ==1.42.*     , directory ==1.3.*-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9     , filepath >=1.4 && <1.6-    , hex-text ==0.1.*     , hs-hath     , http-client ==0.7.*     , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*     , http-types ==0.12.*     , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3     , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*     , network >=3.1 && <3.3     , optparse-applicative ==0.18.*     , polysemy ==1.9.*     , process ==1.6.*     , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21     , sqlite-simple ==0.4.*-    , stm ==2.5.*     , string-interpolate ==0.3.*-    , suspend ==0.2.*     , template-haskell >=2.19 && <2.24     , text >=2.0 && <2.2     , time >=1.12 && <1.15-    , timers ==0.2.*     , tls >=2.0 && <2.4     , tls-session-manager >=0.0.5 && <0.1-    , transformers ==0.6.*     , unix ==2.8.*     , unliftio ==0.2.*-    , unordered-containers ==0.2.*     , wai ==3.2.*     , wai-extra ==3.1.*     , warp ==3.4.*@@ -803,75 +459,44 @@       app   default-extensions:       NoImplicitPrelude-      TypeApplications       OverloadedStrings-      ScopedTypeVariables       QuasiQuotes       LambdaCase-      ViewPatterns-      MultiParamTypeClasses       GADTs   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N -main-is Migrate   build-depends:-      aeson >=2.1 && <2.3-    , async ==2.2.*-    , base >=4.7 && <4.22-    , base64 ==1.0.*+      base >=4.7 && <4.22     , bytestring >=0.11 && <0.13     , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*     , co-log-polysemy >=0.0.1.5 && <0.0.2     , conduit ==1.3.*     , containers >=0.6 && <0.8     , crypton ==1.0.*-    , crypton-connection ==0.4.*     , crypton-x509 ==1.7.*-    , crypton-x509-store ==1.6.*     , dhall ==1.42.*-    , directory-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9+    , directory ==1.3.*     , filepath >=1.4 && <1.6-    , hex-text ==0.1.*     , hs-hath     , hs-hath-minio-    , hs-hath-polysemy-zoo     , http-client ==0.7.*     , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*     , http-types ==0.12.*     , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3     , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*     , network >=3.1 && <3.3-    , optparse-applicative+    , optparse-applicative ==0.18.*     , polysemy ==1.9.*     , process ==1.6.*     , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21     , sqlite-simple ==0.4.*-    , stm ==2.5.*     , string-interpolate ==0.3.*-    , suspend ==0.2.*     , template-haskell >=2.19 && <2.24     , text >=2.0 && <2.2     , time >=1.12 && <1.15-    , timers ==0.2.*     , tls >=2.0 && <2.4     , tls-session-manager >=0.0.5 && <0.1-    , transformers ==0.6.*     , unix ==2.8.*     , unliftio ==0.2.*-    , unordered-containers ==0.2.*     , wai ==3.2.*     , wai-extra ==3.1.*     , warp ==3.4.*@@ -881,12 +506,19 @@   type: exitcode-stdio-1.0   main-is: Spec.hs   other-modules:+      APISpec+      ConnectionLimitSpec       FilesystemSpec-      FileURIRenderSpec+      FileURISpec+      FloodControlSpec+      GallerySpec       Integration-      InterfaceAPIOptimizationSpec+      LocateSpec       MockRPC+      ProtocolSpec       R2Spec+      RPCSpec+      SQLiteSpec       Paths_hs_hath   autogen-modules:       Paths_hs_hath@@ -894,76 +526,47 @@       test   default-extensions:       NoImplicitPrelude-      TypeApplications       OverloadedStrings-      ScopedTypeVariables       QuasiQuotes       LambdaCase-      ViewPatterns-      MultiParamTypeClasses       GADTs-  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints+  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N   build-depends:-      aeson >=2.1 && <2.3-    , async ==2.2.*+      QuickCheck >=2.14 && <2.17     , base >=4.7 && <4.22-    , base64 ==1.0.*     , bytestring >=0.11 && <0.13     , co-log >=0.6 && <0.8-    , co-log-core ==0.3.*     , co-log-polysemy >=0.0.1.5 && <0.0.2     , conduit ==1.3.*     , containers >=0.6 && <0.8     , crypton ==1.0.*-    , crypton-connection ==0.4.*     , crypton-x509 ==1.7.*     , crypton-x509-store ==1.6.*     , dhall ==1.42.*-    , directory-    , exceptions ==0.10.*-    , extra >=1.7 && <1.9-    , filepath-    , hex-text ==0.1.*+    , directory ==1.3.*+    , filepath >=1.4 && <1.6     , hs-hath-    , hs-hath-polysemy-zoo     , hspec ==2.11.*     , http-client ==0.7.*     , http-client-tls ==0.3.*-    , http-conduit ==2.3.*-    , http-media ==0.8.*     , http-types ==0.12.*     , lrucache ==1.2.*-    , mason >=0.2.6 && <0.3     , memory ==0.18.*-    , monad-control ==1.0.*-    , monad-loops ==0.4.*-    , mtl ==2.3.*     , network >=3.1 && <3.3     , optparse-applicative ==0.18.*     , polysemy ==1.9.*     , process ==1.6.*     , relude ==1.2.*-    , resourcet ==1.3.*-    , safe-exceptions ==0.1.*-    , servant >=0.20.2 && <0.21-    , servant-client >=0.20.2 && <0.21-    , servant-client-core >=0.20.2 && <0.21-    , servant-server >=0.20.2 && <0.21     , sqlite-simple ==0.4.*-    , stm ==2.5.*     , string-interpolate ==0.3.*-    , suspend ==0.2.*     , template-haskell >=2.19 && <2.24-    , temporary+    , temporary ==1.3.*     , text >=2.0 && <2.2     , time >=1.12 && <1.15-    , timers ==0.2.*     , tls >=2.0 && <2.4     , tls-session-manager >=0.0.5 && <0.1-    , transformers ==0.6.*     , unix ==2.8.*     , unliftio ==0.2.*-    , unordered-containers ==0.2.*     , wai ==3.2.*     , wai-extra ==3.1.*     , warp ==3.4.*
− src/CLI.hs
@@ -1,69 +0,0 @@-{-# LANGUAGE RecordWildCards #-}--module CLI ( Options(..), parseOptions, applyOptionsToConfig ) where--import           Options.Applicative--import           Relude--import           Types               ( ClientConfig(..) )---- | Command line options-data Options-  = Options-  { optConfigPath        :: !FilePath      -- ^ Path to configuration file-  , optDownloadDir       :: !(Maybe Text)  -- ^ Override download directory-  , optCachePath         :: !(Maybe Text)  -- ^ Override cache path-  , optSkipStartupVerify :: !Bool         -- ^ Skip startup cache verification-  , optDisableRateLimit  :: !Bool          -- ^ Disable rate limiting (for NAT/proxy scenarios)-  , optTrustProxyHeaders :: !Bool         -- ^ Trust X-Forwarded-For header for real IP-  , optValidateConfig    :: !Bool          -- ^ Validate configuration and exit-  , optShowVersion       :: !Bool          -- ^ Show version information-  }-  deriving ( Show )---- | Parse command line options-parseOptions :: IO Options-parseOptions = execParser opts-  where-    opts-      = info-        (optionsParser <**> helper)-        (fullDesc-         <> progDesc "Haskell implementation of the Hath client"-         <> header "hs-hath - A Hentai@Home client in Haskell")---- | Options parser-optionsParser :: Parser Options-optionsParser-  = Options-  <$> strOption-    (long "config"-     <> short 'c'-     <> metavar "FILE"-     <> value "./client-login"-     <> help "Configuration file path (default: ./client-login)")-  <*> optional-    (strOption-       (long "download-dir"-        <> short 'd'-        <> metavar "DIR"-        <> help "Override download directory from config"))-  <*> optional-    (strOption (long "cache-path" <> metavar "PATH" <> help "Override cache path from config"))-  <*> switch (long "skip-startup-verify" <> help "Skip cache integrity verification at startup")-  <*> switch-    (long "disable-rate-limit"-     <> help "Disable rate limiting (use when behind NAT/proxy that doesn't forward real IP)")-  <*> switch-    (long "trust-proxy-headers"-     <> help "Trust X-Forwarded-For header for real client IP (use when behind trusted proxy)")-  <*> switch (long "validate-config" <> help "Validate configuration and exit")-  <*> switch (long "version" <> short 'v' <> help "Show version information and exit")---- | Apply CLI options to client configuration-applyOptionsToConfig :: Options -> ClientConfig -> ClientConfig-applyOptionsToConfig Options { .. } config-  = config { downloadDir = fromMaybe (downloadDir config) optDownloadDir-           , cachePath   = fromMaybe (cachePath config) optCachePath-           }
− src/FileVerification.hs
@@ -1,210 +0,0 @@-{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE TemplateHaskell #-}-{-# LANGUAGE TypeApplications #-}--module FileVerification-  ( -- * Types-    VerificationResult(..)-  , VerificationStats(..)-    -- * Verification Operations-  , verifyFile-  , verifyAllFiles-  , verifyRandomFile-  , shouldVerifyFile-    -- * Effect-  , FileVerification(..)-  , runFileVerification-  , runFileVerificationIO-    -- * Constants-  , verificationCooldownSeconds-  , minVerificationIntervalSeconds-  ) where--import           Data.Time.Clock        ( NominalDiffTime, UTCTime, diffUTCTime, getCurrentTime )--import           Database.SQLite.Simple--import           Polysemy-import           Polysemy.Operators--import           Relude--import           Storage.Database       ( FileRecord(..) )--import           Types                  ( FileURI(..), parseFileURI )--import           Utils                  ( hash )---- | Result of verifying a single file-data VerificationResult-  = VerificationOK-  | VerificationCorrupted {-# UNPACK #-} !Text  -- ^ File ID of corrupted file-  | VerificationMissing {-# UNPACK #-} !Text    -- ^ File ID that should exist but doesn't-  deriving ( Show, Eq, Generic )---- | Statistics from a verification run-data VerificationStats-  = VerificationStats { verifiedCount    :: {-# UNPACK #-} !Int-                      , corruptedCount   :: {-# UNPACK #-} !Int-                      , missingCount     :: {-# UNPACK #-} !Int-                      , deletedCount     :: {-# UNPACK #-} !Int-                      , verificationTime :: {-# UNPACK #-} !NominalDiffTime-                      }-  deriving ( Show, Eq, Generic )--instance Semigroup VerificationStats where-  a <> b-    = VerificationStats-    { verifiedCount    = verifiedCount a + verifiedCount b-    , corruptedCount   = corruptedCount a + corruptedCount b-    , missingCount     = missingCount a + missingCount b-    , deletedCount     = deletedCount a + deletedCount b-    , verificationTime = verificationTime a + verificationTime b-    }--instance Monoid VerificationStats where-  mempty = VerificationStats 0 0 0 0 0---- | Cooldown between verifying the same file (1 week in seconds)-verificationCooldownSeconds :: Int64-verificationCooldownSeconds = 7 * 24 * 60 * 60---- | Minimum interval between any verification checks (2 seconds)-minVerificationIntervalSeconds :: NominalDiffTime-minVerificationIntervalSeconds = 2---- | Check if a file should be verified based on last verification time-shouldVerifyFile :: UTCTime -> UTCTime -> Bool-shouldVerifyFile lastVerified now-  = diffUTCTime now lastVerified > fromIntegral verificationCooldownSeconds---- | Verify a single file's integrity by comparing stored hash with computed hash-verifyFile :: FileRecord -> VerificationResult-verifyFile record-  = let-      storedFileId = fileRecordFileId record-      content      = fileRecordBytes record-      uri          = parseFileURI (encodeUtf8 storedFileId)-      -- Extract expected hash from FileURI (already ByteString)-      -- FileURI format: hash-size-xres-yres-ext-      expectedHash = fileHash uri-      computedHash = hash @ByteString content-    in -      if expectedHash == computedHash-        then VerificationOK-        else VerificationCorrupted storedFileId---- | Effect for file verification operations-data FileVerification m a where-  -- | Verify all files in the cache (for startup)-  VerifyAllFilesE :: FileVerification m VerificationStats-  -- | Verify a single random file (for periodic checks)-  VerifyRandomFileE :: FileVerification m (Maybe VerificationResult)-  -- | Get the last verification time-  GetLastVerificationTime :: FileVerification m (Maybe UTCTime)-  -- | Update the last verification time-  SetLastVerificationTime :: UTCTime -> FileVerification m ()--makeSem ''FileVerification---- | Run file verification with SQLite connection-runFileVerification :: Member (Embed IO) r-                    => Connection-                    -> TVar (Maybe UTCTime)  -- ^ Last verification time-                    -> FileVerification : r @> a-                    -> r @> a-runFileVerification conn lastVerifVar = interpret $ \case-  VerifyAllFilesE           -> embed @IO $ verifyAllFiles conn--  VerifyRandomFileE         -> embed @IO $ verifyRandomFile conn lastVerifVar--  GetLastVerificationTime   -> embed @IO $ readTVarIO lastVerifVar--  SetLastVerificationTime t -> embed @IO $ atomically $ writeTVar lastVerifVar (Just t)---- | Verify all files in the database using streaming to avoid loading all into memory-verifyAllFiles :: Connection -> IO VerificationStats-verifyAllFiles conn = do-  startTime <- getCurrentTime--  -- Use IORefs to accumulate results while streaming-  statsRef <- newIORef $! VerificationStats 0 0 0 0 0-  toDeleteRef <- newIORef ([] :: [ Text ])--  -- Stream files one at a time using fold_ to avoid loading all bytes into memory-  fold_-    conn-    "SELECT lru_counter, s4, file_id, file_name, bytes FROM files"-    ()-    (\() record -> do-       case verifyFile record of-         VerificationOK -> modifyIORef' statsRef $ \s -> let-             !v = verifiedCount s + 1-           in -             s { verifiedCount = v }-         VerificationCorrupted fileId -> do-           modifyIORef' statsRef $ \s -> let-               !c = corruptedCount s + 1-             in -               s { corruptedCount = c }-           modifyIORef' toDeleteRef (fileId :)-         VerificationMissing fileId -> do-           modifyIORef' statsRef $ \s -> let-               !m = missingCount s + 1-             in -               s { missingCount = m }-           modifyIORef' toDeleteRef (fileId :))--  -- Delete corrupted files-  toDelete <- readIORef toDeleteRef-  forM_ toDelete $ \fileId -> execute conn "DELETE FROM files WHERE file_id = ?" (Only fileId)--  endTime <- getCurrentTime-  let !elapsed = diffUTCTime endTime startTime--  stats <- readIORef statsRef-  pure $! stats { deletedCount = length toDelete, verificationTime = elapsed }---- | Verify a random file from the cache--- Returns Nothing if verification is on cooldown or no files exist-verifyRandomFile :: Connection -> TVar (Maybe UTCTime) -> IO (Maybe VerificationResult)-verifyRandomFile conn lastVerifVar = do-  now <- getCurrentTime-  lastVerif <- readTVarIO lastVerifVar--  -- Check cooldown-  case lastVerif of-    Just t-      | diffUTCTime now t < minVerificationIntervalSeconds -> pure Nothing  -- On cooldown-    _      -> do-      -- Get a random file that hasn't been verified recently-      -- We use RANDOM() for simplicity; production might want better randomization-      maybeRecord <- listToMaybe-        <$> query_-          conn-          "SELECT lru_counter, s4, file_id, file_name, bytes FROM files ORDER BY RANDOM() LIMIT 1"--      case maybeRecord of-        Nothing     -> pure Nothing  -- No files in cache-        Just record -> do-          -- Update last verification time-          atomically $ writeTVar lastVerifVar (Just now)--          -- Force evaluation of the result to ensure file bytes are processed-          -- before the record goes out of scope-          let !result = verifyFile record--          -- If corrupted, delete the file-          case result of-            VerificationCorrupted fileId -> do-              execute conn "DELETE FROM files WHERE file_id = ?" (Only fileId)-              pure $! Just result-            _ -> pure $! Just result---- | Run file verification in IO-runFileVerificationIO-  :: Connection -> TVar (Maybe UTCTime) -> [ FileVerification, Embed IO, Final IO ] @> a -> IO a-runFileVerificationIO conn lastVerifVar-  = runFinal . embedToFinal . runFileVerification conn lastVerifVar
+ src/Hath/AtomicFile.hs view
@@ -0,0 +1,22 @@+module Hath.AtomicFile ( writeFileAtomic ) where++import           Control.Exception ( bracketOnError, try )++import qualified Data.ByteString   as BS++import           Relude++import           System.Directory  ( createDirectoryIfMissing, removeFile, renameFile )+import           System.FilePath   ( takeDirectory, takeFileName )+import           System.IO         ( hClose, openBinaryTempFile )++-- | Write through a temp file in the same directory, then rename: a crash or a+-- concurrent reader never sees a partial file. Missing directories are created.+writeFileAtomic :: FilePath -> ByteString -> IO ()+writeFileAtomic path bytes = do+  let dir = takeDirectory path+  createDirectoryIfMissing True dir+  bracketOnError+    (openBinaryTempFile dir (takeFileName path <> ".tmp"))+    (\( tmp, h ) -> hClose h >> void (try @SomeException (removeFile tmp)))+    $ \( tmp, h ) -> BS.hPut h bytes >> hClose h >> renameFile tmp path
+ src/Hath/CLI.hs view
@@ -0,0 +1,68 @@+{-# LANGUAGE RecordWildCards #-}++module Hath.CLI ( Options(..), parseOptions, applyOptionsToConfig ) where++import           Hath.Config         ( ClientConfig(..) )+import           Hath.Version        ( versionInfo )++import           Options.Applicative++import           Relude++data Options+  = Options { optConfigPath        :: !FilePath+            , optDownloadDir       :: !(Maybe Text)+            , optCachePath         :: !(Maybe Text)+            , optSkipStartupVerify :: !Bool+            , optDisableRateLimit  :: !Bool+            , optTrustProxyHeaders :: !Bool+            , optValidateConfig    :: !Bool+            }+  deriving ( Show )++parseOptions :: IO Options+parseOptions = execParser opts+  where+    opts+      = info+        (optionsParser <**> versionOption <**> helper)+        (fullDesc+         <> progDesc "Haskell implementation of the Hath client"+         <> header "hs-hath - A Hentai@Home client in Haskell")++    versionOption+      = infoOption+        (toString versionInfo)+        (long "version" <> short 'v' <> help "Show version information and exit")++optionsParser :: Parser Options+optionsParser+  = Options+  <$> strOption+    (long "config"+     <> short 'c'+     <> metavar "FILE"+     <> value "./client-login"+     <> help "Configuration file path (default: ./client-login)")+  <*> optional+    (strOption+       (long "download-dir"+        <> short 'd'+        <> metavar "DIR"+        <> help "Override download directory from config"))+  <*> optional+    (strOption (long "cache-path" <> metavar "PATH" <> help "Override cache path from config"))+  <*> switch (long "skip-startup-verify" <> help "Skip cache integrity verification at startup")+  <*> switch+    (long "disable-rate-limit"+     <> help "Disable rate limiting (use when behind NAT/proxy that doesn't forward real IP)")+  <*> switch+    (long "trust-proxy-headers"+     <> help "Trust X-Forwarded-For header for real client IP (use when behind trusted proxy)")+  <*> switch (long "validate-config" <> help "Validate configuration and exit")++applyOptionsToConfig :: Options -> ClientConfig -> ClientConfig+applyOptionsToConfig Options { .. } config+  = config { downloadDir = fromMaybe (downloadDir config) optDownloadDir+           , cachePath   = fromMaybe (cachePath config) optCachePath+           }
+ src/Hath/Cache.hs view
@@ -0,0 +1,42 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE TemplateHaskell #-}++-- | The file cache as the rest of the client sees it. The backends+-- ("Hath.Cache.Filesystem", "Hath.Cache.R2", "Hath.Cache.SQLite") interpret it.+module Hath.Cache+  ( Cache(..)+  , CacheHit(..)+  , lookupFile+  , storeFile+  , deleteFile+  , CacheRunner(..)+  ) where++import           Colog          ( Message )+import           Colog.Polysemy ( Log )++import           Hath.Error     ( HathError )+import           Hath.FileURI   ( FileURI )++import           Polysemy+import           Polysemy.Error ( Error )++import           Relude++data CacheHit = InMemory !ByteString | Redirect !ByteString | LocalFile !FilePath+  deriving ( Eq, Show )++data Cache m a where+  LookupFile :: FileURI -> Cache m (Maybe CacheHit)+  -- | Keep a file whose content has already been verified against its id.+  StoreFile :: FileURI -> ByteString -> Cache m ()+  -- | Delete a file the network has blacklisted. A file we do not have is no error.+  DeleteFile :: FileURI -> Cache m ()++makeSem ''Cache++newtype CacheRunner+  = CacheRunner { runCacheWith :: forall r a. Members '[ Embed IO, Log Message, Error HathError ] r+                               => Sem (Cache : r) a+                               -> Sem r a+                }
+ src/Hath/Cache/Filesystem.hs view
@@ -0,0 +1,156 @@+{-# LANGUAGE DataKinds #-}++-- | The Filesystem backend: files on disk as @root\/xxxx\/fileid@. Hits are served+-- straight from the file ('LocalFile'); the file's mtime is the LRU clock, and+-- 'runEvictor' keeps the cache under the disk limit the server assigns.+module Hath.Cache.Filesystem ( FsCache, newFsCache, runCacheFilesystem, runEvictor, sweep ) where++import           Colog                   ( Message )+import           Colog.Polysemy          ( Log )++import           Control.Concurrent      ( threadDelay )++import qualified Data.ByteString.Char8   as BS+import           Data.String.Interpolate ( i )++import           Foreign.C.Types         ( CTime(..) )++import           Hath.AtomicFile         ( writeFileAtomic )+import           Hath.Cache              ( Cache(..), CacheHit(..) )+import           Hath.Error              ( HathError(..) )+import           Hath.FileURI            ( FileURI(..), renderFileURIBytes )+import           Hath.Log                ( logError )++import           Polysemy+import           Polysemy.Error          ( Error, throw )++import           Relude++import           System.Directory        ( createDirectoryIfMissing, listDirectory, removeFile )+import           System.FilePath         ( (</>), takeFileName )+import qualified System.Posix.Files      as Posix++import           UnliftIO.Exception      ( tryAny )++data FsCache+  = FsCache { fsRoot :: !FilePath+            , fsUsed :: !(IORef Int64)+              -- ^ bytes on disk as of the last sweep, plus writes and deletes since+            }++newFsCache :: FilePath -> IO FsCache+newFsCache root = do+  createDirectoryIfMissing True root+  FsCache root <$> newIORef 0++fileURIToPath :: FilePath -> FileURI -> FilePath+fileURIToPath root uri = root </> take 4 fileId </> fileId+  where+    -- no directory traversal through a crafted file id. A file id is ASCII+    -- ('Hath.FileURI.parseFileURI' admits nothing else), so Char8 reads it exactly.+    fileId = takeFileName (BS.unpack (renderFileURIBytes uri))++addUsed :: FsCache -> Int64 -> IO ()+addUsed fs n = atomicModifyIORef' (fsUsed fs) $ \used -> ( used + n, () )++runCacheFilesystem :: Members '[ Embed IO, Log Message, Error HathError ] r+                   => FsCache+                   -> Sem (Cache : r) a+                   -> Sem r a+runCacheFilesystem fs = interpret $ \case+  LookupFile uri      -> embed $ do+    let path = fileURIToPath (fsRoot fs) uri+    tryAny (Posix.getSymbolicLinkStatus path) >>= \case+      Right st+        | Posix.isRegularFile st && fromIntegral (Posix.fileSize st) == fileSize uri -> do+          -- a hit makes the file the newest, so eviction takes it last+          void $ tryAny $ Posix.touchFile path+          pure $ Just $ LocalFile path+        | Posix.isRegularFile st -> do+          -- truncated (written in place by an older version): drop it so it gets refetched+          void $ removeCached fs path st+          pure Nothing+      _        -> pure Nothing++  StoreFile uri bytes -> do+    let path = fileURIToPath (fsRoot fs) uri+    embed (tryAny (writeFileAtomic path bytes)) >>= \case+      Left err -> do+        logError [i|Filesystem write failed for #{path}: #{err}|]+        throw $ StorageError (show err)+      Right () -> embed $ addUsed fs (fromIntegral $ BS.length bytes)++  DeleteFile uri      -> do+    let path = fileURIToPath (fsRoot fs) uri+    embed (tryAny (Posix.getSymbolicLinkStatus path)) >>= \case+      Left _   -> pass+      Right st -> embed (removeCached fs path st) >>= \case+        Left err -> throw $ StorageError [i|Could not delete #{path}: #{err}|]+        Right () -> pass++removeCached :: FsCache -> FilePath -> Posix.FileStatus -> IO (Either SomeException ())+removeCached fs path st = do+  removed <- tryAny (removeFile path)+  whenRight_ removed $ \() -> addUsed fs (negate $ fromIntegral $ Posix.fileSize st)+  pure removed++-- | Evict this far below the limit, so eviction does not run on every write:+-- 100 MiB like the official client, less for tiny limits.+wantFree :: Int64 -> Int64+wantFree limit = min (100 * 1024 * 1024) (limit `div` 10)++-- | Keep the cache under the disk limit (0 = no limit yet), forever. The first+-- sweep learns how much is already on disk; after that we only rescan when over.+runEvictor :: FsCache -> IO Int64 -> IO Void+runEvictor fs getLimit+  = attempt >> infinitely (do+                             threadDelay (10 * 1000000)+                             limit <- getLimit+                             used <- readIORef (fsUsed fs)+                             when (limit > 0 && used > limit) attempt)+  where+    attempt = tryAny (getLimit >>= sweep fs) >>= \case+      Left err           -> putStrLn [i|Cache eviction failed: #{err}|]+      Right ( 0, _ )     -> pass+      Right ( n, bytes ) -> putStrLn+        [i|Evicted #{n} least recently served files (#{bytes `div` 1048576} MiB) to stay under the disk limit|]++-- | Rescan the cache; if it is over the limit, delete the least recently served+-- files until it is 'wantFree' under. Returns the files and bytes evicted.+-- ponytail: a full scan per sweep, fine for ~10^5 files; keep an index if caches grow far past that+sweep :: FsCache -> Int64 -> IO ( Int, Int64 )+sweep fs limit = do+  before <- readIORef (fsUsed fs)+  files <- scanCache (fsRoot fs)+  let total   = sum [ size | ( _, size, _ ) <- files ]+      victims+        | limit > 0 && total > limit+          = takeBytes (total - limit + wantFree limit) (sortOn fst3 files)+        | otherwise = []+  freed <- fmap catMaybes $ forM victims $ \( _, size, rel ) -> rightToMaybe . (size <$)+    <$> tryAny (removeFile (fsRoot fs </> decodeUtf8 (fromShort rel)))+  -- keep what was written meanwhile+  atomicModifyIORef' (fsUsed fs) $ \used -> ( used - before + total - sum freed, () )+  pure ( length freed, sum freed )+  where+    fst3 ( t, _, _ ) = t++    takeBytes n (x@( _, size, _ ) : xs)+      | n > 0 = x : takeBytes (n - size) xs+    takeBytes _ _ = []++-- | (mtime, size, path relative to root) of our files: root/xxxx/xxxx….+scanCache :: FilePath -> IO [ ( Int64, Int64, ShortByteString ) ]+scanCache root = do+  dirs <- filter ((== 4) . length) <$> listDirectory root+  fmap concat $ forM dirs $ \dir -> do+    names <- fromRight [] <$> tryAny (listDirectory (root </> dir))+    fmap catMaybes $ forM (filter (dir `isPrefixOf`) names) $ \name -> do+      let rel = dir </> name+      tryAny (Posix.getSymbolicLinkStatus (root </> rel)) <&> \case+        Right st+          | Posix.isRegularFile st -> let+              CTime mtime = Posix.modificationTime st+            in +              Just ( mtime, fromIntegral (Posix.fileSize st), toShort (encodeUtf8 rel) )+        _        -> Nothing
+ src/Hath/Cache/R2.hs view
@@ -0,0 +1,158 @@+{-# LANGUAGE DataKinds #-}++-- | The R2 backend (any S3-compatible store): hits are answered with a redirect+-- to a presigned link, so the reader fetches the file from the bucket itself.+module Hath.Cache.R2+  ( R2Connection(..)+  , mkR2Connection+  , runCacheR2+  , runR2+  , fileURIToKey+  , putR2Object+  ) where++import           Colog                   ( Message )+import           Colog.Polysemy          ( Log )++import qualified Conduit                 as C++import           Control.Exception       ( try )++import qualified Data.ByteArray          as BA+import qualified Data.ByteString         as BS+import           Data.Cache.LRU.IO       ( AtomicLRU, newAtomicLRU )+import qualified Data.Cache.LRU.IO       as LRU+import qualified Data.Text               as T++import           GHC.Clock               ( getMonotonicTime )++import           Hath.Cache              ( Cache(..), CacheHit(..) )+import           Hath.Config             ( R2Config(..) )+import           Hath.Error              ( HathError(..) )+import           Hath.FileURI            ( FileURI, renderFileURIText )+import           Hath.Log                ( logError, logWarning )++import           Network.HTTP.Client     ( newManager )+import           Network.HTTP.Client.TLS ( tlsManagerSettings )+import           Network.Minio           ( AccessKey(..)+                                         , CredentialValue(..)+                                         , SecretKey(..)+                                         , defaultPutObjectOptions+                                         , presignedGetObjectUrl+                                         , setCreds+                                         , setRegion+                                         )+import qualified Network.Minio           as Minio+import           Network.Minio.S3API     ( headObject )++import           Polysemy+import           Polysemy.Error          ( Error, throw )++import           Relude++data R2Connection+  = R2Connection { r2MinioConn  :: !Minio.MinioConn+                 , r2ConnBucket :: !Text+                 , r2UrlCache   :: !(AtomicLRU FileURI ( Double, ByteString ))+                   -- ^ Presigned links and when we stop handing them out (monotonic seconds).+                 }++-- | Connect with the endpoint and bucket from the config and the credentials+-- from @R2_ACCESS_KEY@ and @R2_SECRET_KEY@.+mkR2Connection :: R2Config -> IO (Either Text R2Connection)+mkR2Connection cfg = do+  accessKeyMay <- lookupEnv "R2_ACCESS_KEY"+  secretKeyMay <- lookupEnv "R2_SECRET_KEY"+  case ( accessKeyMay, secretKeyMay ) of+    ( Nothing, _ ) -> pure $ Left "R2_ACCESS_KEY environment variable not set"+    ( _, Nothing ) -> pure $ Left "R2_SECRET_KEY environment variable not set"+    ( Just accessKey, Just secretKey ) -> do+      let creds+            = CredentialValue+            { cvAccessKey    = AccessKey $ toText accessKey+            , cvSecretKey    = SecretKey $ BA.convert (encodeUtf8 @String @ByteString secretKey)+            , cvSessionToken = Nothing+            }+          -- the IsString instance parses the endpoint URL+          baseConnInfo = fromString $ toString $ r2Endpoint cfg+      manager <- newManager tlsManagerSettings+      minioConn <- Minio.mkMinioConn (setCreds creds $ setRegion "auto" baseConnInfo) manager+      -- ponytail: fixed capacity; links expire after 10 min, so it only has to hold ~10 min of distinct hits+      urlCache <- newAtomicLRU (Just 100000)+      pure+        $ Right+          R2Connection+          { r2MinioConn = minioConn, r2ConnBucket = r2Bucket cfg, r2UrlCache = urlCache }++-- | R2 object key for a file: @abcd\/\<file id>@, abcd being the first 4 hex digits+-- of the hash. Every client sharing the bucket must use this layout (the Java+-- fork's S3CacheBackend.objectKey does).+fileURIToKey :: FileURI -> Text+fileURIToKey uri = T.take 4 fileId <> "/" <> fileId+  where+    fileId = renderFileURIText uri++runR2 :: R2Connection -> Minio.Minio a -> IO (Either Text a)+runR2 conn action = try @SomeException (Minio.runMinioWith (r2MinioConn conn) action) <&> \case+  Left err         -> Left (show err)+  Right (Left err) -> Left (show err)+  Right (Right a)  -> Right a++putR2Object :: R2Connection -> FileURI -> ByteString -> IO (Either Text ())+putR2Object conn uri bytes+  = runR2 conn+  $ Minio.putObject+    (r2ConnBucket conn)+    (fileURIToKey uri)+    (C.yield bytes)+    (Just (fromIntegral (BS.length bytes)))+    defaultPutObjectOptions++-- | A lookup that fails for any reason is a miss (with a warning); a failed+-- write is an error.+runCacheR2 :: Members '[ Embed IO, Log Message, Error HathError ] r+           => R2Connection+           -> Sem (Cache : r) a+           -> Sem r a+runCacheR2 conn = interpret $ \case+  LookupFile uri      -> do+    let objectKey = fileURIToKey uri+    now <- embed getMonotonicTime+    embed (LRU.lookup uri (r2UrlCache conn)) >>= \case+      Just ( expiry, link )+        | now < expiry -> pure $ Just $ Redirect link+      _ -> do+        -- ask the bucket whether it has the object before sending anyone there+        found <- embed $ minio $ headObject bucket objectKey []+        case found of+          Right _ -> fmap Redirect <$> embed (rememberLink uri now)+          Left _  -> do+            logWarning ("Object not found in R2 storage: " <> objectKey)+            pure Nothing++  StoreFile uri bytes -> embed (putR2Object conn uri bytes) >>= \case+    Left err -> do+      logError ("R2 write failed for " <> fileURIToKey uri <> ": " <> err)+      throw $ StorageError err+    -- it is there now, as after a HEAD: the next hit (often seconds away) needs none+    Right () -> embed $ getMonotonicTime >>= void . rememberLink uri++  DeleteFile uri      -> embed (minio $ Minio.removeObject bucket (fileURIToKey uri)) >>= \case+    Left err -> throw $ StorageError ("R2 delete failed for " <> fileURIToKey uri <> ": " <> err)+    -- forget its link, or hits would be sent to the deleted object for up to 10 min+    Right () -> embed $ void $ LRU.delete uri (r2UrlCache conn)+  where+    bucket = r2ConnBucket conn++    minio :: Minio.Minio a -> IO (Either Text a)+    minio = runR2 conn++    -- Sign a link (local, no request) and reuse it for 10 min.+    rememberLink :: FileURI -> Double -> IO (Maybe ByteString)+    rememberLink uri now+      -- valid 20 min, handed out for 10, so every link given lasts at least 10 min+       = minio (presignedGetObjectUrl bucket (fileURIToKey uri) 1200 [] []) >>= \case+        Left _     -> pure Nothing+        Right link -> do+          LRU.insert uri ( now + 600, link ) (r2UrlCache conn)+          pure (Just link)
+ src/Hath/Cache/SQLite.hs view
@@ -0,0 +1,83 @@+{-# LANGUAGE DataKinds #-}++-- | The SQLite backend: files are blobs in one database file. Nothing is ever+-- evicted; kept for existing setups.+module Hath.Cache.SQLite+  ( initializeDB+  , runCacheSQLite+  , VerificationStats(..)+  , verifyAllFiles+  ) where++import qualified Data.Text              as T+import           Data.Time.Clock        ( NominalDiffTime, diffUTCTime, getCurrentTime )++import           Database.SQLite.Simple++import           Hath.Cache             ( Cache(..), CacheHit(..) )+import           Hath.FileURI           ( FileURI(..), parseFileURI, renderFileURIText )+import           Hath.Protocol          ( sha1Hex )++import           Polysemy++import           Relude++initializeDB :: Connection -> IO ()+initializeDB conn = do+  execute_+    conn+    "CREATE TABLE IF NOT EXISTS files (\+    \  lru_counter INTEGER NOT NULL DEFAULT 0,\+    \  s4 TEXT NOT NULL,\+    \  file_id TEXT PRIMARY KEY,\+    \  file_name TEXT,\+    \  bytes BLOB NOT NULL\+    \) strict"+  execute_ conn "pragma journal_mode=delete"+  execute_ conn "pragma synchronous=normal"+  execute_ conn "pragma temp_store=memory"++runCacheSQLite :: Member (Embed IO) r => Connection -> Sem (Cache : r) a -> Sem r a+runCacheSQLite conn = interpret $ \case+  LookupFile uri      -> embed $ do+    rows <- query conn "SELECT bytes FROM files WHERE file_id = ?" (Only (renderFileURIText uri))+    pure $ InMemory . fromOnly <$> listToMaybe rows++  StoreFile uri bytes -> embed $ do+    let fileId = renderFileURIText uri+    execute+      conn+      "INSERT OR REPLACE INTO files (lru_counter, s4, file_id, file_name, bytes) VALUES (?, ?, ?, ?, ?)"+      ( 1 :: Int64, T.take 4 fileId, fileId, Nothing :: Maybe Text, bytes )++  DeleteFile uri      -> embed+    $ execute conn "DELETE FROM files WHERE file_id = ?" (Only (renderFileURIText uri))++data VerificationStats+  = VerificationStats { verifiedCount    :: !Int+                      , corruptedCount   :: !Int+                        -- ^ these were deleted+                      , verificationTime :: !NominalDiffTime+                      }+  deriving ( Show, Eq )++intact :: Text -> ByteString -> Bool+intact fileId bytes = (fileHash <$> parseFileURI (encodeUtf8 fileId)) == Just (sha1Hex bytes)++-- | Check every file against its id and delete the ones that no longer match.+-- Rows are streamed, so the cache is never in memory as a whole.+verifyAllFiles :: Connection -> IO VerificationStats+verifyAllFiles conn = do+  start <- getCurrentTime+  ( verified, corrupted ) <- fold_ conn "SELECT file_id, bytes FROM files" ( 0, [] )+    $ \( !ok, bad ) ( fileId, bytes ) -> pure+    $ if intact fileId bytes+      then ( ok + 1, bad )+      else ( ok, fileId : bad )+  forM_ corrupted $ \fileId -> execute conn "DELETE FROM files WHERE file_id = ?" (Only fileId)+  end <- getCurrentTime+  pure+    VerificationStats { verifiedCount    = verified+                      , corruptedCount   = length corrupted+                      , verificationTime = diffUTCTime end start+                      }
+ src/Hath/Config.hs view
@@ -0,0 +1,80 @@+-- | The @client-login@ file: who we are and where the cache lives.+module Hath.Config+  ( ClientConfig(..)+  , ClientProxy(..)+  , CacheBackend(..)+  , R2Config(..)+  , defaultClientConfig+  , readClientConfig+  ) where++import           Dhall  ( FromDhall, ToDhall, auto, input )++import           Relude++-- | The field names are the keys of the Dhall record, and the record must match+-- exactly, so removing or renaming a field breaks every existing config file.+-- That is why the fields nothing reads are still here.+data ClientConfig+  = ClientConfig+  { clientId     :: !Text+  , key          :: !Text+    -- ^ The client key: it signs our requests and proves the server's replies.+  , version      :: !Text+    -- ^ Not read; the build we announce is 'Hath.Protocol.clientBuild'.+  , proxy        :: !(Maybe ClientProxy)+    -- ^ Not read; no request goes through a proxy.+  , downloadDir  :: !Text+  , cachePath    :: !Text+    -- ^ A directory (Filesystem) or a database file (SQLite); R2 has no use for it.+  , cacheBackend :: !CacheBackend+  , r2Config     :: !(Maybe R2Config)+  , lruCacheSize :: !Int64+    -- ^ Not read.+  }+  deriving ( Show, Generic )++instance FromDhall ClientConfig++instance ToDhall ClientConfig++data ClientProxy = ClientProxy { host :: !Text, port :: !Integer, auth :: !(Maybe ( Text, Text )) }+  deriving ( Show, Generic )++instance FromDhall ClientProxy++instance ToDhall ClientProxy++data CacheBackend = CacheBackendSQLite | CacheBackendR2 | CacheBackendFilesystem+  deriving ( Show, Eq, Generic )++instance FromDhall CacheBackend++instance ToDhall CacheBackend++-- | Where the bucket is. Its credentials come from the environment+-- (@R2_ACCESS_KEY@, @R2_SECRET_KEY@), not from the config file.+data R2Config = R2Config { r2Endpoint :: !Text, r2Bucket :: !Text }+  deriving ( Show, Generic )++instance FromDhall R2Config++instance ToDhall R2Config++defaultClientConfig :: ClientConfig+defaultClientConfig+  = ClientConfig+  { clientId     = ""+  , key          = ""+  , version      = ""+  , proxy        = Nothing+  , downloadDir  = ""+  , cachePath    = ""+  , cacheBackend = CacheBackendSQLite+  , r2Config     = Nothing+  , lruCacheSize = 100 * 1024 * 1024+  }++-- | Evaluate a Dhall expression, usually a path such as @./client-login@.+readClientConfig :: Text -> IO ClientConfig+readClientConfig = input auto
+ src/Hath/Download.hs view
@@ -0,0 +1,85 @@+-- | Downloads from other servers: reading a reply's body with a limit on the+-- wait for every read, and whole downloads as the official FileDownloader makes+-- them (proxy tests, gallery files).+module Hath.Download ( fetchFile, receive ) where++import qualified Data.ByteString.Char8   as BS+import qualified Data.List               as List+import           Data.String.Interpolate ( i )++import           Hath.Protocol           ( outgoingHeaders )+import           Hath.RPC                ( connectMicros, httpManager, worthRetrying )++import qualified Network.HTTP.Client     as HC+import           Network.HTTP.Types      ( hContentLength, statusCode )++import           Relude++import           UnliftIO                ( timeout, tryAny )++-- | Download as the official FileDownloader does (FileDownloader.java:147-299):+-- up to three attempts, one right after another, each with 5 s to connect and+-- 10 s for the reply's headers and for every read. An attempt succeeds if its+-- reply has a status below 400 and announces its length, at most+-- @max_allowed_filesize@, and that many bytes arrive. @keep@ gets them chunk by+-- chunk with the offset of each, so a new attempt starts again at 0. A 404 or+-- 410 is not tried again. Right is the length, Left why the last attempt failed.+fetchFile :: Int -> HC.Request -> (Int -> ByteString -> IO ()) -> IO (Either Text Int)+fetchFile maxAllowedFileSize request keep = go (3 :: Int)+  where+    go attemptsLeft = attempt >>= \case+      Left ( status, why )+        | attemptsLeft > 1, worthRetrying status -> go (attemptsLeft - 1)+        | otherwise -> pure (Left why)+      Right size           -> pure (Right size)++    -- the length, or else the status that failed it (none if something else did) and why+    attempt+      = either (\e -> Left ( Nothing, describe e )) id+      <$> tryAny (HC.withResponse sent httpManager answer)++    answer response = case ( statusCode (HC.responseStatus response), announced ) of+      ( status, _ )+        | status >= 400 -> pure $ Left ( Just status, [i|HTTP #{status}|] )+      ( _, Nothing )   -> pure $ Left ( Nothing, "no Content-Length" )+      ( _, Just size )+        | size > maxAllowedFileSize+          -> pure $ Left ( Nothing, [i|#{size} bytes, more than max_allowed_filesize|] )+        | otherwise -> receive readMicros size (HC.responseBody response) keep <&> \case+          Left why -> Left ( Nothing, why )+          Right () -> Right size+      where+        announced+          = readMaybe . BS.unpack =<< List.lookup hContentLength (HC.responseHeaders response)++    sent+      = request { HC.requestHeaders  = outgoingHeaders [ ( "Connection", "Close" ) ]+                , HC.responseTimeout = HC.responseTimeoutMicro (connectMicros + readMicros)+                }++    readMicros = 10 * 1000000++-- | Read a body of @size@ bytes, handing each chunk to @keep@ with the offset it+-- starts at, and waiting up to @readMicros@ for every read. Left says why it+-- broke off: upstream failed or sent too little or too much, or a read waited+-- too long.+receive :: Int -> Int -> HC.BodyReader -> (Int -> ByteString -> IO ()) -> IO (Either Text ())+receive readMicros size body keep = either (Left . describe) id <$> tryAny (go 0)+  where+    go received = timeout readMicros (HC.brRead body) >>= \case+      Nothing+        -> pure $ Left [i|nothing came for #{readMicros `div` 1000000} s after #{received} bytes|]+      Just chunk+        | BS.null chunk -> pure+          $ if received == size+            then Right ()+            else Left [i|it ended after #{received} of #{size} bytes|]+        -- the caller's buffer may hold no more; a chunked body can bring more than announced+        | BS.length chunk > size - received -> pure $ Left [i|it sent more than #{size} bytes|]+        | otherwise -> keep received chunk >> go (received + BS.length chunk)++-- | Why a download failed, in http-client's words where it has them.+describe :: SomeException -> Text+describe e = case fromException e of+  Just (HC.HttpExceptionRequest _ content) -> show content+  _ -> show e
+ src/Hath/Error.hs view
@@ -0,0 +1,16 @@+module Hath.Error ( HathError(..) ) where++import           Relude++data HathError+  = TransportError !Text+    -- ^ The HTTP exchange with the RPC server failed, every time it was tried.+  | EmptyResponse+  | RequestFailure !Text+    -- ^ The RPC server answered with this status instead of @OK@.+  | CertificateFailure !Text+    -- ^ The PKCS#12 file from @get_cert@ could not be read.+  | StorageError !Text+  deriving ( Show, Eq )++instance Exception HathError
+ src/Hath/FileURI.hs view
@@ -0,0 +1,86 @@+-- | File ids as the H\@H network writes them:+-- @\<sha1>-\<size>-\<xres>-\<yres>-\<type>@, or @\<sha1>-\<size>-\<type>@ for a+-- file without a resolution.+module Hath.FileURI+  ( FileURI(..)+  , parseFileURI+  , renderFileURIBytes+  , renderFileURIText+  , mimeType+  ) where++import qualified Data.ByteString.Char8 as BS+import           Data.Char             ( isDigit )+import qualified Data.List             as List++import           Prelude               ( Show(show) )++import           Relude                hiding ( show )+import qualified Relude++data FileURI+  = FileURI { fileHash :: !ByteString+              -- ^ SHA-1 of the content, in hex+            , fileSize :: !Int+            , fileXRes :: !Int+              -- ^ 0, as is 'fileYRes', for an id without a resolution+            , fileYRes :: !Int+            , fileExt  :: !ByteString+            }+  deriving ( Eq, Ord )++-- | Shows the file id itself, which is what logs want.+instance Show FileURI where+  show = toString . renderFileURIText++-- | A file id the official client accepts (HVFile.isValidHVFileid and+-- getHVFileFromFileid): a lower-case SHA-1, a size of up to 10 digits that fits+-- a Java int, a resolution of up to 5 digits a side or none, and a known type.+parseFileURI :: ByteString -> Maybe FileURI+parseFileURI bytes = case BS.split '-' bytes of+  [ hash, size, ext ] -> fileURI hash size (Just 0) (Just 0) ext+  [ hash, size, xres, yres, ext ] -> fileURI hash size (number 5 xres) (number 5 yres) ext+  _ -> Nothing+  where+    fileURI hash size xres yres ext = do+      guard (BS.length hash == 40 && BS.all isHex hash && isJust (List.lookup ext fileTypes))+      FileURI hash <$> mfilter (<= 2147483647) (number 10 size) <*> xres <*> yres <*> pure ext++    isHex c = isDigit c || (c >= 'a' && c <= 'f')++    number digits s = do+      guard (not (BS.null s) && BS.length s <= digits && BS.all isDigit s)+      fst <$> BS.readInt s++-- | The id of a file, as the official client writes it (HVFile.getFileid).+renderFileURIBytes :: FileURI -> ByteString+renderFileURIBytes uri+  | fileXRes uri > 0+    = BS.intercalate+      "-"+      [ fileHash uri+      , Relude.show (fileSize uri)+      , Relude.show (fileXRes uri)+      , Relude.show (fileYRes uri)+      , fileExt uri+      ]+  | otherwise = BS.intercalate "-" [ fileHash uri, Relude.show (fileSize uri), fileExt uri ]++renderFileURIText :: FileURI -> Text+renderFileURIText = decodeUtf8 . renderFileURIBytes++mimeType :: FileURI -> ByteString+mimeType uri = fromMaybe "application/octet-stream" (List.lookup (fileExt uri) fileTypes)++-- | The types a file id may have, and their Content-Type (HVFile.getMimeType).+fileTypes :: [ ( ByteString, ByteString ) ]+fileTypes+  = [ ( "jpg", "image/jpeg" )+    , ( "png", "image/png" )+    , ( "gif", "image/gif" )+    , ( "mp4", "video/mp4" )+    , ( "wbm", "video/webm" )+    , ( "wbp", "image/webp" )+    , ( "avf", "image/avif" )+    , ( "jxl", "image/jxl" )+    ]
+ src/Hath/Gallery.hs view
@@ -0,0 +1,166 @@+-- | What the gallery downloader is told to fetch, and the directory it goes into.+module Hath.Gallery+  ( GalleryMetadata(..)+  , GalleryFile(..)+  , parseMetadata+  , galleryDirectoryName+  , fallbackDirectoryName+  ) where++import qualified Data.ByteString.Char8 as BS+import           Data.Char             ( isAsciiLower, isAsciiUpper, isDigit )+import           Data.List             ( dropWhileEnd )+import qualified Data.Text             as T++import           Hath.Protocol         ( sameSecret, sha1Hex )++import           Relude++data GalleryMetadata+  = GalleryMetadata+  { galleryID          :: !Int+  , galleryMinXRes     :: !ByteString+  , galleryTitle       :: !Text+    -- ^ cleaned up as the official client cleans it for a directory name+  , galleryInformation :: !ByteString+    -- ^ the INFORMATION section, which goes into @galleryinfo.txt@+  , galleryFileList    :: ![ GalleryFile ]+  }+  deriving ( Eq )++data GalleryFile+  = GalleryFile { galleryFilePage  :: !Int+                , galleryFileIndex :: !Int+                , galleryFileName  :: !ByteString+                , galleryFileXRes  :: !ByteString+                , galleryFileHash  :: !(Maybe ByteString)+                  -- ^ Nothing: generated on demand, so there is nothing to verify against+                , galleryFileExt   :: !ByteString+                }+  deriving ( Eq )++-- | Parse a @\/16\/dl@ fetchqueue reply (hathdl format). Left carries the reason it was+-- refused: @NO_PENDING_DOWNLOADS@ and other non-lists, or a list failing the 1.6.6 checks.+-- The file list must carry @DLHASH@ = sha1(client key, then @"\\n" ++ line@ for every+-- file line), and names and types must be ones the gallery system can produce, which+-- keeps a forged list from writing outside the download directory.+parseMetadata :: ByteString -> ByteString -> Either Text GalleryMetadata+parseMetadata clientKey bytes = case break (== "FILELIST") (BS.split '\n' bytes) of+  ( _, [] ) -> Left $ decodeUtf8 $ BS.strip $ BS.take 64 bytes+  ( header, _ : rest ) -> do+    let ( fileSection, informationSection ) = break (== "INFORMATION") rest+        fileLines = filter (not . BS.null) fileSection+        listHash = sha1Hex (clientKey <> foldMap ("\n" <>) fileLines)+        field name+          = listToMaybe+            [ BS.drop 1 value+            | line <- header+            , let ( f, value ) = BS.break (== ' ') line+            , f == name+            ]+    gid <- maybeToRight "invalid GID" $ mfilter (> 0) $ readIntExact =<< field "GID"+    minXRes <- maybeToRight "invalid MINXRES" $ mfilter validXRes $ field "MINXRES"+    title <- maybeToRight "no TITLE" $ field "TITLE"+    unless (any (sameSecret listHash) (field "DLHASH"))+      $ Left "DLHASH does not match the file list"+    files <- traverse parseFile fileLines+    pure+      GalleryMetadata+      { galleryID          = gid+      , galleryMinXRes     = minXRes+      , galleryTitle       = cleanTitle (decodeUtf8 title)+        -- Java's split drops the empty lines at the end+      , galleryInformation = foldMap (<> "\n") $ dropWhileEnd BS.null $ drop 1 informationSection+      , galleryFileList    = files+      }+  where+    parseFile line = case BS.split ' ' line of+      [ page, index, xres, sha1, ext, name ]+        | Just p <- readIntExact page, Just ix <- readIntExact index, validName name+        , ext `elem` [ "jpg", "gif", "png", "webp" ] -> Right+          GalleryFile { galleryFilePage  = p+                      , galleryFileIndex = ix+                      , galleryFileName  = name+                      , galleryFileXRes  = xres+                      , galleryFileHash  = if sha1 == "unknown"+                          then Nothing+                          else Just sha1+                      , galleryFileExt   = ext+                      }+      _ -> Left $ "invalid file line: " <> decodeUtf8 line++    validName n+      = not (BS.null n)+      && BS.length n <= 250+      && not (".." `BS.isInfixOf` n)+      && BS.all+        (\c -> isAsciiUpper c || isAsciiLower c || isDigit c || c `elem` ("._-" :: String))+        n++    validXRes x = x == "org" || (not (BS.null x) && BS.all isDigit x)++    readIntExact s = case BS.readInt s of+      Just ( n, "" ) -> Just n+      _ -> Nothing++-- | The title as the official client cleans it (GalleryDownloader.java:256). Its+-- pattern takes out @*@, @"@, @\\@, @<@, @>@ and the sequence @:|?@ (the @|@ there+-- is escaped, so a lone @:@, @|@ or @?@ stays), then makes every run of ASCII+-- whitespace one space and trims the ends. Taking out the sequence first gives+-- what Java's single pass gives: none of its characters is taken out alone.+cleanTitle :: Text -> Text+cleanTitle+  = T.intercalate " "+  . filter (not . T.null)+  . T.split (`elem` (" \t\n\v\f\r" :: String))+  . T.filter (`notElem` ("*\"\\<>" :: String))+  . T.replace ":|?" ""++-- | The name of the gallery's directory in the download directory+-- (GalleryDownloader.java:259-285): the title, then @[gid]@, or @[gid-<minxres>x]@+-- unless the original size was asked for, so that each size gets a directory of+-- its own. A title that would make the name longer than @max_filename_length@+-- UTF-16 code units, as Java counts its length, is cut short and ends in+-- "...". Nothing if the name would leave the download directory: the official+-- client refuses such a list.+galleryDirectoryName :: Int -> GalleryMetadata -> Maybe Text+galleryDirectoryName maxLength gallery+  | "/" `T.isInfixOf` name = Nothing+  | otherwise = Just name+  where+    title   = galleryTitle gallery++    postfix = " [" <> show (galleryID gallery) <> xresSuffix gallery <> "]"++    name+      | utf16Length title + T.length postfix > maxLength+        = takeUtf16 (maxLength - T.length postfix - 3) title <> "..." <> postfix+      | otherwise = title <> postfix++-- | The name to use when the file system will not take 'galleryDirectoryName'+-- (GalleryDownloader.java:294-298): the same without the title.+fallbackDirectoryName :: GalleryMetadata -> Text+fallbackDirectoryName gallery = show (galleryID gallery) <> xresSuffix gallery++xresSuffix :: GalleryMetadata -> Text+xresSuffix gallery+  | galleryMinXRes gallery == "org" = ""+  | otherwise = "-" <> decodeUtf8 (galleryMinXRes gallery) <> "x"++utf16Length :: Text -> Int+utf16Length = sum . map utf16Units . toString++-- | The characters that start within the first @n@ UTF-16 code units. Like+-- Java's @offsetByCodePoints(0, codePointCount(0, n))@, this keeps whole a+-- character whose surrogate pair the cut would split.+takeUtf16 :: Int -> Text -> Text+takeUtf16 n = toText . go 0 . toString+  where+    go used (c : rest)+      | used < n = c : go (used + utf16Units c) rest+    go _ _ = []++utf16Units :: Char -> Int+utf16Units c+  | ord c > 0xFFFF = 2+  | otherwise = 1
+ src/Hath/Gallery/Downloader.hs view
@@ -0,0 +1,235 @@+{-# LANGUAGE DataKinds #-}++-- | The gallery downloader: works through the galleries queued for this client+-- on the H\@H site, into the download directory, as the official+-- GalleryDownloader does.+module Hath.Gallery.Downloader ( runGalleryDownloader, downloadQueue, anotherRound ) where++import           Colog                   ( Message )+import           Colog.Polysemy          ( Log )++import           Control.Concurrent      ( threadDelay )++import qualified Data.ByteString.Char8   as BS+import           Data.String.Interpolate ( i )++import           Hath.AtomicFile         ( writeFileAtomic )+import           Hath.Config             ( ClientConfig(..) )+import           Hath.Download           ( fetchFile )+import           Hath.Error              ( HathError )+import           Hath.Gallery            ( GalleryFile(..)+                                         , GalleryMetadata(..)+                                         , fallbackDirectoryName+                                         , galleryDirectoryName+                                         )+import           Hath.Log                ( logInfo, logWarning )+import           Hath.Protocol           ( sha1Hex )+import           Hath.RPC                ( RPC+                                         , galleryFileSource+                                         , nextGallery+                                         , reportDownloadFailures+                                         , runRPCIO+                                         )+import           Hath.Settings           ( HathSettings(..) )+import           Hath.Stats              ( StatsEnv, addDlBytes, incDlFile, incDlTask )++import qualified Network.HTTP.Client     as HC++import           Polysemy+import           Polysemy.Error          ( Error, catch )+import           Polysemy.Reader         ( Reader, ask, runReader )++import           Relude                  hiding ( Reader, ask, runReader )++import           System.Directory        ( createDirectoryIfMissing, removeFile )+import           System.FilePath         ( (<.>), (</>) )+import           System.IO               ( hPutStrLn )++import           UnliftIO                ( tryAny )++-- | Work through the download queue each time the server asks+-- (@start_downloader@ fills @requested@). A request that comes while a run is+-- under way is dropped, as the official client ignores it+-- (HentaiAtHomeClient.startDownloader): the run goes on until the queue is+-- empty anyway.+runGalleryDownloader :: ClientConfig -> StatsEnv -> TVar HathSettings -> MVar () -> IO Void+runGalleryDownloader config stats settings requested = infinitely $ do+  takeMVar requested+  tryAny (downloadQueue config stats settings) >>= \case+    Left e           -> failed (show e)+    Right (Left e)   -> failed (show e)+    Right (Right ()) -> pass+  void $ tryTakeMVar requested+  where+    failed e = hPutStrLn stderr $ "Gallery downloader error: " <> e++-- | Download the queued galleries one after another (GalleryDownloader.java:58-126).+-- Each fetchqueue after the first marks the gallery before it done, whether or+-- not all of its files came, and brings the next one. The run ends when the+-- queue is empty, when the server refuses the request, or when we refuse its list.+downloadQueue :: ClientConfig -> StatsEnv -> TVar HathSettings -> IO (Either HathError ())+downloadQueue config stats settings = runRPCIO config $ runReader stats $ go Nothing+  where+    go done = nextGallery done >>= \case+      Left "NO_PENDING_DOWNLOADS" -> logInfo "No more galleries to download"+      Left reason -> logWarning [i|The gallery downloader stops: #{reason}|]+      Right gallery -> do+        incDlTask+        current <- embed @IO (readTVarIO settings)+        galleryDirectory (maxFilenameLength current) gallery >>= \case+          Nothing        -> logWarning+            [i|Unexpected download location for gallery #{galleryID gallery}; the gallery downloader stops|]+          Just directory -> do+            downloadGallery (maxAllowedFileSize current) directory gallery+            go (Just gallery)++-- | Make the gallery's directory and give its path. If the file system will+-- not take the name, the official client suspects it cannot store Unicode and+-- falls back on one without the title (GalleryDownloader.java:287-298).+-- Nothing if the name would leave the download directory.+galleryDirectory :: Members '[ Reader ClientConfig, Log Message, Embed IO ] r+                 => Int+                 -> GalleryMetadata+                 -> Sem r (Maybe FilePath)+galleryDirectory maxLength gallery = do+  downloads <- toString . downloadDir <$> ask+  forM (galleryDirectoryName maxLength gallery) $ \name -> do+    let path = downloads </> toString name+    embed (tryAny (createDirectoryIfMissing True path)) >>= \case+      Right () -> pure path+      Left e   -> do+        logWarning+          [i|Could not create the gallery directory "#{name}" (#{e}); using one without the title|]+        let fallback = downloads </> toString (fallbackDirectoryName gallery)+        embed $ createDirectoryIfMissing True fallback+        pure fallback++-- | Download a gallery's files, write its @galleryinfo.txt@ if all of them came+-- (GalleryDownloader.java:132-150), and report the sources that failed.+downloadGallery :: Members '[ RPC, Reader StatsEnv, Log Message, Error HathError, Embed IO ] r+                => Int+                -> FilePath+                -> GalleryMetadata+                -> Sem r ()+downloadGallery maxSize directory gallery = do+  logInfo [i|Starting download of gallery: #{galleryTitle gallery}|]+  ( complete, failures ) <- downloadFiles maxSize directory gallery+  if complete+    then do+      logInfo [i|Finished download of gallery: #{galleryTitle gallery}|]+      whenLeftM_+        (embed+         $ tryAny+         $ writeFileAtomic (directory </> "galleryinfo.txt") (galleryInformation gallery))+        $ \e -> logWarning [i|Could not write galleryinfo.txt: #{e}|]+    else logWarning [i|Permanently failed downloading gallery: #{galleryTitle gallery}|]+  -- the official client goes on whether or not the report gets through+  reportDownloadFailures failures `catch` \(e :: HathError)+    -> logWarning [i|Could not report download failures: #{e}|]++-- | Fetch the gallery's files into @directory@, going through those still+-- missing again while 'anotherRound' allows (GalleryDownloader.java:71-118).+-- True if all of them are there in the end; and the sources that failed, as+-- @host-fileindex-xres@.+downloadFiles :: Members '[ RPC, Reader StatsEnv, Log Message, Error HathError, Embed IO ] r+              => Int+              -> FilePath+              -> GalleryMetadata+              -> Sem r ( Bool, [ Text ] )+downloadFiles maxSize directory gallery = go 0 0 [] [ ( file, 1 ) | file <- files ]+  where+    files = galleryFileList gallery++    -- the files still missing, each with the number its next attempt gets+    go rounds failed failures missing+      | null missing = pure ( True, failures )+      | not (anotherRound (length files) rounds failed) = pure ( False, failures )+      | otherwise = do+        outcomes <- mapM (uncurry (ensureFile maxSize directory gallery)) missing+        let stillMissing+              = [ ( file, attempt + 1 ) | ( ( file, attempt ), Failed _ ) <- zip missing outcomes ]+        go+          (rounds + 1)+          (failed + length stillMissing)+          (failures <> [ source | Failed (Just source) <- outcomes ])+          stillMissing++-- | Whether to go through a gallery's missing files once more, after this many+-- rounds and failed downloads (GalleryDownloader.java:74).+anotherRound :: Int -> Int -> Int -> Bool+anotherRound fileCount rounds failed = rounds < 9 && failed < fileCount * 2++data Outcome+  = Present+    -- ^ it was there already, intact+  | Downloaded+  | Failed !(Maybe Text)+-- ^ with the source that failed, if the server named one++-- | Get one file into the directory unless it is there already, intact+-- (GalleryDownloader.java:392-474): ask the server where to download it from,+-- this being attempt @attempt@ at it, and check what comes. The official+-- client then waits 1 s after a download, 5 s after a failure.+ensureFile :: Members '[ RPC, Reader StatsEnv, Log Message, Error HathError, Embed IO ] r+           => Int+           -> FilePath+           -> GalleryMetadata+           -> GalleryFile+           -> Int+           -> Sem r Outcome+ensureFile maxSize directory gallery file attempt = embed (intactAt path file) >>= \case+  True  -> pure Present+  False -> do+    outcome <- download+    embed $ threadDelay $ case outcome of+      Downloaded -> 1000000+      _          -> 5000000+    pure outcome+  where+    name = BS.unpack (galleryFileName file) <.> BS.unpack (galleryFileExt file)++    path = directory </> name++    download = do+      source <- galleryFileSource gallery file attempt `catch` \(e :: HathError)+        -> Nothing <$ logWarning [i|dlfetch failed for #{name}: #{e}|]+      case HC.parseRequest . BS.unpack =<< source of+        Nothing      -> do+          logWarning [i|No source to download #{name} from|]+          pure (Failed Nothing)+        Just request -> embed (fetchBytes maxSize request) >>= \case+          Right bytes+            | hasExpectedHash file bytes -> do+              embed $ writeFileAtomic path bytes+              incDlFile+              addDlBytes (BS.length bytes)+              logInfo [i|Downloaded #{name} of gallery #{galleryID gallery}|]+              pure Downloaded+          Right _     -> failedFrom request "its SHA-1 is not the one listed"+          Left why    -> failedFrom request why++    failedFrom request (why :: Text) = do+      logWarning [i|Failed to download #{name} (attempt #{attempt}): #{why}|]+      pure $ Failed $ Just [i|#{HC.host request}-#{galleryFileIndex file}-#{galleryFileXRes file}|]++-- | Whether the file is there, not empty, and has the expected hash if one is+-- known. One that is not is removed, as by the official client.+intactAt :: FilePath -> GalleryFile -> IO Bool+intactAt path file = do+  content <- fromRight "" <$> tryAny (readFileBS path)+  let intact = not (BS.null content) && hasExpectedHash file content+  unless intact $ void $ tryAny $ removeFile path+  pure intact++-- | A file generated on demand has no known hash; like the official client, accept it.+hasExpectedHash :: GalleryFile -> ByteString -> Bool+hasExpectedHash file content = maybe True (== sha1Hex content) (galleryFileHash file)++fetchBytes :: Int -> HC.Request -> IO (Either Text ByteString)+fetchBytes maxSize request = do+  chunks <- newIORef []+  -- a chunk at offset 0 begins another attempt, so what came before is dropped+  let keep 0 chunk = writeIORef chunks [ chunk ]+      keep _ chunk = modifyIORef' chunks (chunk :)+  fetched <- fetchFile maxSize request keep+  forM fetched $ \_ -> BS.concat . reverse <$> readIORef chunks
+ src/Hath/Locate.hs view
@@ -0,0 +1,266 @@+{-# LANGUAGE DataKinds #-}++-- | Finding a requested file: in the cache, or else upstream.+module Hath.Locate ( Located(..), locateResource, usableReply, heldBytes ) where++import           Colog                    ( Message )+import           Colog.Polysemy           ( Log )++import           Control.Concurrent       ( forkIOWithUnmask )+import           Control.Exception        ( finally, mask_, onException )++import qualified Data.ByteString.Char8    as BS+import           Data.ByteString.Internal ( fromForeignPtr, fromForeignPtr0, mallocByteString )+import           Data.ByteString.Unsafe   ( unsafeUseAsCStringLen )+import qualified Data.List                as List+import           Data.String.Interpolate  ( i )++import           Foreign.ForeignPtr       ( ForeignPtr, withForeignPtr )+import           Foreign.Marshal.Utils    ( copyBytes )+import           Foreign.Ptr              ( castPtr, plusPtr )++import           Hath.Cache               ( Cache, CacheHit, lookupFile, storeFile )+import           Hath.Config              ( ClientConfig )+import           Hath.Download            ( receive )+import           Hath.Error               ( HathError )+import           Hath.FileURI             ( FileURI(..) )+import           Hath.Log                 ( logInfo, logWarning )+import           Hath.Protocol            ( hathRequestToken, outgoingHeaders, sha1Hex )+import           Hath.RPC                 ( RPC, connectMicros, fileSources, httpManager )+import           Hath.Stats               ( StatsEnv, addDownload, incFetched )++import qualified Network.HTTP.Client      as HC+import           Network.HTTP.Types       ( Header+                                          , Status+                                          , hContentLength+                                          , status404+                                          , status500+                                          , status502+                                          , status503+                                          , statusCode+                                          )++import           Polysemy+import           Polysemy.Error           ( Error, catch )+import           Polysemy.Final           ( withWeavingToFinal )+import           Polysemy.Reader          ( Reader, ask )++import           Relude                   hiding ( Reader, ask )++import           System.IO.Error          ( ioError, mkIOError, resourceVanishedErrorType )+import           System.IO.Unsafe         ( unsafePerformIO )++import           UnliftIO                 ( retrySTM, timeout, try )++data Located+  = Stored !CacheHit+    -- | A cache miss, coming in from upstream on a thread of its own, which+    -- caches the file once it is complete. Call it with a chunk writer to+    -- relay the file as it arrives; it throws if the download breaks off.+  | Streamed !((ByteString -> IO ()) -> IO ())++-- | Look in the cache; on a miss, ask the RPC server where the file is, by the+-- @fileindex@ and @xres@ of the request and the file id as the reader asked for+-- it, and start fetching it. Left is the status the reader gets when that+-- fails (see 'openUpstream'), or 503 while the downloads under way hold all+-- they may ('bufferBudget').+--+-- Like the official client, we serve any file the server sends readers here+-- for; static ranges only matter for pruning the cache.+locateResource+  :: Members+    '[ Cache+     , RPC+     , Reader StatsEnv+     , Log Message+     , Reader ClientConfig+     , Error HathError+     , Embed IO+     , Final IO+     ]+    r+  => Int+  -> ByteString+  -> FileURI+  -> ( ByteString, ByteString )+  -> Sem r (Either Status Located)+locateResource maxAllowedFileSize fileId uri ( fileIndex, xres ) = lookupFile uri >>= \case+  Just hit -> pure $ Right $ Stored hit+  Nothing  -> do+    logInfo [i|Fetching resource: #{uri}|]+    openUpstream maxAllowedFileSize fileId uri fileIndex xres >>= \case+      Left status    -> do+        logWarning [i|Could not fetch #{uri}: #{statusCode status}|]+        pure (Left status)+      Right response -> do+        -- runs on the download's thread, after the handler has returned+        finish <- deferToIO $ \case+          Left why      -> logWarning [i|Fetching #{uri} failed: #{why}|]+          Right content -> do+            addDownload (BS.length content)+            -- a failed cache write only costs us the next hit+            catch @HathError (storeFile uri content)+              $ \e -> logWarning [i|Cache write failed for #{uri}: #{e}|]+        embed (download uri response finish) >>= \case+          Just relay -> Right (Streamed relay) <$ incFetched+          Nothing    -> do+            logWarning+              [i|Not fetching #{uri}: the downloads under way hold #{bufferBudget `div` 1048576} MiB already|]+            pure (Left status503)++-- | Open the first source that answers with the file, as the official+-- ProxyFileDownloader does; the caller reads the body and must close the+-- response. Without sources the reader gets a 404. If no source will do, a 502+-- once one of them has answered, or a 500 if none could be reached: the+-- official client changes its status only for a source it connected to+-- (ProxyFileDownloader.java:71-111).+openUpstream :: Members '[ RPC, Reader ClientConfig, Log Message, Error HathError, Embed IO ] r+             => Int+             -> ByteString+             -> FileURI+             -> ByteString+             -> ByteString+             -> Sem r (Either Status (HC.Response HC.BodyReader))+openUpstream maxAllowedFileSize fileId uri fileIndex xres = do+  urls <- filter (not . BS.null) <$> fileSources fileIndex xres fileId `catch` \(e :: HathError)+    -> [] <$ logWarning [i|srfetch failed for #{uri}: #{e}|]+  cfg <- ask @ClientConfig+  let request url = do+        req <- HC.parseRequest (BS.unpack url)+        pure+          req { HC.requestHeaders+                  = outgoingHeaders [ ( "Hath-Request", hathRequestToken cfg fileId ) ]+                -- 5 s to connect, then 30 s for the headers (ProxyFileDownloader.java:86-87)+              , HC.responseTimeout = HC.responseTimeoutMicro (connectMicros + seconds 30)+              }+      open url = case request url of+        Nothing  -> pure Unreachable+        Just req -> try @IO @HC.HttpException (HC.responseOpen req httpManager) >>= \case+          Right response+            | usableReply+                maxAllowedFileSize+                uri+                (statusCode (HC.responseStatus response))+                (HC.responseHeaders response) -> pure (Opened response)+            | otherwise -> HC.responseClose response $> Answered+          Left (HC.HttpExceptionRequest _ HC.ConnectionTimeout) -> pure Unreachable+          Left (HC.HttpExceptionRequest _ (HC.ConnectionFailure _)) -> pure Unreachable+          Left _ -> pure Answered+      firstOpened failure = \case+        []         -> pure (Left failure)+        url : rest -> open url >>= \case+          Opened response -> pure (Right response)+          Answered        -> firstOpened status502 rest+          Unreachable     -> firstOpened failure rest+  if null urls+    then pure (Left status404)+    else embed (firstOpened status500 urls)++data Attempt+  = Opened !(HC.Response HC.BodyReader)+  | Answered+    -- ^ it answered, but not with the file+  | Unreachable++-- | Whether an upstream reply, given its status and headers, is the file+-- (ProxyFileDownloader.java:92-107): it must announce the file's length, and a+-- file larger than @max_allowed_filesize@ is not taken.+usableReply :: Int -> FileURI -> Int -> [ Header ] -> Bool+usableReply maxAllowedFileSize uri status headers+  = status == 200 && announced == Just (fileSize uri) && fileSize uri <= maxAllowedFileSize+  where+    announced = readMaybe . BS.unpack =<< List.lookup hContentLength headers++-- | Fetch a body on a thread of its own, into a buffer of the file's size. The+-- reader relays from the buffer, and the buffer, once verified, is what the+-- cache keeps, so the file is in memory once. A reader that goes away leaves+-- the download running, as the official ProxyFileDownloader's thread does.+-- @finish@ gets the verified file, or why there is none. Nothing, with the+-- response closed, if the buffer would not fit the 'bufferBudget'.+download :: FileURI+         -> HC.Response HC.BodyReader+         -> (Either Text ByteString -> IO ())+         -> IO (Maybe ((ByteString -> IO ()) -> IO ()))+download uri response finish = mask_ $ reserve size >>= \case+  False -> Nothing <$ HC.responseClose response+  -- what is reserved must be released whatever happens, or the budget shrinks for good+  True  -> flip onException (release size >> HC.responseClose response) $ do+    buffer <- mallocByteString size+    progress <- newTVarIO (Receiving 0)+    let keep offset chunk = do+          copyInto buffer offset chunk+          atomically $ writeTVar progress (Receiving (offset + BS.length chunk))+    void $ forkIOWithUnmask $ \unmask -> flip finally (release size) $ unmask $ do+      -- 30 s for every read and 300 s for the whole (ProxyFileDownloader.java:186-200)+      received <- fromMaybe (Left "it took over 300 s")+        <$> timeout (seconds 300) (receive (seconds 30) size (HC.responseBody response) keep)+        `finally` HC.responseClose response+      atomically $ writeTVar progress (either (const Broken) (const Received) received)+      finish $ received >> verified (fromForeignPtr0 buffer size)+    pure (Just (relay buffer progress))+  where+    size = fileSize uri++    verified content+      | sha1Hex content == fileHash uri = Right content+      | otherwise = Left "its SHA-1 does not match the file id"++    relay buffer progress write = go 0+      where+        go sent = atomically (beyond sent) >>= \case+          Receiving filled -> write (fromForeignPtr buffer sent (filled - sent)) >> go filled+          Received         -> when (sent < size) $ write (fromForeignPtr buffer sent (size - sent))+          -- the kind of error a peer that went away raises: Warp drops the+          -- reader's connection, as it must for a body shorter than announced,+          -- and does not print it (the download logs why)+          Broken           -> ioError+            (mkIOError resourceVanishedErrorType "upstream broke off" Nothing Nothing)++        beyond sent = readTVar progress >>= \case+          Receiving filled+            | filled <= sent -> retrySTM+          settled          -> pure settled++data Progress = Receiving !Int | Received | Broken++-- | What the downloads under way may hold between them. A file being fetched+-- is in memory until it is cached, and every request for it meanwhile fetches+-- it again, so without a limit one valid link, asked for often enough, takes+-- all the memory there is.+-- ponytail: a fixed 512 MiB; make it an option if a node has less to spare+bufferBudget :: Int+bufferBudget = 512 * 1024 * 1024++-- | Whether a download of @size@ bytes may start while those under way hold+-- @taken@. A file larger than the budget is fetched when nothing else is.+withinBudget :: Int -> Int -> Bool+withinBudget taken size = taken == 0 || taken + size <= bufferBudget++held :: TVar Int+held = unsafePerformIO (newTVarIO 0)++{-# NOINLINE held #-}++heldBytes :: IO Int+heldBytes = readTVarIO held++reserve :: Int -> IO Bool+reserve size = atomically $ do+  taken <- readTVar held+  let fits = withinBudget taken size+  fits <$ when fits (writeTVar held (taken + size))++release :: Int -> IO ()+release size = atomically $ modifyTVar' held (subtract size)++copyInto :: ForeignPtr Word8 -> Int -> ByteString -> IO ()+copyInto buffer offset chunk = unsafeUseAsCStringLen chunk $ \( source, len )+  -> withForeignPtr buffer $ \start -> copyBytes (start `plusPtr` offset) (castPtr source) len++-- | In microseconds.+seconds :: Int -> Int+seconds = (* 1000000)++-- | Lower a Sem action into IO so it can run after the request handler has returned.+deferToIO :: Member (Final IO) r => (x -> Sem r ()) -> Sem r (x -> IO ())+deferToIO k = withWeavingToFinal $ \s lower _ -> pure ((\x -> void (lower (k x <$ s))) <$ s)
+ src/Hath/Log.hs view
@@ -0,0 +1,22 @@+module Hath.Log ( logInfo, logWarning, logError ) where++import           Colog          ( Message, Msg(..), Severity(..) )+import           Colog.Polysemy ( Log )+import qualified Colog.Polysemy as Colog++import           GHC.Stack      ( emptyCallStack )++import           Polysemy       ( Member, Sem )++import           Relude++logInfo, logWarning, logError :: Member (Log Message) r => Text -> Sem r ()+logInfo = logAt Info++logWarning = logAt Warning++logError = logAt Error++logAt :: Member (Log Message) r => Severity -> Text -> Sem r ()+logAt severity text+  = Colog.log Msg { msgSeverity = severity, msgStack = emptyCallStack, msgText = text }
+ src/Hath/Protocol.hs view
@@ -0,0 +1,215 @@+-- | The wire formats and signatures of the Hentai\@Home protocol (official client+-- 1.6.6), as pure functions.+module Hath.Protocol+  ( clientBuild+  , responseHeaders+  , outgoingHeaders+  , sha1Hex+  , sha1HexText+  , rpcActionKey+  , rpcQuery+  , serverCommandKey+  , speedTestKey+  , keystampHash+  , hathRequestToken+  , keystampValid+  , commandTimeValid+  , fetchParams+  , sameSecret+  , verifyRPCResponse+  , rpcPayload+  , retryWithServerTime+  , parseURLParams+  ) where++import           Crypto.Hash             ( SHA1 )+import qualified Crypto.Hash             as Crypto++import           Data.ByteArray          ( constEq )+import           Data.ByteArray.Encoding ( Base(Base16), convertToBase )+import qualified Data.ByteString.Char8   as BS+import           Data.Char               ( isDigit, toLower )+import qualified Data.Map.Strict         as Map++import           Hath.Config             ( ClientConfig(..) )+import           Hath.Error              ( HathError(..) )++import           Network.HTTP.Types      ( Header, urlEncode )++import           Relude++-- | The protocol level this client implements (official 1.6.6), sent as @clientbuild@.+clientBuild :: Int64+clientBuild = 181++userAgent :: Header+userAgent = ( "User-Agent", "Hentai@Home 1.6.6" )++-- | The headers of a response whose body has this Content-Type and length, as+-- the official client writes them (HTTPSession.java:129-139); Warp adds the+-- Date. The official client leaves the length out of an empty response and+-- closes the connection; Warp would send that body chunked, so we say it is empty.+responseHeaders :: ByteString -> Int -> [ Header ]+responseHeaders contentType len+  = [ ( "Server", "Genetic Lifeform and Distributed Open Server 1.6.6" )+    , ( "Connection", "close" )+    , ( "Content-Type", contentType )+    , ( "Content-Length", show len )+    ]+  <> [ ( "Cache-Control", "public, max-age=31536000" ) | len > 0 ]++-- | The headers of a request of ours: the given ones and the official client's+-- User-Agent. The empty Accept-Encoding keeps http-client from adding+-- @gzip@, which the official client never asks for.+outgoingHeaders :: [ Header ] -> [ Header ]+outgoingHeaders headers = headers <> [ userAgent, ( "Accept-Encoding", "" ) ]++-- | SHA-1 in lower-case hex, the form every H\@H signature and file id uses.+sha1Hex :: ByteString -> ByteString+sha1Hex = convertToBase Base16 . Crypto.hash @ByteString @SHA1++sha1HexText :: ByteString -> Text+sha1HexText = decodeUtf8 . sha1Hex++signature :: [ ByteString ] -> ByteString+signature = sha1Hex . BS.intercalate "-"++-- | @actkey@ of an RPC request, for its @act@, @add@ and @acttime@.+rpcActionKey :: ClientConfig -> Text -> Text -> Int64 -> Text+rpcActionKey cfg act add time+  = decodeUtf8+  $ signature+    [ "hentai@home"+    , encodeUtf8 act+    , encodeUtf8 add+    , encodeUtf8 (clientId cfg)+    , show time+    , encodeUtf8 (key cfg)+    ]++-- | The query of a request to the RPC server, in the order the official client+-- writes it (ServerHandler.getURLQueryString), @add@ included even when empty.+-- The official client does not escape the values. We do, except the @;@+-- between the parts of @add@: a legitimate request goes out byte for byte the+-- same, and what a reader puts in @fileindex@ or @xres@ cannot add parameters.+rpcQuery :: ClientConfig -> Text -> Text -> Int64 -> ByteString+rpcQuery cfg act add time+  = BS.intercalate+    "&"+    [ name <> "=" <> escape value+    | ( name, value )+    <- [ ( "clientbuild", show clientBuild )+       , ( "act", encodeUtf8 act )+       , ( "add", encodeUtf8 add )+       , ( "cid", encodeUtf8 (clientId cfg) )+       , ( "acttime", show time )+       , ( "actkey", encodeUtf8 (rpcActionKey cfg act add time) )+       ]+    ]+  where+    escape = BS.intercalate ";" . map (urlEncode True) . BS.split ';'++-- | The key the RPC server puts on a @servercmd@ request. Only it knows our key.+serverCommandKey :: ClientConfig -> Text -> Text -> Int -> Text+serverCommandKey cfg command additional time+  = decodeUtf8+  $ signature+    [ "hentai@home-servercmd"+    , encodeUtf8 command+    , encodeUtf8 additional+    , encodeUtf8 (clientId cfg)+    , show time+    , encodeUtf8 (key cfg)+    ]++-- | @testkey@ of a @\/t\/@ speed test of the given size and time.+speedTestKey :: ClientConfig -> Int -> Int64 -> ByteString+speedTestKey cfg size time+  = signature+    [ "hentai@home-speedtest"+    , show size+    , show time+    , encodeUtf8 (clientId cfg)+    , encodeUtf8 (key cfg)+    ]++-- | The second half of the keystamp that lets a reader fetch a file id at a given time.+keystampHash :: ClientConfig -> Int64 -> ByteString -> ByteString+keystampHash cfg time fileId+  = BS.take 10 $ signature [ show time, fileId, encodeUtf8 (key cfg), "hotlinkthis" ]++-- | The @Hath-Request@ header that authenticates us to an image server, for a+-- file id as the reader asked for it.+hathRequestToken :: ClientConfig -> ByteString -> ByteString+hathRequestToken cfg fileId+  = encodeUtf8 (clientId cfg) <> "-" <> sha1Hex (encodeUtf8 (key cfg) <> fileId)++-- | Whether a keystamp, @\<time>-\<hash>@, entitles a reader to a file id at+-- the server's time @now@ (HTTPResponse.java:174-186): issued less than 15+-- minutes from it either way, with the hash in either case.+keystampValid :: ClientConfig -> Int64 -> ByteString -> ByteString -> Bool+keystampValid cfg now fileId stamp = case BS.split '-' stamp of+  [ time, hash ]+    | Just ( issued, "" ) <- BS.readInt time -> abs (now - fromIntegral issued) < 900+      && sameSecret (BS.map toLower hash) (keystampHash cfg (fromIntegral issued) fileId)+  _ -> False++-- | Whether a key or signature we were sent is the one we computed. The+-- comparison takes as long wherever the two first differ, so the time a+-- refusal takes does not tell a forger how much of a guess was right.+sameSecret :: ByteString -> ByteString -> Bool+sameSecret = constEq++-- | Whether the time of a server command or a speed test is close enough to+-- the server's @now@ (Settings.MAX_KEY_TIME_DRIFT).+commandTimeValid :: Int64 -> Int64 -> Bool+commandTimeValid now time = abs (now - time) <= 300++-- | The @fileindex@ and @xres@ of an @\/h\/@ request, which a file missing from+-- the cache is fetched by, if they have the official form: a number, and a+-- number or @org@ (HTTPResponse.java:194).+fetchParams :: Map ByteString ByteString -> Maybe ( ByteString, ByteString )+fetchParams options = do+  fileIndex <- Map.lookup "fileindex" options+  xres <- Map.lookup "xres" options+  guard (number fileIndex && (xres == "org" || number xres))+  pure ( fileIndex, xres )+  where+    number s = not (BS.null s) && BS.all isDigit s++-- | Check the 1.6.6 RPC signature. A signed reply starts with @OK-\<time>-\<sha1>@, the+-- sha1 covering @\<time>-\<client key>@ then @"\\n" ++ line@ for every later line. A+-- valid reply comes back with that first line reduced to @OK@; a forged or unsigned+-- OK becomes @RESPONSE_HASH_FAILED@; any other status (FAIL, KEY_EXPIRED, ...) passes+-- through.+verifyRPCResponse :: ByteString -> ByteString -> ByteString+verifyRPCResponse clientKey bytes = case BS.split '\n' bytes of+  status : rest+    | Just ( time, sig ) <- BS.break (== '-') <$> BS.stripPrefix "OK-" status+    , sameSecret sig ("-" <> sha1Hex (time <> "-" <> clientKey <> foldMap ("\n" <>) body))+      -> BS.intercalate "\n" ("OK" : body)+    | "OK" `BS.isPrefixOf` status -> "RESPONSE_HASH_FAILED"+    where+      -- Java's split drops trailing empty lines, and the hash is over what it keeps+      body = reverse $ dropWhile BS.null $ reverse rest+  _ -> bytes++-- | The lines after an @OK@ status line; any other status is the error.+rpcPayload :: ByteString -> Either HathError [ ByteString ]+rpcPayload reply = case BS.lines reply of+  [] -> Left EmptyResponse+  "OK" : payload -> Right payload+  status : _ -> Left $ RequestFailure $ decodeUtf8 status++-- | Whether to take the server's time and send the request again. On+-- @KEY_EXPIRED@ the official client refreshes server_stat and retries once,+-- but only a request without @add@ (ServerResponse.java:103-107).+retryWithServerTime :: Text -> ByteString -> Bool+retryWithServerTime add reply = add == "" && BS.takeWhile (/= '\n') reply == "KEY_EXPIRED"++-- | Parse @key1=value1;key2=value2;flag@. A key without a value maps to the+-- empty string, and a value may itself contain @=@.+parseURLParams :: ByteString -> Map ByteString ByteString+parseURLParams = Map.fromList . map keyValue . filter (not . BS.null) . BS.split ';'+  where+    keyValue = second (BS.drop 1) . BS.break (== '=')
+ src/Hath/RPC.hs view
@@ -0,0 +1,385 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE TemplateHaskell #-}++-- | Our side of the RPC with the H\@H server (protocol 1.6.6): the effect, its+-- HTTP interpreter, and one function per request.+module Hath.RPC+  ( RPC(..)+  , runRPC+  , runRPCIO+  , worthRetrying+  , httpManager+  , connectMicros+  , serverStat+  , clientLogin+  , clientStart+  , StartFailure(..)+  , startFailure+  , clientStop+  , stillAlive+  , toldToStop+  , overload+  , clientSettings+  , getCertificate+  , getBlacklist+  , fileSources+  , nextGallery+  , galleryFileSource+  , reportDownloadFailures+  , failureReport+  ) where++import           Colog                        ( Message, richMessageAction )+import           Colog.Polysemy               ( Log, runLogAction )++import           Control.Exception            ( try )++import           Crypto.Store.PKCS12          ( readP12FileFromMemory+                                              , recover+                                              , recoverAuthenticated+                                              , toCredential+                                              , toProtectionPassword+                                              )++import qualified Data.ByteString.Char8        as BS+import qualified Data.ByteString.Lazy         as LBS+import           Data.String.Interpolate      ( i )+import qualified Data.Text                    as T+import           Data.X509                    ( CertificateChain, PrivKey )++import           Hath.Config                  ( ClientConfig(..) )+import           Hath.Error                   ( HathError(..) )+import           Hath.FileURI                 ( FileURI, parseFileURI )+import           Hath.Gallery                 ( GalleryFile(..)+                                              , GalleryMetadata(..)+                                              , parseMetadata+                                              )+import           Hath.Log                     ( logInfo, logWarning )+import           Hath.Protocol                ( clientBuild+                                              , outgoingHeaders+                                              , retryWithServerTime+                                              , rpcPayload+                                              , rpcQuery+                                              , verifyRPCResponse+                                              )+import           Hath.ServerTime              ( serverNow, setServerTime )+import           Hath.Settings                ( HathSettings+                                              , defaultHathSettings+                                              , parseSettings+                                              , settingNumber+                                              )++import qualified Network.HTTP.Client          as HC+import           Network.HTTP.Client.Internal ( throwHttp )+import           Network.HTTP.Client.TLS      ( tlsManagerSettings )+import           Network.HTTP.Types           ( statusCode, statusIsSuccessful )++import           Polysemy+import           Polysemy.Error               ( Error, errorToIOFinal, fromEither, throw )+import           Polysemy.Reader              ( Reader, ask, runReader )++import           Relude                       hiding ( Reader, ask, runReader )++import           System.IO.Unsafe             ( unsafePerformIO )+import           System.Timeout               ( timeout )++-- | A request to the RPC server: its @act@ and @add@ (empty when it has no arguments).+data RPC m a where+  -- | @GET \/16\/rpc@. The reply's signature has been checked: see 'verifyRPCResponse'.+  CallRPC :: Text -> Text -> RPC m ByteString+  -- | @GET \/16\/dl@, the gallery downloader's endpoint, whose replies carry their+  -- own signature ('parseMetadata' checks it).+  CallDownloader :: Text -> Text -> RPC m ByteString++makeSem ''RPC++runRPC+  :: Members '[ Embed IO, Error HathError, Reader ClientConfig ] r => Sem (RPC : r) a -> Sem r a+runRPC = interpret $ \case+  -- get_cert answers with the PKCS#12 file itself, which is not a signed reply+  CallRPC "get_cert" add -> request "/16/rpc" "get_cert" add+  CallRPC act add        -> do+    reply <- signed act add+    if retryWithServerTime add reply+      then do+        stat <- signed "server_stat" ""+        whenRight_ (rpcPayload stat) (embed . recordServerTime)+        signed act add+      else pure reply+  CallDownloader act add -> request "/16/dl" act add+  where+    signed act add = do+      cfg <- ask @ClientConfig+      verifyRPCResponse (encodeUtf8 (key cfg)) <$> request "/16/rpc" act add++-- | Run RPC requests on their own, outside the web server.+runRPCIO :: ClientConfig+         -> Sem '[ RPC, Log Message, Reader ClientConfig, Error HathError, Embed IO, Final IO ] a+         -> IO (Either HathError a)+runRPCIO cfg+  = runFinal+  . embedToFinal @IO+  . errorToIOFinal @HathError+  . runReader cfg+  . runLogAction @IO richMessageAction+  . runRPC++-- | The RPC server, unless @HATH_RPC_HOST@ (@host@ or @host:port@) names another;+-- the tests and hath-bench point us at theirs that way.+rpcServer :: IO ( String, Int )+rpcServer = lookupEnv "HATH_RPC_HOST" <&> \case+  Nothing       -> ( "rpc.hentaiathome.net", 80 )+  Just override -> case break (== ':') override of+    ( host, ':' : port ) -> ( host, fromMaybe 80 (readMaybe port) )+    ( host, _ )          -> ( host, 80 )++-- | How long to wait for a reply, connecting included. The official client+-- waits up to an hour; a reader waiting on @srfetch@ would not.+rpcTimeoutMicros :: Int+rpcTimeoutMicros = 60 * 1000000++-- | The connections of every request we send: to the RPC server, for proxy+-- tests, to image servers. Each gives up after 'connectMicros' if it cannot+-- connect, as the official client's do (FileDownloader.java:165,+-- ProxyFileDownloader.java:86). http-client has no setting for that alone (its+-- response timeout counts the connection too), so every request sets its own+-- response timeout.+httpManager :: HC.Manager+httpManager+  = unsafePerformIO+  $ HC.newManager+    tlsManagerSettings+    { HC.managerRawConnection = within5s <$> HC.managerRawConnection tlsManagerSettings+    , HC.managerTlsConnection = within5s <$> HC.managerTlsConnection tlsManagerSettings+    }+  where+    within5s connect address host port+      = timeout connectMicros (connect address host port)+      >>= maybe (throwHttp HC.ConnectionTimeout) pure++{-# NOINLINE httpManager #-}++connectMicros :: Int+connectMicros = 5 * 1000000++-- | One GET to the RPC server, signed with the client key and the server's time.+request :: Members '[ Embed IO, Error HathError, Reader ClientConfig ] r+        => ByteString+        -> Text+        -> Text+        -> Sem r ByteString+request endpoint act add = do+  cfg <- ask @ClientConfig+  reply <- embed $ do+    ( host, port ) <- rpcServer+    time <- serverNow+    exchange+      HC.defaultRequest+      { HC.host = BS.pack host+      , HC.port = port+      , HC.path = endpoint+      , HC.queryString = rpcQuery cfg act add time+      , HC.requestHeaders = outgoingHeaders [ ( "Connection", "Close" ) ]+      , HC.responseTimeout = HC.responseTimeoutMicro rpcTimeoutMicros+      }+  fromEither reply++-- | The longest reply we take, as the official client reads one into memory+-- (FileDownloader.java:186). The RPC server is reached over plain HTTP: who+-- sits in between could otherwise send a reply without end.+maxReplyBytes :: Int+maxReplyBytes = 10485760++-- | Send a request and read the body of its 2xx reply. Like the official+-- FileDownloader, make up to three attempts, one right after another+-- (FileDownloader.java:147, 280-292).+exchange :: HC.Request -> IO (Either HathError ByteString)+exchange req = go (3 :: Int)+  where+    go attemptsLeft = attempt >>= \case+      Left ( status, _ )+        | attemptsLeft > 1, worthRetrying status -> go (attemptsLeft - 1)+      outcome -> pure $ first (TransportError . snd) outcome++    -- the reply's body, or else the status it came with (none if the exchange broke off) and why+    attempt          = try @HC.HttpException (HC.withResponse req httpManager reply) <&> \case+      Left (HC.HttpExceptionRequest _ content) -> Left ( Nothing, show content )+      Left e -> Left ( Nothing, show e )+      Right ( status, body )+        | not (statusIsSuccessful status) -> Left $ httpError $ statusCode status+        | tooLong body -> Left ( Nothing, "a reply of more than 10 MiB" )+        | otherwise -> Right $ toStrict body++    -- one byte past the limit shows that there was more+    reply response+      = (,) (HC.responseStatus response)+      <$> HC.brReadSome (HC.responseBody response) (maxReplyBytes + 1)++    tooLong body = LBS.length body > fromIntegral maxReplyBytes++    httpError status = ( Just status, "HTTP " <> show status )++-- | Whether an attempt that failed with this HTTP status (none if the exchange+-- broke off) is worth repeating. FileDownloader repeats every failure but 404+-- and 410, which HttpURLConnection reports as FileNotFoundException.+worthRetrying :: Maybe Int -> Bool+worthRetrying = maybe True (`notElem` [ 404, 410 ])++payload :: Member (Error HathError) r => ByteString -> Sem r [ ByteString ]+payload = fromEither . rpcPayload++-- | The payload of an OK reply that carries settings. Like the official client,+-- take the server's clock from any of them that has it.+settingsPayload :: Members '[ RPC, Error HathError, Embed IO ] r => Text -> Sem r [ ByteString ]+settingsPayload act = do+  settings <- payload =<< callRPC act ""+  embed $ recordServerTime settings+  pure settings++recordServerTime :: [ ByteString ] -> IO ()+recordServerTime = traverse_ setServerTime . settingNumber "server_time"++-- | Check in with the RPC server and take its clock. Exit if it no longer+-- accepts our build (official Settings.java:187-196).+serverStat :: Members '[ RPC, Error HathError, Log Message, Embed IO ] r => Sem r ()+serverStat = do+  stat <- settingsPayload "server_stat"+  when (settingNumber "min_client_build" stat > Just clientBuild)+    $ embed @IO+    $ die+      [i|The H@H network no longer accepts client build #{clientBuild} (official 1.6.6), the one hs-hath implements. Update hs-hath.|]+  when (settingNumber "cur_client_build" stat > Just clientBuild)+    $ logWarning+      [i|The H@H network has a client newer than build #{clientBuild}, the one hs-hath implements.|]++clientLogin :: Members '[ RPC, Error HathError, Log Message, Embed IO ] r => Sem r HathSettings+clientLogin = do+  logInfo "Logging in"+  parseSettings defaultHathSettings <$> settingsPayload "client_login"++-- | Tell the server we are listening. It tests our port before it answers.+clientStart :: Members '[ RPC, Error HathError ] r => Sem r ()+clientStart = void $ payload =<< callRPC "client_start" ""++-- | What a failed client_start leaves us to do (official ServerHandler.notifyStart).+data StartFailure+  = Refused !Text+    -- ^ the server will not have us: exit, saying why+  | Unannounced !Text+    -- ^ say what went wrong and stay up without the network, as the official+    -- client does, so that the owner can test the port+  deriving ( Eq, Show )++-- | Read a failed client_start for a client listening on this port.+startFailure :: Int -> HathError -> StartFailure+startFailure port = \case+  RequestFailure status+    | "FAIL_OTHER_CLIENT_CONNECTED" `T.isPrefixOf` status -> Refused+      "The server detected that another client was already connected from this computer or local network. You can only have one client running per public IPv4 address."+    | "FAIL_CID_IN_USE" `T.isPrefixOf` status -> Refused+      "The server detected that another client is already using this client ID. If the client was not shut down properly and your IP changed, you may have to wait up to 24 hours for the old session to time out."+    | "FAIL_CONNECT_TEST" `T.isPrefixOf` status -> Unannounced+      [i|The client has failed the external connectivity test: the server could not connect to it. Check that port #{port} is open and forwarded to this computer, and that your incoming and outgoing IP address are the same. The client keeps running so that you can test the port; restart it once that is fixed.|]+  failure -> Unannounced+    [i|Startup notification failed (#{failure}). The client keeps running, but the network will not send it anything; restart it.|]++clientStop :: Members '[ RPC, Error HathError ] r => Sem r ()+clientStop = void $ payload =<< callRPC "client_stop" ""++stillAlive :: Members '[ RPC, Error HathError ] r => Sem r ()+stillAlive = void $ payload =<< callRPC "still_alive" ""++-- | Whether a failed still_alive means we must shut down: the server answers+-- @TERM_BAD_NETWORK@ to a client it cannot reach (official CakeSphere.run). Any+-- other failure waits for the next heartbeat.+toldToStop :: HathError -> Bool+toldToStop = \case+  RequestFailure status -> "TERM_BAD_NETWORK" `T.isPrefixOf` status+  _ -> False++-- | Tell the server we are close to our connection limit; it then sends us+-- fewer readers for a while (official ServerHandler.notifyOverload).+overload :: Members '[ RPC, Error HathError ] r => Sem r ()+overload = void $ payload =<< callRPC "overload" ""++-- | The settings as the server has them now, applied onto the ones we hold.+clientSettings+  :: Members '[ RPC, Error HathError, Embed IO ] r => HathSettings -> Sem r HathSettings+clientSettings current = parseSettings current <$> settingsPayload "client_settings"++-- | Our TLS certificate. The server sends a PKCS#12 file whose password is the client key.+getCertificate :: Members '[ RPC, Error HathError, Reader ClientConfig ] r+               => Sem r ( CertificateChain, PrivKey )+getCertificate = do+  p12 <- callRPC "get_cert" ""+  cfg <- ask @ClientConfig+  let password = encodeUtf8 (key cfg)+      opened   = do+        container <- readP12FileFromMemory p12+        ( _, pkcs12 ) <- recoverAuthenticated password container+        recover (toProtectionPassword password) (toCredential pkcs12)+  case opened of+    Left err -> throw $ CertificateFailure $ show err+    Right Nothing -> throw $ CertificateFailure "no credential"+    Right (Just credential) -> pure credential++-- | Files added to the blacklist in the last so many seconds. A line that is+-- not a file id is skipped, as the official client does.+getBlacklist :: Members '[ RPC, Error HathError ] r => Int -> Sem r [ FileURI ]+getBlacklist seconds+  = mapMaybe parseFileURI <$> (payload =<< callRPC "get_blacklist" (show seconds))++-- | URLs a file of our static ranges can be fetched from (@srfetch@), given+-- the @fileindex@ and @xres@ of the request that asked for it and the file id+-- as it asked for it.+fileSources :: Members '[ RPC, Error HathError ] r+            => ByteString+            -> ByteString+            -> ByteString+            -> Sem r [ ByteString ]+fileSources fileIndex xres fileId+  = payload =<< callRPC "srfetch" (decodeUtf8 $ BS.intercalate ";" [ fileIndex, xres, fileId ])++-- | The next gallery queued for download, or why there is none+-- (@NO_PENDING_DOWNLOADS@, @INVALID_REQUEST@, or a list that was refused). The+-- same request marks the gallery before it done and takes it off the queue+-- (GalleryDownloader.java:163-170).+nextGallery :: Members '[ RPC, Reader ClientConfig ] r+            => Maybe GalleryMetadata+            -> Sem r (Either Text GalleryMetadata)+nextGallery done = do+  cfg <- ask @ClientConfig+  parseMetadata (encodeUtf8 (key cfg)) <$> callDownloader "fetchqueue" (maybe "" marked done)+  where+    marked gallery = [i|#{galleryID gallery};#{galleryMinXRes gallery}|]++-- | Where to download a gallery file from (@dlfetch@), the first line of an OK+-- reply, as the official client takes it (ServerHandler.java:341-359); the last+-- argument numbers this attempt at the file, from 1.+galleryFileSource :: Members '[ RPC, Error HathError ] r+                  => GalleryMetadata+                  -> GalleryFile+                  -> Int+                  -> Sem r (Maybe ByteString)+galleryFileSource gallery file attempt+  = listToMaybe+  <$> (payload+       =<< callRPC+         "dlfetch"+         [i|#{galleryID gallery};#{galleryFilePage file};#{galleryFileIndex file};#{galleryFileXRes file};#{attempt}|])++reportDownloadFailures :: Member RPC r => [ Text ] -> Sem r ()+reportDownloadFailures = traverse_ (void . callRPC "dlfails") . failureReport++-- | The @add@ of a dlfails report: the distinct failures, as+-- @host-fileindex-xres@, if there are 1 to 50 of them. More than that probably+-- means the trouble is ours, so the official client keeps quiet+-- (ServerHandler.java:366-371).+failureReport :: [ Text ] -> Maybe Text+failureReport failures+  | count >= 1 && count <= 50 = Just (T.intercalate ";" distinct)+  | otherwise = Nothing+  where+    distinct = ordNub failures++    count    = length distinct
+ src/Hath/Server.hs view
@@ -0,0 +1,373 @@+-- | The web server: the application it serves, and the client's life around+-- it, from client_start to client_stop.+module Hath.Server+  ( startServer+  , ServerOptions(..)+  , makeApplication+  , ServerEnv(..)+  , ServerAction(..)+  , needsRenewal+  ) where++import           Control.Concurrent                   ( threadDelay )++import           Data.Hourglass                       ( Elapsed(..), Seconds(..), timeGetElapsed )+import           Data.String.Interpolate              ( i )+import           Data.X509                            ( CertificateChain(..)+                                                      , PrivKey+                                                      , certValidity+                                                      )+import qualified Data.X509                            as X509++import           Database.SQLite.Simple               ( withConnection )++import           Hath.Cache                           ( CacheRunner(..), deleteFile )+import           Hath.Cache.Filesystem                ( newFsCache+                                                      , runCacheFilesystem+                                                      , runEvictor+                                                      )+import           Hath.Cache.R2                        ( mkR2Connection, runCacheR2 )+import           Hath.Cache.SQLite                    ( initializeDB, runCacheSQLite )+import           Hath.Config                          ( CacheBackend(..), ClientConfig(..) )+import           Hath.Error                           ( HathError )+import           Hath.Gallery.Downloader              ( runGalleryDownloader )+import           Hath.Log                             ( logWarning )+import           Hath.RPC                             ( StartFailure(..)+                                                      , clientSettings+                                                      , clientStart+                                                      , clientStop+                                                      , getBlacklist+                                                      , getCertificate+                                                      , overload+                                                      , runRPCIO+                                                      , startFailure+                                                      , stillAlive+                                                      , toldToStop+                                                      )+import           Hath.Server.App                      ( ServerAction(..), ServerEnv(..) )+import           Hath.Server.ConnectionLimit          ( connectionLimit )+import           Hath.Server.FloodControl             ( isRPCServer, newFloodControl )+import           Hath.Server.Handlers                 ( application )+import           Hath.Server.LegacyCiphers+import           Hath.Server.Middleware+import           Hath.ServerTime                      ( serverNow )+import           Hath.Settings                        ( HathSettings(..) )+import           Hath.Stats                           ( newStatsEnv )++import           Network.TLS                          ( Credentials(Credentials)+                                                      , TLSException(HandshakeFailed)+                                                      , Version(..)+                                                      )+import           Network.TLS.Extra                    ( ciphersuite_all )+import qualified Network.TLS.SessionManager           as SM+import           Network.Wai                          ( Application, Request )+import           Network.Wai.Handler.Warp             ( defaultOnException+                                                      , defaultSettings+                                                      , setBeforeMainLoop+                                                      , setGracefulShutdownTimeout+                                                      , setInstallShutdownHandler+                                                      , setOnClose+                                                      , setOnException+                                                      , setOnOpen+                                                      , setPort+                                                      )+import           Network.Wai.Handler.WarpTLS          ( OnInsecure(AllowInsecure)+                                                      , TLSSettings(..)+                                                      , WarpTLSException(ClientClosedConnectionPrematurely)+                                                      , runTLS+                                                      , tlsSettingsSni+                                                      )+import           Network.Wai.Middleware.RealIp        ( realIpHeader )+import           Network.Wai.Middleware.RequestLogger ( logStdout )++import           Polysemy.Error                       ( catch )++import           Relude++import           UnliftIO                             ( TChan+                                                      , mapConcurrently_+                                                      , race+                                                      , readTChan+                                                      , tryAny+                                                      , waitCatch+                                                      , withAsync+                                                      , writeTChan+                                                      )++makeApplication :: ServerEnv -> Application+makeApplication env = middleware (application env)+  where+    middleware = realIp . floodControl . tracingConnections stats . tracingTimeUsage stats++    stats = envStats env++    realIp+      | envTrustProxyHeaders env = realIpHeader "X-Forwarded-For"+      | otherwise = id++    floodControl = maybe id (`rateLimitMiddleware` fromRPCServer) (envFloodControl env)++    fromRPCServer ip = (`isRPCServer` ip) . rpcServerIPs <$> readTVarIO (envSettings env)++data ServerOptions+  = ServerOptions { disableRateLimit  :: !Bool+                    -- ^ no flood control, for when every reader appears to come from one address+                  , trustProxyHeaders :: !Bool+                    -- ^ take the reader's address from @X-Forwarded-For@+                  }++-- | Serve until told to shut down, then return; exit if the RPC server will not+-- have us. The settings and certificate are the ones it gave us at login.+startServer :: ClientConfig+            -> ServerOptions+            -> HathSettings+            -> ( CertificateChain, PrivKey )+            -> TChan ServerAction+            -> IO ()+startServer config options settings certs actions = do+  settingsVar <- newTVarIO settings+  downloadRequest <- newEmptyMVar+  stats <- newStatsEnv+  floodControl <- if disableRateLimit options+    then pure Nothing+    else Just <$> newFloodControl+  -- an empty token would let in anyone who sends "Bearer "+  metricsToken <- mfilter (not . null) <$> lookupEnv "HATH_METRICS_TOKEN"+  runGalleryDownloader config stats settingsVar downloadRequest+    `alongside` withCacheBackend+      config+      (diskLimitBytes <$> readTVarIO settingsVar)+      (\cache -> serverLoop+         ServerEnv { envConfig = config+                   , envSettings = settingsVar+                   , envActions = actions+                   , envDownloadRequest = downloadRequest+                   , envStats = stats+                   , envCache = cache+                   , envFloodControl = floodControl+                   , envTrustProxyHeaders = trustProxyHeaders options+                   , envMetricsToken = encodeUtf8 <$> metricsToken+                   }+         certs)++alongside :: IO a -> IO b -> IO b+alongside background action = withAsync background (const action)++-- | Open the cache the config asks for. The Filesystem backend also gets its+-- evictor, which follows the disk limit the server assigns.+withCacheBackend :: ClientConfig -> IO Int64 -> (CacheRunner -> IO a) -> IO a+withCacheBackend config diskLimit use = case cacheBackend config of+  CacheBackendSQLite     -> withConnection (toString $ cachePath config) $ \conn -> do+    initializeDB conn+    use CacheRunner { runCacheWith = runCacheSQLite conn }+  CacheBackendR2         -> case r2Config config of+    Nothing -> die "cacheBackend=r2 but r2Config is missing"+    Just r2 -> mkR2Connection r2 >>= \case+      Left err   -> die $ "Failed to initialize R2: " <> toString err+      Right conn -> use CacheRunner { runCacheWith = runCacheR2 conn }+  CacheBackendFilesystem -> do+    fs <- newFsCache (toString $ cachePath config)+    runEvictor fs diskLimit `alongside` use CacheRunner { runCacheWith = runCacheFilesystem fs }++-- | Bind the port, announce ourselves with client_start, then serve and+-- handle commands until one ends the client (official HentaiAtHomeClient.run):+--+-- * 'Cert' fetches a new certificate and swaps it in for the next handshakes;+--   the listener and open transfers carry on.+-- * 'GracefulShutdown' ends the client as 'shutDown' says.+serverLoop :: ServerEnv -> ( CertificateChain, PrivKey ) -> IO ()+serverLoop env certs = do+  let config = envConfig env+  credentials <- newIORef certs+  port <- clientPort <$> readTVarIO (envSettings env)+  listening <- newEmptyMVar+  closeListener <- newEmptyMVar+  ( admit, release ) <- connectionLimit (envSettings env) (notifyOverload config)+  let warpSettings+        = setBeforeMainLoop (putMVar listening ())+        $ setInstallShutdownHandler (putMVar closeListener)+        $ setOnOpen admit+        $ setOnClose release+        $ setGracefulShutdownTimeout (Just 25)+        $ setOnException quietHandshakes+        $ setPort port defaultSettings+  withAsync (runTLS (tlsSettings credentials) warpSettings $ logStdout (makeApplication env))+    $ \web -> do+      -- the server tests our port before it answers client_start+      race (waitCatch web) (takeMVar listening) >>= \case+        Left outcome -> die [i|Could not listen on port #{port}: #{outcome}|]+        Right ()     -> pass+      started <- announce config port+      let untilShutdown+            | started = housekeeping env credentials `alongside` commands env credentials+            | otherwise = commands env credentials+      race (waitCatch web) untilShutdown >>= \case+        Left outcome -> die [i|The web server stopped: #{outcome}|]+        Right ()+          | started -> shutDown config (join $ readMVar closeListener) (void $ waitCatch web)+          | otherwise -> pass+  where+    tlsSettings credentials+      -- asked on every handshake (SNI or not), so a swapped certificate applies at once+      = (tlsSettingsSni $ \_ -> Credentials . pure <$> readIORef credentials)+        { onInsecure = AllowInsecure+        , tlsCiphers = ciphersuite_all+            ++ [ cipher_ECDHE_ECDSA_AES128CBC_SHA+               , cipher_ECDHE_RSA_AES128CBC_SHA+               , cipher_ECDHE_RSA_AES256CBC_SHA+               ]+          -- tls >= 2 speaks nothing older; it rejects SSL3/TLS1.0/TLS1.1 hellos before looking at this+        , tlsAllowedVersions = [ TLS12, TLS13 ]+          -- resumed handshakes skip the certificate and its RSA signature (several ms on a small VM)+        , tlsSessionManagerConfig = Just SM.defaultConfig+        }++-- | Warp's exception printer, minus connections that never got past TLS:+-- scanners probing old protocols and cipher suites, and clients that connect+-- and send nothing (thousands a day). None of them reached the application.+quietHandshakes :: Maybe Request -> SomeException -> IO ()+quietHandshakes req e+  | Just (HandshakeFailed _) <- fromException e = pass+  | Just ClientClosedConnectionPrematurely <- fromException e = pass+  | otherwise = defaultOnException req e++notifyOverload :: ClientConfig -> IO ()+notifyOverload config = runRPCIO config overload >>= \case+  Left failure -> putStrLn [i|Overload notification failed: #{failure}|]+  Right ()     -> pass++-- | Send client_start, once. True if the server took us; if it refuses us for+-- good, exit.+announce :: ClientConfig -> Int -> IO Bool+announce config port = runRPCIO config clientStart >>= \case+  Right ()     -> True <$ putStrLn "Start notification successful."+  Left failure -> case startFailure port failure of+    Refused why     -> die (toString why)+    Unannounced why -> False <$ putStrLn (toString why)++commands :: ServerEnv -> IORef ( CertificateChain, PrivKey ) -> IO ()+commands env credentials = atomically (readTChan (envActions env)) >>= \case+  GracefulShutdown -> pass+  Cert -> do+    renewed <- refreshCerts (envConfig env)+    now <- serverNow+    case renewed of+      Just new@( chain, _ )+        | not (needsRenewal now chain) -> writeIORef credentials new >> commands env credentials+        -- the official client refuses such a certificate too+        | otherwise -> putStrLn "The new certificate expires within a day as well; shutting down"+      Nothing -> putStrLn "Could not get a new certificate; shutting down"++-- | What the official client does once client_start went through: refresh the+-- settings, delete the files blacklisted in the last three days, then, at its+-- intervals, send still_alive (110 s), delete what the last 12 hours+-- blacklisted (6 h) and look at the certificate (5 min). What ends the client+-- goes through the command channel.+housekeeping :: ServerEnv -> IORef ( CertificateChain, PrivKey ) -> IO ()+housekeeping env credentials = do+  logFailure refreshSettings+  logFailure $ applyBlacklist config (envCache env) 259200+  mapConcurrently_+    (\( delay, task ) -> infinitely $ threadDelay delay >> logFailure task)+    [ ( seconds 110, heartbeat )+    , ( hours 6, applyBlacklist config (envCache env) 43200 )+    , ( minutes 5, checkCertificate )+    ]+  where+    config = envConfig env++    refreshSettings = readTVarIO (envSettings env) >>= runRPCIO config . clientSettings >>= \case+      Right settings -> atomically $ writeTVar (envSettings env) settings+      Left failure   -> putStrLn [i|Failed to refresh settings: #{failure}|]++    heartbeat = runRPCIO config stillAlive >>= \case+      Right ()     -> pass+      Left failure+        | toldToStop failure -> do+          putStrLn+            "The client is shutting down since the network is misconfigured; correct firewall/forwarding settings then restart the client."+          send GracefulShutdown+        | otherwise -> putStrLn [i|Failed stillAlive test: #{failure}; will retry later|]++    -- the official client gives up a day before the certificate expires; we fetch another first+    checkCertificate = do+      now <- serverNow+      ( chain, _ ) <- readIORef credentials+      when (needsRenewal now chain) $ do+        putStrLn "The certificate expires within a day; fetching a new one"+        send Cert++    send = atomically . writeTChan (envActions env)++    -- an exception one task did not expect must not stop the heartbeat+    logFailure task = tryAny task >>= \case+      Left e   -> putStrLn [i|Housekeeping task failed: #{e}|]+      Right () -> pass++    seconds n = n * 1000000++    minutes n = seconds (n * 60)++    hours n = minutes (n * 60)++-- | Delete the files blacklisted in the last so many seconds (official+-- CacheHandler.processBlacklist). The list can be long: only its length is logged.+applyBlacklist :: ClientConfig -> CacheRunner -> Int -> IO ()+applyBlacklist config cache seconds = do+  outcome <- runRPCIO config $ do+    files <- getBlacklist seconds+    -- one file that cannot be deleted must not keep the others+    runCacheWith cache $ for_ files $ \uri -> deleteFile uri `catch` \(failure :: HathError)+      -> logWarning [i|Could not delete blacklisted file #{uri}: #{failure}|]+    pure (length files)+  putStrLn $ case outcome of+    Left failure -> [i|Failed to retrieve the file blacklist, will try again later: #{failure}|]+    Right count  -> [i|Blacklist: #{count} files, deleted from the cache where present|]++-- | End a client that client_start announced (official+-- HentaiAtHomeClient.shutdown and httpServerShutdown): client_stop, so the+-- network sends no one new; 5 s more of serving, for readers already on their+-- way; then no new connections, and up to 25 s for open ones to finish.+shutDown :: ClientConfig -> IO () -> IO () -> IO ()+shutDown config closeListener drained = do+  runRPCIO config clientStop >>= \case+    Left failure -> putStrLn [i|Shutdown notification failed: #{failure}|]+    Right ()     -> pass+  putStrLn "Shutdown in progress - please wait up to 30 seconds"+  threadDelay 5000000+  closeListener+  drained++-- | Whether a certificate must be replaced at this Unix time: with less than a+-- day left, the official client will not serve it (HTTPServer.isCertExpired).+needsRenewal :: Int64 -> CertificateChain -> Bool+needsRenewal now = \case+  CertificateChain (leaf : _) -> notAfter leaf < now + 86400+  CertificateChain []         -> True+  where+    notAfter leaf = case timeGetElapsed (snd $ certValidity $ X509.getCertificate leaf) of+      Elapsed (Seconds unixTime) -> unixTime++-- | Fetch a new certificate, retrying three times with exponential backoff+-- (1, 2, 4 seconds). Nothing if none comes.+refreshCerts :: ClientConfig -> IO (Maybe ( CertificateChain, PrivKey ))+refreshCerts config = go (0 :: Int)+  where+    go retries = runRPCIO config getCertificate >>= \case+      Right newCerts -> pure (Just newCerts)+      Left e+        | retries >= 3 -> do+          putStrLn $ "Failed to refresh certs: " <> show e+          putStrLn "Giving up after 3 retries"+          pure Nothing+        | otherwise -> do+          let delaySeconds = 2 ^ retries :: Int+          putStrLn $ "Failed to refresh certs: " <> show e+          putStrLn+            $ "Retrying in "+            <> show delaySeconds+            <> " seconds (attempt "+            <> show (retries + 1)+            <> "/3)"+          threadDelay (1000000 * delaySeconds)+          go (retries + 1)
+ src/Hath/Server/API.hs view
@@ -0,0 +1,132 @@+-- | The HTTP interface this client offers to readers and to the RPC server:+-- what a request path asks for, read as the official client reads it+-- (HTTPResponse.parseRequest), and the server commands it understands.+module Hath.Server.API+  ( Route(..)+  , route+  , ServerCommand(..)+  , ProxyTest(..)+  , parseServerCommand+  ) where++import qualified Data.ByteString.Char8 as BSC+import           Data.List             ( dropWhileEnd )+import qualified Data.Map.Strict       as Map+import qualified Data.Text             as T++import           Network.HTTP.Types    ( Status, status400, status403, status404 )++import           Relude++data Route+  = Favicon+  | Robots+  | File !ByteString !ByteString+    -- ^ @\/h\/\<file id>\/\<options>\/\<file name>@: a file for a reader. The+    -- file name is optional.+  | ServerCmd !Text !Text !Int !Text+    -- ^ @\/servercmd\/\<command>\/\<additional>\/\<time>\/\<key>@: a command+    -- from the RPC server+  | Test !Int !Int64 !ByteString+    -- ^ @\/t\/\<size>\/\<time>\/\<key>\/\<anything>@: a speed test, for the RPC+    -- server or another client. The last part is optional.+  | Metrics+  | AdminSettings+  | Refuse !Status+-- ^ a path the official client answers with this error. Where its numbers are+-- not numbers, it drops the connection instead.++-- | The route of a request path. Like the official client, we do not+-- percent-decode the path, except @%3d@ (=): the keys of server commands and+-- speed tests are computed over it as it is. Empty parts at its end do not count.+route :: ByteString -> Route+route path = case dropWhileEnd BSC.null (BSC.split '/' (unescapeEquals path)) of+  [ "", "favicon.ico" ] -> Favicon+  [ "", "robots.txt" ] -> Robots+  [ "", "metrics" ] -> Metrics+  [ "", "admin", "settings" ] -> AdminSettings+  "" : "h" : fileId : options : _ -> File fileId options+  "" : "h" : _ -> Refuse status400+  "" : "servercmd" : command : additional : time : commandKey : _ -> maybe+    (Refuse status403)+    (\t -> ServerCmd (decodeUtf8 command) (decodeUtf8 additional) t (decodeUtf8 commandKey))+    (number time)+  "" : "servercmd" : _ -> Refuse status403+  "" : "t" : size : time : testKey : _ -> fromMaybe (Refuse status400)+    $ Test <$> number size <*> number time <*> pure testKey+  "" : "t" : _ -> Refuse status400+  _ -> Refuse status404++unescapeEquals :: ByteString -> ByteString+unescapeEquals path = case BSC.breakSubstring "%3d" path of+  ( before, after )+    | BSC.null after -> before+    | otherwise -> before <> "=" <> unescapeEquals (BSC.drop 3 after)++data ServerCommand+  = StillAlive+  | ThreadedProxyTest !ProxyTest+    -- ^ download from another client and report how it went+  | SpeedTest !Int+    -- ^ send this many bytes+  | RefreshSettings+  | StartDownloader+    -- ^ there is a gallery in your download queue+  | RefreshCerts+  deriving ( Eq, Show )++-- | Downloads from another client's @\/t\/@ route, to time them.+data ProxyTest+  = ProxyTest { proxyTestCount :: !Int, proxyTestSize :: !Int64, proxyTestURL :: !String }+  deriving ( Eq, Show )++-- | A command by its name, in any case, with the parameters it takes+-- (HTTPResponse.processRemoteAPICommand). Nothing for an unknown name or a+-- parameter it cannot read: the official client answers both with INVALID_COMMAND.+parseServerCommand :: Text -> Map ByteString ByteString -> Maybe ServerCommand+parseServerCommand name params = case T.toLower name of+  "still_alive" -> Just StillAlive+  "threaded_proxy_test" -> ThreadedProxyTest <$> parseProxyTest params+  "speed_test" -> SpeedTest <$> maybe (Just 1000000) number (Map.lookup "testsize" params)+  "refresh_settings" -> Just RefreshSettings+  "start_downloader" -> Just StartDownloader+  "refresh_certs" -> Just RefreshCerts+  _ -> Nothing++parseProxyTest :: Map ByteString ByteString -> Maybe ProxyTest+parseProxyTest params = do+  hostname <- param "hostname"+  port <- param "port" >>= number @Int+  size <- param "testsize" >>= number+  count <- param "testcount" >>= number+  time <- param "testtime" >>= number @Int64+  testKey <- param "testkey"+  let protocol = fromMaybe "http" (param "protocol")+  pure+    ProxyTest+    { proxyTestCount = count+    , proxyTestSize  = size+    , proxyTestURL   = BSC.unpack+        $ mconcat+          [ protocol+          , "://"+          , hostname+          , ":"+          , show port+          , "/t/"+          , show size+          , "/"+          , show time+          , "/"+          , testKey+          , "/0"+          ]+    }+  where+    param name = Map.lookup name params++-- | A number as Java's Integer.parseInt reads it: digits, perhaps signed, and nothing else.+number :: Num a => ByteString -> Maybe a+number s = case BSC.readInt s of+  Just ( n, "" ) -> Just (fromIntegral n)+  _ -> Nothing
+ src/Hath/Server/App.hs view
@@ -0,0 +1,79 @@+{-# LANGUAGE DataKinds #-}++module Hath.Server.App ( ServerEnv(..), ServerAction(..), App, runApp ) where++import           Colog                    ( Message, richMessageAction )+import           Colog.Polysemy           ( Log, runLogAction )++import           Hath.Cache               ( Cache, CacheRunner(..) )+import           Hath.Config              ( ClientConfig )+import           Hath.Error               ( HathError )+import           Hath.RPC                 ( RPC, runRPC )+import           Hath.Server.FloodControl ( FloodControl )+import           Hath.Settings            ( HathSettings )+import           Hath.Stats               ( StatsEnv )++import           Network.HTTP.Types       ( Status, status500 )++import           Polysemy+import           Polysemy.Error           ( Error, errorToIOFinal, mapError )+import           Polysemy.Reader          ( Reader, runReader )++import           Relude                   hiding ( Reader, runReader )++import           UnliftIO                 ( TChan )++data ServerAction+  = Cert+    -- ^ fetch a new certificate and serve it from now on+  | GracefulShutdown+-- ^ tell the RPC server we are leaving, let open transfers finish, then return++data ServerEnv+  = ServerEnv+  { envConfig :: !ClientConfig+  , envSettings :: !(TVar HathSettings)+    -- ^ as assigned by the RPC server, and refreshed on its command+  , envActions :: !(TChan ServerAction)+  , envDownloadRequest :: !(MVar ())+    -- ^ filled by @start_downloader@ for the gallery downloader+  , envStats :: !StatsEnv+  , envCache :: !CacheRunner+  , envFloodControl :: !(Maybe FloodControl)+    -- ^ Nothing: flood control is off (@--disable-rate-limit@)+  , envTrustProxyHeaders :: !Bool+    -- ^ take the reader's address from @X-Forwarded-For@+  , envMetricsToken :: !(Maybe ByteString)+    -- ^ @HATH_METRICS_TOKEN@: who sends it may read @\/metrics@ and+    -- @\/admin\/settings@. Nothing: nobody may.+  }++-- | The effects of a request handler. It ends a request with an error by+-- throwing its status.+type App+  = Sem+    '[ Cache+     , RPC+     , Log Message+     , Reader StatsEnv+     , Reader ClientConfig+     , Error HathError+     , Error Status+     , Embed IO+     , Final IO+     ]++-- | Run a handler. The effects are interpreted afresh for every request, so+-- whatever must outlive one request lives in the 'ServerEnv'. Left is the error+-- status the handler threw, or 500 for an error it did not handle.+runApp :: ServerEnv -> App a -> IO (Either Status a)+runApp env+  = runFinal+  . embedToFinal @IO+  . errorToIOFinal @Status+  . mapError @HathError (const status500)+  . runReader (envConfig env)+  . runReader (envStats env)+  . runLogAction @IO @Message richMessageAction+  . runRPC+  . runCacheWith (envCache env)
+ src/Hath/Server/ConnectionLimit.hs view
@@ -0,0 +1,72 @@+-- | The cap on open connections, and the overload notice near it, as the+-- official client applies them to a connection it accepts+-- (HTTPServer.java:242-255).+module Hath.Server.ConnectionLimit ( Admission(..), admission, overloadDue, connectionLimit ) where++import           Control.Concurrent       ( forkIO )++import           Data.String.Interpolate  ( i )++import           GHC.Clock                ( getMonotonicTime )++import           Hath.Server.FloodControl ( isLocalIP, isRPCServer, sockAddrIP )+import           Hath.Settings            ( HathSettings(rpcServerIPs), maxConnections )++import           Network.Socket           ( SockAddr )++import           Relude++data Admission+  = Refused+  | Admitted+  | AdmittedNearLimit+    -- ^ more than 80 % of the limit is taken: time to tell the server+  deriving ( Eq, Show )++-- | Admit a connection under this limit while so many others are open: the+-- official client counts the connections open before this one, exempt ones+-- included.+admission :: Int -> Int -> Admission+admission limit open+  | open > limit = Refused+  | open * 5 > limit * 4 = AdmittedNearLimit+  | otherwise = Admitted++-- | Whether to send an overload notice at this monotonic time, given when the+-- last one went: at most one every 30 s (ServerHandler.notifyOverload).+overloadDue :: Double -> Maybe Double -> Bool+overloadDue now = maybe True (< now - 30)++-- | Warp's 'Network.Wai.Handler.Warp.setOnOpen' and+-- 'Network.Wai.Handler.Warp.setOnClose' hooks for the limit. A refused+-- connection is closed without a reply. Near the limit, the notice runs on a+-- thread of its own, so that the connection does not wait for the RPC server.+connectionLimit :: TVar HathSettings -> IO () -> IO ( SockAddr -> IO Bool, SockAddr -> IO () )+connectionLimit settingsVar notifyOverload = do+  open <- newIORef (0 :: Int)+  lastNotice <- newIORef Nothing+  let onOpen peer = do+        before <- atomicModifyIORef' open $ \n -> ( n + 1, n )+        settings <- readTVarIO settingsVar+        if exempt settings peer+          then pure True+          else case admission (maxConnections settings) before of+            Refused           -> False+              <$ putStrLn+                [i|Exceeded the maximum allowed number of incoming connections (#{maxConnections settings}).|]+            Admitted          -> pure True+            AdmittedNearLimit -> do+              now <- getMonotonicTime+              due <- atomicModifyIORef' lastNotice $ \sent -> if overloadDue now sent+                then ( Just now, True )+                else ( sent, False )+              when due $ void $ forkIO notifyOverload+              pure True+      -- Warp calls this for a refused connection as well+      onClose _ = atomicModifyIORef' open $ \n -> ( n - 1, () )+  pure ( onOpen, onClose )+  where+    -- local addresses and the RPC servers' are not limited, as in the official+    -- client; nor is a peer without an IP address (a Unix socket)+    exempt settings+      = maybe True (\ip -> isLocalIP ip || isRPCServer (rpcServerIPs settings) ip) . sockAddrIP
+ src/Hath/Server/FloodControl.hs view
@@ -0,0 +1,135 @@+-- | Flood control as the official client does it (HTTPServer.FloodControlEntry):+-- every request adds one hit, hits drain at one per second, and more than+-- 'floodLimit' hits blocks the address for 'blockSeconds'.+module Hath.Server.FloodControl+  ( IP(..)+  , sockAddrIP+  , parseIPv4+  , isRPCServer+  , isLocalIP+  , IPRecord+  , IPMap+  , registerHit+  , FloodControl+  , newFloodControl+  , checkRateLimit+  ) where++import           Data.Bits             ( (.&.), shiftR )+import qualified Data.ByteString.Char8 as BS+import qualified Data.Map.Strict       as Map++import           Network.Socket        ( HostAddress+                                       , HostAddress6+                                       , SockAddr(..)+                                       , hostAddress6ToTuple+                                       , hostAddressToTuple+                                       , tupleToHostAddress+                                       )++import           Relude++data IP = IPv4 !HostAddress | IPv6 !HostAddress6+  deriving ( Eq, Ord, Show )++sockAddrIP :: SockAddr -> Maybe IP+sockAddrIP (SockAddrInet _ address) = Just $ IPv4 address+sockAddrIP (SockAddrInet6 _ _ address _) = Just $ IPv6 address+sockAddrIP _ = Nothing++-- | @a.b.c.d@ or @::ffff:a.b.c.d@, the form @rpc_server_ip@ uses.+-- ponytail: IPv4 only; the RPC servers have no IPv6 addresses.+parseIPv4 :: ByteString -> Maybe IP+parseIPv4 s = case traverse octet (BS.split '.' v4) of+  Just [ a, b, c, d ] -> Just (IPv4 (tupleToHostAddress ( a, b, c, d )))+  _ -> Nothing+  where+    v4         = fromMaybe s (BS.stripPrefix "::ffff:" s)++    octet part = case BS.readInt (BS.strip part) of+      Just ( n, "" )+        | n >= 0 && n <= 255 -> Just (fromIntegral n)+      _ -> Nothing++-- | Whether an address is one of those @rpc_server_ip@ lists (Settings.isValidRPCServer).+-- Warp listens on IPv4 here; on a dual-stack socket the RPC servers would+-- show up as @::ffff:a.b.c.d@ and not match.+isRPCServer :: [ ByteString ] -> IP -> Bool+isRPCServer rpcServers ip = ip `elem` mapMaybe parseIPv4 rpcServers++-- | Loopback, private and link-local addresses, which the official client+-- exempts from flood control (HTTPServer.localNetworkPattern).+isLocalIP :: IP -> Bool+isLocalIP (IPv4 address) = localV4 (hostAddressToTuple address)+isLocalIP (IPv6 address) = case hostAddress6ToTuple address of+  ( 0, 0, 0, 0, 0, 0, 0, 1 )        -> True+  -- IPv4-mapped (::ffff:a.b.c.d), as seen on dual-stack sockets+  ( 0, 0, 0, 0, 0, 0xffff, hi, lo ) -> localV4 ( byte hi 8, byte hi 0, byte lo 8, byte lo 0 )+  ( w, _, _, _, _, _, _, _ )        -> (w .&. 0xfe00) == 0xfc00 -- fc00::/7+  where+    byte w n = fromIntegral (w `shiftR` n)++localV4 :: ( Word8, Word8, Word8, Word8 ) -> Bool+localV4 = \case+  ( 127, _, _, _ ) -> True+  ( 10, _, _, _ ) -> True+  ( 192, 168, _, _ ) -> True+  ( 172, b, _, _ ) -> b >= 16 && b <= 31+  ( 169, 254, _, _ ) -> True+  _ -> False++floodLimit :: Int+floodLimit = 10++blockSeconds :: Double+blockSeconds = 60++-- | Entries idle for this long carry no state worth keeping: their hits have+-- drained and any block has expired.+idleSeconds :: Double+idleSeconds = 60++-- | Times are monotonic seconds.+data IPRecord+  = IPRecord { hitCount    :: !Int+               -- ^ hits left after draining+             , lastHit     :: !Double+             , bannedUntil :: !Double+             }+  deriving ( Eq, Show )++type IPMap = Map IP IPRecord++-- | Count a request from an address at a time. The Bool says whether it may+-- proceed. A request from a blocked address is refused without being counted,+-- as in the official client.+registerHit :: Double -> IP -> IPMap -> ( IPMap, Bool )+registerHit now ip records = case Map.lookup ip live of+  Just record+    | bannedUntil record > now -> ( live, False )+  known       -> let+      previous = fromMaybe (IPRecord 0 now 0) known+      hits     = max 0 (hitCount previous - floor (now - lastHit previous)) + 1+      blocked  = hits > floodLimit+      record+        = IPRecord { hitCount    = hits+                   , lastHit     = now+                   , bannedUntil = if blocked+                       then now + blockSeconds+                       else bannedUntil previous+                   }+    in +      ( Map.insert ip record live, not blocked )+  where+    live+      | Map.size records > 1000 = Map.filter ((> now - idleSeconds) . lastHit) records+      | otherwise = records++newtype FloodControl = FloodControl (IORef IPMap)++newFloodControl :: IO FloodControl+newFloodControl = FloodControl <$> newIORef Map.empty++-- | 'registerHit' on the shared records; False if the request should be refused.+checkRateLimit :: FloodControl -> IP -> Double -> IO Bool+checkRateLimit (FloodControl records) ip now = atomicModifyIORef' records (registerHit now ip)
+ src/Hath/Server/Handlers.hs view
@@ -0,0 +1,269 @@+module Hath.Server.Handlers ( application ) where++import           Control.Concurrent       ( forkIO, threadDelay )++import           Data.ByteString.Builder  ( byteString )+import qualified Data.ByteString.Char8    as BS+import qualified Data.List                as List+import qualified Data.Map.Strict          as Map+import           Data.String.Interpolate  ( i )++import           Hath.Cache               ( CacheHit(..) )+import           Hath.Config              ( ClientConfig )+import           Hath.Download            ( fetchFile )+import           Hath.Error               ( HathError )+import           Hath.FileURI             ( FileURI(fileSize), mimeType, parseFileURI )+import           Hath.Locate              ( Located(..), locateResource )+import           Hath.Log                 ( logInfo, logWarning )+import           Hath.Protocol            ( commandTimeValid+                                          , fetchParams+                                          , keystampValid+                                          , parseURLParams+                                          , responseHeaders+                                          , sameSecret+                                          , serverCommandKey+                                          , speedTestKey+                                          )+import           Hath.RPC                 ( clientSettings )+import           Hath.Server.API          ( ProxyTest(..)+                                          , Route(..)+                                          , ServerCommand(..)+                                          , parseServerCommand+                                          , route+                                          )+import           Hath.Server.App          ( App, ServerAction(..), ServerEnv(..), runApp )+import           Hath.Server.FloodControl ( isLocalIP, isRPCServer, sockAddrIP )+import           Hath.ServerTime          ( serverNow )+import           Hath.Settings            ( HathSettings(maxAllowedFileSize, rpcServerIPs) )+import           Hath.Stats               ( addDownload, addUpload, incServed, readMetrics )++import qualified Network.HTTP.Client      as HC+import           Network.HTTP.Types       ( Header+                                          , Status+                                          , hAuthorization+                                          , hCacheControl+                                          , hContentLength+                                          , hLocation+                                          , methodGet+                                          , methodHead+                                          , status200+                                          , status301+                                          , status302+                                          , status400+                                          , status403+                                          , status404+                                          , statusCode+                                          )+import           Network.Socket           ( SockAddr )+import qualified Network.Wai              as Wai++import           Polysemy                 ( embed )+import           Polysemy.Error           ( catch, throw )+import           Polysemy.Reader          ( ask )++import           Relude                   hiding ( ask )++import           UnliftIO                 ( getMonotonicTime+                                          , replicateConcurrently+                                          , timeout+                                          , writeTChan+                                          )++-- | The client's HTTP interface. Like the official client, it reads GET and+-- HEAD requests only (HTTPSession.java:107), and answers a HEAD as the GET,+-- without the body: Warp leaves that out.+application :: ServerEnv -> Wai.Application+application env req respond+  | Wai.requestMethod req `notElem` [ methodGet, methodHead ] = respond (errorPage status400)+  | otherwise = runApp env (handle env req) >>= respond . either errorPage id++handle :: ServerEnv -> Wai.Request -> App Wai.Response+handle env req = case route (Wai.rawPathInfo req) of+  Favicon -> pure $ redirect status301 "https://e-hentai.org/favicon.ico" []+  Robots -> pure $ bytes status200 "text/plain; charset=ISO-8859-1" "User-agent: *\nDisallow: /"+  File fileId options -> resource env (Wai.requestMethod req == methodHead) fileId options+  ServerCmd command additional time commandKey+    -> serverCommand env (Wai.remoteHost req) command additional time commandKey+  Test size time testKey -> speedTest size time testKey+  Metrics -> tokenHolder >> ours <$> readMetrics+  AdminSettings -> tokenHolder >> ours . show <$> embed @IO (readTVarIO (envSettings env))+  Refuse status -> throw status+  where+    -- /metrics and /admin/settings are not the official client's, and tell+    -- what a reader has no business knowing (the RPC servers' addresses, for+    -- one). They answer whoever sends the token; with none set, they are not there.+    tokenHolder = case envMetricsToken env of+      Nothing    -> throw status404+      Just token -> unless (sameSecret sent ("Bearer " <> token)) $ throw status403++    sent        = fromMaybe "" (List.lookup hAuthorization (Wai.requestHeaders req))++    -- what they say changes, so they do not carry the official client's year+    -- of Cache-Control+    ours+      = Wai.mapResponseHeaders (filter ((/= hCacheControl) . fst))+      . bytes status200 "text/plain;charset=utf-8"+      . encodeUtf8++-- | @\/h\/\<file id>\/\<options>@: a file for a reader (HTTPResponse.java:161-228).+-- The options carry the keystamp the H\@H server issued for this file, which+-- entitles the reader to it, and the @fileindex@ and @xres@ to fetch it by.+resource :: ServerEnv -> Bool -> ByteString -> ByteString -> App Wai.Response+resource env headOnly fileId optionsText = do+  now <- embed serverNow+  cfg <- ask @ClientConfig+  unless (keystampValid cfg now fileId (fromMaybe "" (Map.lookup "keystamp" options)))+    $ throw status403+  ( uri, fetchBy )+    <- maybe (throw status404) pure ((,) <$> parseFileURI fileId <*> fetchParams options)+  let headers         = responseHeaders (mimeType uri) (fileSize uri)+      served response = response <$ unless headOnly (incServed >> addUpload (fileSize uri))+  settings <- embed @IO (readTVarIO (envSettings env))+  locateResource (maxAllowedFileSize settings) fileId uri fetchBy >>= either throw pure >>= \case+    Stored (Redirect url)+      -> pure $ redirect status302 url [ ( "Cache-Control", "public, max-age=600" ) ]+    -- Warp sets the length of a file it sends, and answers Range and If-Modified-Since+    Stored (LocalFile path) -> served+      $ Wai.responseFile status200 (filter ((/= hContentLength) . fst) headers) path Nothing+    Stored (InMemory content) -> served $ Wai.responseLBS status200 headers (fromStrict content)+    -- a cache miss: relay upstream bytes as they arrive, like the official proxy+    -- mode. Warp does not run the body of a HEAD response; the file is fetched+    -- all the same, as by the official client.+    Streamed relay -> served $ Wai.responseStream status200 headers $ \write flush -> relay+      (\chunk -> write (byteString chunk) >> flush)+  where+    options = parseURLParams optionsText++-- | @\/servercmd\/\<command>\/\<additional>\/\<time>\/\<key>@: a command from the+-- RPC server (HTTPResponse.java:230-260). Its key proves the sender: only the+-- RPC server knows ours. Like the official client, we also want it to come+-- from one of the RPC servers' addresses, as flood control sees the peer. But+-- behind a tunnel, or a proxy that does not pass addresses on, every peer is a+-- local one and the RPC server would never get through, so a local peer's+-- command is judged by its key alone.+serverCommand :: ServerEnv -> SockAddr -> Text -> Text -> Int -> Text -> App Wai.Response+serverCommand env peer command additional time commandKey = do+  settings <- embed @IO (readTVarIO (envSettings env))+  now <- embed serverNow+  cfg <- ask @ClientConfig+  unless+    (maybe False (\ip -> isLocalIP ip || isRPCServer (rpcServerIPs settings) ip) (sockAddrIP peer))+    $ throw status403+  unless+    (commandTimeValid now (fromIntegral time)+     && sameSecret+       (encodeUtf8 commandKey)+       (encodeUtf8 (serverCommandKey cfg command additional time)))+    $ throw status403+  maybe+    (pure $ text "INVALID_COMMAND")+    (runCommand env)+    (parseServerCommand command (parseURLParams (encodeUtf8 additional)))++runCommand :: ServerEnv -> ServerCommand -> App Wai.Response+runCommand env = \case+  StillAlive -> pure $ text "I feel FANTASTIC and I'm still alive"+  ThreadedProxyTest test -> text <$> runProxyTest env test+  SpeedTest size -> do+    addDownload size+    pure $ filler size+  RefreshSettings -> do+    logInfo "Refreshing settings"+    -- the official client answers the same whether the refresh worked+    refresh `catch` \(e :: HathError) -> logWarning [i|Failed to refresh settings: #{e}|]+    pure $ text ""+  StartDownloader -> do+    logInfo "Starting gallery downloader"+    void $ embed @IO $ tryPutMVar (envDownloadRequest env) ()+    pure $ text ""+  RefreshCerts -> do+    logInfo "Refreshing certificates"+    void $ embed $ forkIO $ do+      threadDelay (2 * 1000000)+      atomically $ writeTChan (envActions env) Cert+    pure $ text ""+  where+    refresh = do+      refreshed <- clientSettings =<< embed @IO (readTVarIO (envSettings env))+      embed @IO $ atomically $ writeTVar (envSettings env) refreshed++-- | @\/t\/\<size>\/\<time>\/\<key>@: a speed test, for the RPC server or for+-- another client it sent ('ThreadedProxyTest') (HTTPResponse.java:261-290).+speedTest :: Int -> Int64 -> ByteString -> App Wai.Response+speedTest size time testKey = do+  now <- embed serverNow+  cfg <- ask @ClientConfig+  unless (commandTimeValid now time && sameSecret testKey (speedTestKey cfg size time))+    $ throw status403+  addUpload size+  pure $ filler size++bytes :: Status -> ByteString -> ByteString -> Wai.Response+bytes status contentType content+  = Wai.responseLBS status (responseHeaders contentType (BS.length content)) (fromStrict content)++-- | A reply of the official client's HTTPResponseProcessorText.+text :: ByteString -> Wai.Response+text = bytes status200 textType++-- | The page the official client answers an error with (HTTPResponse.getHTTPResponseProcessor).+errorPage :: Status -> Wai.Response+errorPage status = bytes status textType [i|An error has occurred. (#{statusCode status})|]++redirect :: Status -> ByteString -> [ Header ] -> Wai.Response+redirect status url headers+  = Wai.responseLBS status (( hLocation, url ) : headers <> responseHeaders textType 0) ""++-- | What the official client's text replies say they are; its other replies+-- without a file type spell it in lower case ('filler').+textType :: ByteString+textType = "text/html; charset=ISO-8859-1"++-- | @size@ filler bytes, produced one TCP segment's worth at a time+-- (Settings.TCP_PACKET_SIZE), as HTTPResponseProcessorSpeedtest sends them.+filler :: Int -> Wai.Response+filler size+  = Wai.responseStream status200 (responseHeaders "text/html; charset=iso-8859-1" (max 0 size))+  $ \write _ -> let+      go n = when (n > 0) $ write (byteString (BS.take n segment)) >> go (n - BS.length segment)+    in +      go size+  where+    segment = BS.replicate 1460 '6'++-- | Run the downloads concurrently and report, in the official format,+-- @OK:\<successful tests>-\<total milliseconds of the successful tests>@.+runProxyTest :: ServerEnv -> ProxyTest -> App ByteString+runProxyTest env test = case HC.parseRequest (proxyTestURL test) of+  Nothing      -> pure "INVALID_COMMAND"+  Just request -> do+    settings <- embed @IO (readTVarIO (envSettings env))+    results <- embed+      $ replicateConcurrently+        (proxyTestCount test)+        (timedDownload (maxAllowedFileSize settings) (proxyTestSize test) request)+    let succeeded = catMaybes results+    addUpload (proxyTestCount test * fromIntegral (proxyTestSize test))+    pure [i|OK:#{length succeeded}-#{sum succeeded}|]++-- | A cap on one test download, its attempts included. The official client+-- hands FileDownloader 60 s (HTTPResponse.java:105), which it never enforces;+-- without a cap a target that trickles could hold the RPC server's request for minutes.+testTimeoutMicros :: Int+testTimeoutMicros = 60 * 1000000++-- | The milliseconds a test download took, from its first byte to its end, as+-- FileDownloader times it (FileDownloader.java:130-315): the first byte is the+-- first any of its attempts received. It counts if a body of the announced+-- length arrived, at least the test size (HTTPResponse.java:112).+timedDownload :: Int -> Int64 -> HC.Request -> IO (Maybe Int64)+timedDownload maxSize testSize request = do+  firstByte <- newIORef Nothing+  fetched <- timeout testTimeoutMicros $ fetchFile maxSize request $ \_ _+    -> whenNothingM_ (readIORef firstByte) (writeIORef firstByte . Just =<< getMonotonicTime)+  end <- getMonotonicTime+  start <- readIORef firstByte+  pure $ case fetched of+    Just (Right size)+      | fromIntegral size >= testSize -> Just $ maybe 0 (\t -> truncate ((end - t) * 1000)) start+    _ -> Nothing
+ src/Hath/Server/LegacyCiphers.hs view
@@ -0,0 +1,154 @@+{-# LANGUAGE AllowAmbiguousTypes #-}++-- |+-- Module      : LegacyCiphers+-- License     : BSD-3-Clause+-- Maintainer  : Vincent Hanquez+-- Stability   : experimental+-- Portability : unknown+--+-- This module contains legacy cipher definitions borrowed from the+-- tls package (version 1.9.0) by Vincent Hanquez.+--+-- Original source: https://hackage.haskell.org/package/tls-1.9.0/docs/src/Network.TLS.Extra.Cipher.html+--+-- Copyright (c) Vincent Hanquez+--+-- Redistribution and use in source and binary forms, with or without+-- modification, are permitted provided that the following conditions are met:+--+-- 1. Redistributions of source code must retain the above copyright notice,+--    this list of conditions and the following disclaimer.+--+-- 2. Redistributions in binary form must reproduce the above copyright notice,+--    this list of conditions and the following disclaimer in the documentation+--    and/or other materials provided with the distribution.+--+-- 3. Neither the name of the copyright holder nor the names of its contributors+--    may be used to endorse or promote products derived from this software without+--    specific prior written permission.+--+-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+-- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+-- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE+-- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR+-- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF+-- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS+-- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN+-- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)+-- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE+-- POSSIBILITY OF SUCH DAMAGE.+--+-- This file is part of the hs-hath project, which is licensed under GPL-3.0.+-- The BSD-3-Clause license applies to this specific file as noted above.+--+-- Why it exists: the official Java client accepts these ECDHE CBC suites (the+-- JVM's defaults include them), so we offer them too, to keep the same cipher+-- set. tls 2.x dropped CBC suites, which is why they are defined here.+module Hath.Server.LegacyCiphers+  ( cipher_ECDHE_ECDSA_AES128CBC_SHA+  , cipher_ECDHE_RSA_AES128CBC_SHA+  , cipher_ECDHE_RSA_AES256CBC_SHA+  ) where++import           Crypto.Cipher.AES   ( AES128, AES256 )+import           Crypto.Cipher.Types ( BlockCipher+                                     , IV+                                     , cbcDecrypt+                                     , cbcEncrypt+                                     , cipherInit+                                     , makeIV+                                     )+import           Crypto.Error        ( CryptoFailable, throwCryptoError )++import qualified Data.ByteString     as B++import           Network.TLS         ( Version(..) )+import           Network.TLS.Cipher  ( Bulk(..)+                                     , BulkBlock+                                     , BulkDirection(..)+                                     , BulkFunctions(..)+                                     , BulkKey+                                     , Cipher(..)+                                     , CipherKeyExchangeType(..)+                                     , Hash(..)+                                     )++import           Relude++cipher_ECDHE_ECDSA_AES128CBC_SHA :: Cipher+cipher_ECDHE_ECDSA_AES128CBC_SHA+  = Cipher { cipherID          = 0xC009+           , cipherName        = "ECDHE-ECDSA-AES128CBC-SHA"+           , cipherBulk        = bulk_aes128+           , cipherHash        = SHA1+           , cipherPRFHash     = Nothing+           , cipherKeyExchange = CipherKeyExchange_ECDHE_ECDSA+           , cipherMinVer      = Just TLS10+           }++cipher_ECDHE_RSA_AES128CBC_SHA :: Cipher+cipher_ECDHE_RSA_AES128CBC_SHA+  = Cipher { cipherID          = 0xC013+           , cipherName        = "ECDHE-RSA-AES128CBC-SHA"+           , cipherBulk        = bulk_aes128+           , cipherHash        = SHA1+           , cipherPRFHash     = Nothing+           , cipherKeyExchange = CipherKeyExchange_ECDHE_RSA+           , cipherMinVer      = Just TLS10+           }++cipher_ECDHE_RSA_AES256CBC_SHA :: Cipher+cipher_ECDHE_RSA_AES256CBC_SHA+  = Cipher { cipherID          = 0xC014+           , cipherName        = "ECDHE-RSA-AES256CBC-SHA"+           , cipherBulk        = bulk_aes256+           , cipherHash        = SHA1+           , cipherPRFHash     = Nothing+           , cipherKeyExchange = CipherKeyExchange_ECDHE_RSA+           , cipherMinVer      = Just TLS10+           }++bulk_aes128 :: Bulk+bulk_aes128+  = Bulk { bulkName       = "AES128"+         , bulkKeySize    = 16+         , bulkIVSize     = 16+         , bulkExplicitIV = 0+         , bulkAuthTagLen = 0+         , bulkBlockSize  = 16+         , bulkF          = BulkBlockF (cbc @AES128)+         }++bulk_aes256 :: Bulk+bulk_aes256+  = Bulk { bulkName       = "AES256"+         , bulkKeySize    = 32+         , bulkIVSize     = 16+         , bulkExplicitIV = 0+         , bulkAuthTagLen = 0+         , bulkBlockSize  = 16+         , bulkF          = BulkBlockF (cbc @AES256)+         }++noFail :: CryptoFailable a -> a+noFail = throwCryptoError++makeIV_ :: BlockCipher a => B.ByteString -> IV a+makeIV_ = fromMaybe (error "makeIV_") . makeIV++takelast :: Int -> B.ByteString -> B.ByteString+takelast i b = B.drop (B.length b - i) b++-- | AES-CBC as TLS 1.0-1.2 use it: the IV of the next record is the last+-- ciphertext block. The context is set up once per key, not per record.+cbc :: forall c. BlockCipher c => BulkDirection -> BulkKey -> BulkBlock+cbc direction key = case direction of+  BulkEncrypt -> \iv input -> let+      output = cbcEncrypt ctx (makeIV_ iv) input+    in +      ( output, takelast 16 output )+  BulkDecrypt -> \iv input -> ( cbcDecrypt ctx (makeIV_ iv) input, takelast 16 input )+  where+    ctx = noFail (cipherInit key) :: c
+ src/Hath/Server/Middleware.hs view
@@ -0,0 +1,52 @@+module Hath.Server.Middleware ( rateLimitMiddleware, tracingConnections, tracingTimeUsage ) where++import           Control.Exception        ( finally )++import           GHC.Clock                ( getMonotonicTime )++import           Hath.Server.FloodControl ( FloodControl+                                          , IP+                                          , checkRateLimit+                                          , isLocalIP+                                          , sockAddrIP+                                          )+import           Hath.Stats               ( StatsEnv, observeDuration, trackConnection )++import           Network.HTTP.Types       ( mkStatus )+import           Network.Wai              ( Middleware, Response, remoteHost, responseLBS )++import           Relude++-- | Flood control, for a request to any path: the official client counts+-- every connection. Local addresses and those the predicate exempts (the RPC+-- servers, as in the official client) are never limited.+rateLimitMiddleware :: FloodControl -> (IP -> IO Bool) -> Middleware+rateLimitMiddleware floodControl exempt app req respond = case sockAddrIP (remoteHost req) of+  -- a peer without an IP address (a Unix socket) is not limited+  Nothing -> app req respond+  Just ip+    | isLocalIP ip -> app req respond+    | otherwise -> do+      now <- getMonotonicTime+      allowed <- ifM (exempt ip) (pure True) (checkRateLimit floodControl ip now)+      if allowed+        then app req respond+        else respond tooManyRequests++tooManyRequests :: Response+tooManyRequests+  = responseLBS+    (mkStatus 429 "Too Many Requests")+    [ ( "Content-Type", "text/plain" ) ]+    "Too Many Requests"++tracingConnections :: StatsEnv -> Middleware+tracingConnections stats app req respond = trackConnection stats (app req respond)++-- | Record how long each request took, the body included.+tracingTimeUsage :: StatsEnv -> Middleware+tracingTimeUsage stats app req respond = do+  start <- getMonotonicTime+  app req respond `finally` do+    end <- getMonotonicTime+    observeDuration stats (end - start)
+ src/Hath/ServerTime.hs view
@@ -0,0 +1,29 @@+-- | The RPC server's clock. Keystamps, server commands, speed tests and the+-- signatures of our own RPC requests all carry its time, so we sign and check+-- with it rather than ours (official Settings.getServerTime). As the official+-- client does, we keep its offset from our clock in one global: it is read+-- wherever a time is signed or checked, and set from whichever reply carries+-- @server_time@.+module Hath.ServerTime ( serverNow, setServerTime ) where++import           Data.Time.Clock.System ( SystemTime(systemSeconds), getSystemTime )++import           Relude++import           System.IO.Unsafe       ( unsafePerformIO )++-- | Seconds to add to our clock to get the server's.+offset :: IORef Int64+offset = unsafePerformIO (newIORef 0)++{-# NOINLINE offset #-}++-- | The server's time in Unix seconds: ours until a reply has told us its own.+serverNow :: IO Int64+serverNow = (+) <$> readIORef offset <*> ourNow++setServerTime :: Int64 -> IO ()+setServerTime serverTime = ourNow >>= atomicWriteIORef offset . (serverTime -)++ourNow :: IO Int64+ourNow = systemSeconds <$> getSystemTime
+ src/Hath/Settings.hs view
@@ -0,0 +1,86 @@+-- | The settings the RPC server assigns to this client.+module Hath.Settings+  ( HathSettings(..)+  , defaultHathSettings+  , parseSettings+  , settingNumber+  , maxConnections+  ) where++import qualified Data.ByteString.Char8 as BS++import           Relude++data HathSettings+  = HathSettings+  { clientHost :: !ByteString+  , clientPort :: !Int+  , throttleBytes :: !Int64+    -- ^ Bandwidth limit. Nothing enforces it; it sets 'maxConnections'.+  , diskLimitBytes :: !Int64+    -- ^ Cache size set on the H\@H settings page; the Filesystem backend evicts to stay under it.+  , diskRemainingBytes :: !Int64+  , rpcServerIPs :: ![ ByteString ]+    -- ^ As sent by the server, e.g. @::ffff:192.0.2.10@. These are exempt from flood control.+  , maxConnectionsOverride :: !Int+    -- ^ @max_connections@: when above 0, 'maxConnections' is this.+  , maxAllowedFileSize :: !Int+    -- ^ @max_allowed_filesize@: a cache miss or download larger than this is not fetched.+  , maxFilenameLength :: !Int+    -- ^ @max_filename_length@: how long the name of a gallery's directory may get.+  }+  deriving ( Show )++defaultHathSettings :: HathSettings+defaultHathSettings+  = HathSettings+  { clientHost = ""+  , clientPort = 0+  , throttleBytes = 0+  , diskLimitBytes = 0+  , diskRemainingBytes = 0+  , rpcServerIPs = []+  , maxConnectionsOverride = 0+    -- the official client's defaults (Settings.java:58)+  , maxAllowedFileSize = 1073741824+  , maxFilenameLength = 125+  }++-- | Apply @name=value@ lines onto the settings we have. @client_settings@ only+-- carries what changed, so a refresh must leave the other fields alone. Lines+-- we do not know, or whose number does not parse, are skipped.+parseSettings :: HathSettings -> [ ByteString ] -> HathSettings+parseSettings = foldl' apply+  where+    apply settings line = case ( name, readMaybe @Int64 (BS.unpack value) ) of+      ( "host", _ ) -> settings { clientHost = value }+      ( "port", Just n ) -> settings { clientPort = fromIntegral n }+      ( "throttle_bytes", Just n ) -> settings { throttleBytes = n }+      ( "disklimit_bytes", Just n ) -> settings { diskLimitBytes = n }+      ( "diskremaining_bytes", Just n ) -> settings { diskRemainingBytes = n }+      ( "rpc_server_ip", _ ) -> settings { rpcServerIPs = BS.split ';' value }+      ( "max_connections", Just n ) -> settings { maxConnectionsOverride = fromIntegral n }+      ( "max_allowed_filesize", Just n ) -> settings { maxAllowedFileSize = fromIntegral n }+      ( "max_filename_length", Just n ) -> settings { maxFilenameLength = fromIntegral n }+      _ -> settings+      where+        ( name, value ) = nameValue line++-- | How many connections we take at once (Settings.getMaxConnections).+maxConnections :: HathSettings -> Int+maxConnections settings+  | maxConnectionsOverride settings > 0 = maxConnectionsOverride settings+  | otherwise = 20 + fromIntegral (min 480 (throttleBytes settings `quot` 10000))++-- | The number one of a reply's @name=value@ lines gives, such as @server_time@.+settingNumber :: ByteString -> [ ByteString ] -> Maybe Int64+settingNumber wanted reply+  = listToMaybe+    [ n+    | ( name, value ) <- map nameValue reply+    , name == wanted+    , Just n <- [ readMaybe (BS.unpack value) ]+    ]++nameValue :: ByteString -> ( ByteString, ByteString )+nameValue = second (BS.drop 1) . BS.span (/= '=')
+ src/Hath/Stats.hs view
@@ -0,0 +1,160 @@+{-# LANGUAGE DataKinds #-}++-- | The counters behind @\/metrics@, and their rendering in the Prometheus+-- text format.+module Hath.Stats+  ( StatsEnv+  , newStatsEnv+  , addUpload+  , addDownload+  , incServed+  , incFetched+  , incDlTask+  , incDlFile+  , addDlBytes+  , trackConnection+  , observeDuration+  , readMetrics+  , renderMetrics+  ) where++import           Control.Exception ( finally )++import           GHC.Clock         ( getMonotonicTime )++import           Numeric           ( showFFloat )++import           Polysemy+import           Polysemy.Reader   ( Reader, ask )++import           Relude            hiding ( Reader, ask )++data StatsEnv+  = StatsEnv+  { statsStart :: !Double+    -- ^ monotonic seconds; the uptime counts from here+  , statsSentBytes :: !(IORef Int64)+  , statsReceivedBytes :: !(IORef Int64)+  , statsSent :: !(IORef Int64)+  , statsReceived :: !(IORef Int64)+  , statsActiveConnections :: !(IORef Int64)+  , statsDlTasks :: !(IORef Int64)+  , statsDlFiles :: !(IORef Int64)+  , statsDlBytes :: !(IORef Int64)+  , statsDurations :: ![ IORef Int64 ]+    -- ^ requests per 'durationBounds' bucket, then those above the last bound (not cumulative)+  , statsDurationSum :: !(IORef Double)+  }++-- | Upper bounds of the request duration buckets, in seconds. A duration+-- includes sending the body, so large files land in the upper buckets.+durationBounds :: [ Double ]+durationBounds = [ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60 ]++newStatsEnv :: IO StatsEnv+newStatsEnv = do+  start <- getMonotonicTime+  let counter = newIORef 0+  StatsEnv start <$> counter+    <*> counter+    <*> counter+    <*> counter+    <*> counter+    <*> counter+    <*> counter+    <*> counter+    <*> replicateM (length durationBounds + 1) counter+    <*> newIORef 0++add :: Int64 -> IORef Int64 -> IO ()+add n ref = atomicModifyIORef' ref $ \old -> ( old + n, () )++count :: Members '[ Reader StatsEnv, Embed IO ] r => (StatsEnv -> IORef Int64) -> Int -> Sem r ()+count counter n = ask >>= embed . add (fromIntegral n) . counter++addUpload :: Members '[ Reader StatsEnv, Embed IO ] r => Int -> Sem r ()+addUpload = count statsSentBytes++addDownload :: Members '[ Reader StatsEnv, Embed IO ] r => Int -> Sem r ()+addDownload = count statsReceivedBytes++incServed :: Members '[ Reader StatsEnv, Embed IO ] r => Sem r ()+incServed = count statsSent 1++incFetched :: Members '[ Reader StatsEnv, Embed IO ] r => Sem r ()+incFetched = count statsReceived 1++incDlTask :: Members '[ Reader StatsEnv, Embed IO ] r => Sem r ()+incDlTask = count statsDlTasks 1++incDlFile :: Members '[ Reader StatsEnv, Embed IO ] r => Sem r ()+incDlFile = count statsDlFiles 1++addDlBytes :: Members '[ Reader StatsEnv, Embed IO ] r => Int -> Sem r ()+addDlBytes = count statsDlBytes++trackConnection :: StatsEnv -> IO a -> IO a+trackConnection stats action = do+  add 1 (statsActiveConnections stats)+  action `finally` add (-1) (statsActiveConnections stats)++observeDuration :: StatsEnv -> Double -> IO ()+observeDuration stats seconds = do+  whenJust (statsDurations stats !!? length (takeWhile (< seconds) durationBounds)) (add 1)+  atomicModifyIORef' (statsDurationSum stats) $ \total -> ( total + seconds, () )++readMetrics :: Members '[ Reader StatsEnv, Embed IO ] r => Sem r Text+readMetrics = do+  stats <- ask+  embed $ do+    now <- getMonotonicTime+    renderMetrics stats (round (now - statsStart stats))++-- | The metrics in the Prometheus text format, given the uptime in seconds.+-- They come in the order earlier versions printed them.+renderMetrics :: StatsEnv -> Int64 -> IO Text+renderMetrics stats uptime = do+  let counter name help field = scalar "counter" name help <$> readIORef (field stats)+  scalars <- sequence+    [ counter "hath_download_size_bytes_total" "Total bytes downloaded from galleries" statsDlBytes+    , counter "hath_download_file_count_total" "Total gallery files downloaded" statsDlFiles+    , counter "hath_download_task_count_total" "Total gallery download tasks" statsDlTasks+    , counter "hath_cache_received_total" "Total number of cache misses" statsReceived+    , counter "hath_cache_sent_total" "Total number of cache hits" statsSent+    , counter+        "hath_cache_received_bytes_total"+        "Total bytes received into cache"+        statsReceivedBytes+    , counter "hath_cache_sent_bytes_total" "Total bytes sent from cache" statsSentBytes+    , pure $ scalar "gauge" "hath_uptime_seconds" "Server uptime in seconds" uptime+    , scalar "gauge" "hath_active_connections" "Number of active connections"+      <$> readIORef (statsActiveConnections stats)+    ]+  durations <- traverse readIORef (statsDurations stats)+  total <- readIORef (statsDurationSum stats)+  pure $ mconcat scalars <> durationHistogram durations total++scalar :: Text -> Text -> Text -> Int64 -> Text+scalar typ name help value+  = unlines+    [ "# HELP " <> name <> " " <> help+    , "# TYPE " <> name <> " " <> typ+    , name <> " " <> show value+    ]++-- | The request duration histogram, from the count of each bucket: cumulative+-- counts per upper bound (le is inclusive), then the sum and the count.+durationHistogram :: [ Int64 ] -> Double -> Text+durationHistogram counts total+  = unlines+  $ [ "# HELP " <> name <> " Time from request to the last byte sent"+    , "# TYPE " <> name <> " histogram"+    ]+  <> zipWith bucket (map decimal durationBounds <> [ "+Inf" ]) (scanl1 (+) counts)+  <> [ name <> "_sum " <> decimal total, name <> "_count " <> show (sum counts) ]+  where+    name        = "hath_request_duration_seconds"++    bucket le n = name <> "_bucket{le=\"" <> le <> "\"} " <> show n++    decimal x = toText (showFFloat Nothing x "")
+ src/Hath/Version.hs view
@@ -0,0 +1,40 @@+{-# LANGUAGE TemplateHaskell #-}++-- | What this binary was built from, fixed when this module is compiled.+module Hath.Version ( versionInfo ) where++import           Control.Exception   ( catch )++import qualified Data.Text           as T+import           Data.Time.Clock     ( getCurrentTime )+import           Data.Time.Format    ( defaultTimeLocale, formatTime )++import           Language.Haskell.TH ( runIO, stringE )++import           Relude++import           System.Process      ( readProcess )++versionInfo :: Text+versionInfo+  = unlines+    [ "hs-hath " <> commit <> " (" <> date <> ")"+    , "Build Commit: " <> commit+    , "Build Date:   " <> date+    ]+  where+    -- "unknown" when the build does not run in a git checkout+    commit :: Text+    commit+      = T.strip+        $( runIO+          (readProcess "git" [ "rev-parse", "--short=7", "HEAD" ] "" `catch` \(_ :: SomeException)+           -> pure "unknown")+        >>= stringE+        )++    date :: Text+    date+      = $( runIO (formatTime defaultTimeLocale "%Y-%m-%d %H:%M:%S UTC" <$> getCurrentTime)+        >>= stringE+        )
− src/HathNetwork/Genesis.hs
@@ -1,60 +0,0 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE TemplateHaskell #-}--module HathNetwork.Genesis-  ( Genesis(..)-  , fetchSettings-  , fetchCertificate-  , runGenesis-  , runGenesisIO-  ) where--import           Data.X509          ( CertificateChain, PrivKey )--import           Interface.API--import           Polysemy-import           Polysemy.Error     ( Error, errorToIOFinal )-import           Polysemy.Operators-import           Polysemy.Reader    ( Reader, runReader )--import           Relude             hiding ( Reader, runReader )--import           Servant.Client     ( ClientError )--import           Types--data Genesis m a where-  FetchSettings :: Genesis m HathSettings-  FetchCertificate :: Genesis m ( CertificateChain, PrivKey )--makeSem ''Genesis--{-# INLINE runGenesis #-}-runGenesis :: Members '[ Embed IO, Error RPCError, Reader ClientConfig, EHentaiAPI ] r-           => Genesis : r @> a-           -> Sem r a-runGenesis = interpret $ \case-  FetchSettings    -> getSettings-  FetchCertificate -> downloadCertificates--{-# INLINE runGenesisIO #-}-runGenesisIO-  :: ClientConfig-  -> [ Genesis-     , EHentaiAPI-     , Reader ClientConfig-     , Error ClientError-     , Error RPCError-     , Embed IO-     , Final IO-     ] @> a-  -> IO (Either RPCError (Either ClientError a))-runGenesisIO cfg-  = runFinal-  . embedToFinal @IO-  . errorToIOFinal @RPCError-  . errorToIOFinal @ClientError-  . runReader cfg-  . runEHentaiAPI-  . runGenesis
− src/HathNetwork/RPC.hs
@@ -1,113 +0,0 @@-{-# LANGUAGE DataKinds #-}--{-# LANGUAGE DeriveGeneric #-}--{-# LANGUAGE PatternSynonyms #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE TemplateHaskell #-}--module HathNetwork.RPC-  ( RPCResponse(..)-  , parseRPCResponse-  , getPayload-  , RPC(..)-  , serverStat-  , stillAlive-  , clientStart-  , clientStop-  , clientLogin-  , checkGalleryTask-  , notifyGalleryCompletion-  , fetchGalleryFile-  , runRPC-  , runRPCIO-  ) where--import           Colog              ( Message, Severity(Info), richMessageAction )-import           Colog.Polysemy     ( Log, runLogAction )--import           Interface.API      hiding ( StillAlive )--import           Polysemy-import           Polysemy.Error     ( Error, errorToIOFinal )-import           Polysemy.Operators-import           Polysemy.Reader    ( Reader, runReader )--import           Relude             hiding ( Reader, ask, runReader )--import           Servant.Client     ( ClientError )--import           Types--import           Utils              ( log )--data RPC m a where-  -- | Test if remote server is running-  ServerStat :: RPC m Bool-  -- | Heartbeat-  StillAlive :: RPC m ()-  -- | Notify server that client is ready to start a session-  ClientStart :: RPC m Bool-  -- | Notify server that client is stopping a session-  ClientStop :: RPC m Bool-  -- | Login to remote server-  ClientLogin :: RPC m HathSettings-  -- | Check if there is a gallery task to download-  CheckGalleryTask :: RPC m (Maybe GalleryMetadata)-  -- | Notify server that a gallery task has been completed-  NotifyGalleryCompletion :: GalleryMetadata -> RPC m ()-  -- | Fetch gallery files-  FetchGalleryFile :: ( GalleryMetadata, [ GalleryFile ] )-    -> RPC m [ ( GalleryFile, ByteString ) ] -- ^ Successfully downloaded files--makeSem ''RPC--runRPC :: forall a r.-       Members '[ Embed IO, Error RPCError, Reader ClientConfig, EHentaiAPI, Log Message ] r-       => RPC : r @> a-       -> r @> a-runRPC = interpret $ \case-  ServerStat -> do-    b <- checkServerStatus-    log Info $ "Server availablity: " <> show b-    return b-  StillAlive -> heartbeat-  ClientStart -> log Info "Trying to start a session" >> startListening-  ClientStop -> log Info "Stopping a session" >> stopListening-  ClientLogin -> log Info "Logging in" >> login-  CheckGalleryTask -> log Info "Checking for a gallery task" >> (rightToMaybe <$> nextGalleryTask)-  NotifyGalleryCompletion-    metadata -> log Info "Notifying server that a gallery task has been completed"-    >> completeGalleryTask metadata-  FetchGalleryFile-    ( metadata, files ) -> log Info "Fetching gallery files" >> phi metadata [] files-  where-    phi _ acc [] = pure acc-    phi metadata acc (x : xs) = do-      maybeContent <- downloadGalleryFile metadata x-      case maybeContent of-        Just content -> phi metadata (( x, content ) : acc) xs-        Nothing      -> phi metadata acc xs--{-# INLINE runRPCIO #-}-runRPCIO :: ClientConfig-         -> [ RPC-            , EHentaiAPI-            , Log Message-            , Reader ClientConfig-            , Embed IO-            , Error ClientError-            , Error RPCError-            , Final IO-            ] @> a-         -> IO (Either RPCError (Either ClientError a))-runRPCIO cfg-  = runFinal-  . errorToIOFinal @RPCError-  . errorToIOFinal @ClientError-  . embedToFinal @IO-  . runReader cfg-  . runLogAction @IO richMessageAction-  . runEHentaiAPI-  . runRPC-
− src/Interface/API.hs
@@ -1,474 +0,0 @@-{-# LANGUAGE DataKinds #-}--{-# LANGUAGE RecordWildCards #-}--{-# LANGUAGE TemplateHaskell #-}-{-# LANGUAGE TypeOperators #-}--module Interface.API-  (   -- * H@H Client Exposed API-    API-  , api-  , ServerCommand(..)-    -- * EHentai RPC-  , EHentaiAPI(..)-  , ehRPC-  , RPCParams(..)-  , emptyRPCParams-  , runEHentaiAPI-  , checkServerStatus-  , heartbeat-  , startListening-  , stopListening-  , login-  , downloadCertificates-  , getSettings-  , nextGalleryTask-  , completeGalleryTask-  , downloadGalleryFile-  , reportFailures-  , openResource-  , fetchBlacklist-  , runEHentaiAPIIO-  , renderServerCommand-  ) where--import           Crypto.Store.PKCS12      ( readP12FileFromMemory-                                          , recover-                                          , recoverAuthenticated-                                          , toCredential-                                          , toProtectionPassword-                                          )--import qualified Data.ByteString.Char8    as BSC-import qualified Data.ByteString.Lazy     as LBS-import qualified Data.List                as List-import           Data.String.Interpolate  ( i )-import qualified Data.Text                as T-import           Data.Time.Clock.System   ( SystemTime(systemSeconds), getSystemTime )-import           Data.X509                ( CertificateChain, PrivKey )--import qualified Mason.Builder            as BD--import           Network.HTTP.Client      ( BodyReader-                                          , HttpException-                                          , Manager-                                          , Request(responseTimeout, host, requestHeaders)-                                          , Response(responseBody, responseHeaders, responseStatus)-                                          , responseClose-                                          , responseOpen-                                          , defaultManagerSettings-                                          , newManager-                                          , parseRequest-                                          , responseTimeoutMicro-                                          )-import           Network.HTTP.Media       ( (//), (/:) )-import           Network.HTTP.Client.TLS  ( getGlobalManager )-import           Network.HTTP.Simple      ( httpLbs )-import           Network.HTTP.Types       ( hContentLength )-import qualified Network.HTTP.Types       as HT--import           Polysemy-import           Polysemy.Error           ( Error, errorToIOFinal, throw )-import           Polysemy.Operators-import           Polysemy.Reader          ( Reader, ask, runReader )-import           Polysemy.State           ( modify, runState )--import           Relude                   hiding ( Reader-                                                 , State-                                                 , ask-                                                 , modify-                                                 , runReader-                                                 , runState-                                                 )--import           Servant                  hiding ( addHeader )-import           Servant.API.ContentTypes ( AllCTRender(handleAcceptH) )-import           Servant.Client           ( BaseUrl(BaseUrl)-                                          , ClientEnv(makeClientRequest)-                                          , ClientError-                                          , ClientM-                                          , Scheme(Http)-                                          , client-                                          , defaultMakeClientRequest-                                          , mkClientEnv-                                          , runClientM-                                          )-import           Servant.Client.Core      ( addHeader )--import           System.IO.Unsafe         ( unsafePerformIO )--import           Types                    ( ClientConfig-                                          , ClientConfig(..)-                                          , FileURI(..)-                                          , GalleryFile(..)-                                          , GalleryMetadata(galleryMinXRes, galleryID)-                                          , HathSettings-                                          , RPCError(CertificateFailure)-                                          , RPCResponse(statusCode)-                                          , clientBuild-                                          , hentaiHeader-                                          , parseMetadata-                                          , parseRPCResponse-                                          , parseRPCResponse'-                                          , parseSettings-                                          , renderFileURIBytes-                                          , verifyRPCResponse-                                          )--import           UnliftIO                 ( try )--import           Utils                    ( firstJustM, hash )---- | Available server commands-data ServerCommand-  = StillAlive        -- ^ Heartbeat request-  | ThreadedProxyTest -- ^ Multi-threaded speed test-  | SpeedTest        -- ^ Speed test-  | RefreshSettings  -- ^ Reloads hath configuration-  | StartDownloader  -- ^ Initiates gallery downloader-  | RefreshCerts     -- ^ Updates SSL certificates-  deriving ( Show, Eq )---- Custom type conversion for ServerCommand-serverCommandNames :: [ ( Text, ServerCommand ) ]-serverCommandNames-  = [ ( "still_alive", StillAlive )-    , ( "threaded_proxy_test", ThreadedProxyTest )-    , ( "speed_test", SpeedTest )-    , ( "refresh_settings", RefreshSettings )-    , ( "start_downloader", StartDownloader )-    , ( "refresh_certs", RefreshCerts )-    ]--instance FromHttpApiData ServerCommand where-  parseUrlPiece cmd = maybeToRight ("Invalid command: " <> cmd) (List.lookup cmd serverCommandNames)---- | Wire name of a command, needed to verify the servercmd signature.-renderServerCommand :: ServerCommand -> Text-renderServerCommand c = maybe "" fst (find ((== c) . snd) serverCommandNames)--data SpeedTest-  deriving ( Typeable )--instance MimeRender SpeedTest ByteString where-  {-# INLINE mimeRender #-}-  mimeRender _ = LBS.fromStrict--instance Accept SpeedTest where-  {-# INLINE contentTypes #-}-  contentTypes _-    = "text" // "html" /: ( "charset", "iso-8859-1" )-    :| [ "image" // "gif", "image" // "jpeg", "*" // "*" /: ( "q", ".2" ) ]---- floskell-disable--- API type definitions-type API-    =    "favicon.ico" :> Get '[PlainText] NoContent-    :<|> "robots.txt" :> Get '[PlainText] Text-    :<|> "h"-        :> Capture "info" Text-        :> Capture "opts" Text-        :> Capture "filename" Text-        :> RawM-    :<|> "metrics" :> Get '[PlainText] Text-    :<|> "servercmd"-        :> Capture "command" ServerCommand-        :> Capture "additional" Text-        :> Capture "time" Int-        :> Capture "key" Text-        :> StreamGet NoFraming OctetStream (Headers '[Header "Content-Length" Int] (SourceIO ByteString))-    :<|> "t"-        :> Capture "testsize" Int-        :> Capture "testtime" Int64-        :> Capture "testkey" Text-        :> Capture "nothing" Text-        :> StreamGet NoFraming SpeedTest (Headers '[Header "Content-Length" Int] (SourceIO ByteString))-    :<|> "admin"-        :> "settings"-        :> Get '[PlainText] Text-    :<|> Raw  -- This will catch all other routes including HEAD requests--- floskell-enable---- API specification-api :: Proxy API-api = Proxy---- floskell-disable-type EHAPI =-       QueryParam "act" Text-  :>   QueryParam "cid" Text-  :>   QueryParam "add" Text-  :>   QueryParam "acttime" Text-  :>   QueryParam "actkey" Text-  :>   QueryParam "clientbuild" Text-  :>   Get '[PlainText,OctetStream] ByteString--- floskell-enable--instance MimeUnrender PlainText ByteString where-  {-# INLINE mimeUnrender #-}-  mimeUnrender _ = mimeUnrender (Proxy @OctetStream)--data RPCParams-  = RPCParams { act         :: {-# UNPACK #-} !(Maybe Text)-              , cid         :: {-# UNPACK #-} !(Maybe Text)-              , add         :: {-# UNPACK #-} !(Maybe Text)-              , acttime     :: {-# UNPACK #-} !(Maybe Text)-              , actkey      :: {-# UNPACK #-} !(Maybe Text)-              , clientbuild :: {-# UNPACK #-} !(Maybe Text)-              }--emptyRPCParams :: RPCParams-emptyRPCParams-  = RPCParams { act         = Nothing-              , cid         = Nothing-              , add         = Nothing-              , acttime     = Nothing-              , actkey      = Nothing-              , clientbuild = Nothing-              }--ehAPI :: Proxy EHAPI-ehAPI = Proxy--ehAPIM :: RPCParams -> ClientM ByteString-ehAPIM RPCParams { .. } = client ehAPI act cid add acttime actkey clientbuild--data EHentaiAPI m a where-  EhRPC :: RPCParams -> EHentaiAPI m ByteString-  EhGallery :: RPCParams -> EHentaiAPI m ByteString--makeSem ''EHentaiAPI--{-# NOINLINE ehHttpManager #-}-ehHttpManager :: Manager-ehHttpManager = unsafePerformIO $ newManager defaultManagerSettings--{-# INLINE runEHentaiAPIIO #-}-runEHentaiAPIIO :: ClientConfig-                -> [ EHentaiAPI, Reader ClientConfig, Error ClientError, Embed IO, Final IO ] @> a-                -> IO (Either ClientError a)-runEHentaiAPIIO cfg-  = runFinal . embedToFinal . errorToIOFinal @ClientError . runReader cfg . runEHentaiAPI--{-# INLINE runEHentaiAPI #-}-runEHentaiAPI :: forall a r. Members [ Embed IO, Error ClientError, Reader ClientConfig ] r-              => EHentaiAPI : r @> a-              -> r @> a-runEHentaiAPI m = do-  cfg <- ask @ClientConfig-  currentTime <- embed (systemSeconds <$> getSystemTime)-  -- Internal: Override RPC host via HATH_RPC_HOST environment variable-  -- Format: "host" or "host:port" (default port: 80)-  rpcHostOverride <- embed (lookupEnv "HATH_RPC_HOST" :: IO (Maybe String))-  let ( rpcHostName, rpcPort ) = case rpcHostOverride of-        Nothing      -> ( "rpc.hentaiathome.net", 80 )-        Just hostStr -> case break (== ':') hostStr of-          ( h, ':' : p ) -> ( h, fromMaybe 80 (readMaybe p) )-          ( h, _ )       -> ( h, 80 )-  let k :: RPCParams -> String -> Sem r ByteString-      k params endpoint-        = either throw pure-        =<< embed-          (runClientM-             (ehAPIM-                params { acttime     = Just (show currentTime)-                       , clientbuild = Just clientBuild-                       , actkey      = Just $ makeKey params cfg currentTime-                       , cid         = Just (clientId cfg)-                       })-             (mkClientEnv ehHttpManager (BaseUrl Http rpcHostName rpcPort endpoint))-             { makeClientRequest = \baseUrl req -> do-                 servantReq <- defaultMakeClientRequest-                   baseUrl-                   (foldr (uncurry (addHeader @Text)) req hentaiHeader)-                 pure servantReq { responseTimeout = responseTimeoutMicro rpcTimeout } })-      {-# INLINE k #-}-  interpret (\case-               -- get_cert is the raw PKCS#12 file, not a signed reply-               EhRPC params-                 | act params == Just "get_cert" -> k params "/16/rpc"-                 | otherwise -> verifyRPCResponse (encodeUtf8 (key cfg)) <$> k params "/16/rpc"-               -- the gallery list carries its own signature, checked by parseMetadata-               EhGallery params -> k params "/16/dl") m-  where-    rpcTimeout :: Int-    rpcTimeout = 60 * 1000000  -- 60 seconds in microseconds--    makeKey :: RPCParams -> ClientConfig -> Int64 -> Text-    makeKey params cfg time-      = let-          act'      = fromMaybe "" (act params)-          add'      = fromMaybe "" (add params)-          clientId' = clientId cfg-          key'      = key cfg-        in -          -- [i|hentai@home-#{act'}-#{add'}-#{clientId'}-#{time}-#{key'}|]-          hash-          $ BD.toStrictByteString-            ("hentai@home-"-             <> BD.textUtf8 act'-             <> "-"-             <> BD.textUtf8 add'-             <> "-"-             <> BD.textUtf8 clientId'-             <> "-"-             <> BD.int64Dec time-             <> "-"-             <> BD.textUtf8 key')--{-# INLINE checkServerStatus #-}--- checkServerStatus :: Member EHentaiAPI r => Sem r Bool-checkServerStatus :: [ EHentaiAPI, Error RPCError ] >@> Bool-checkServerStatus = do-  res <- ehRPC emptyRPCParams { act = Just "server_stat" }-  case parseRPCResponse res of-    Left _  -> return False-    Right _ -> return True--{-# INLINE heartbeat #-}-heartbeat :: EHentaiAPI -@> ()-heartbeat = void $ ehRPC emptyRPCParams { act = Just "still_alive" }--{-# INLINE startListening #-}-startListening :: EHentaiAPI -@> Bool-startListening = do-  res <- ehRPC emptyRPCParams { act = Just "client_start" }-  case parseRPCResponse res of-    Left _  -> return False-    Right x -> return $ statusCode x == "OK"--{-# INLINE stopListening #-}-stopListening :: EHentaiAPI -@> Bool-stopListening = do-  res <- ehRPC emptyRPCParams { act = Just "client_stop" }-  case parseRPCResponse res of-    Left _  -> return False-    Right _ -> return True--{-# INLINE login #-}-login :: [ EHentaiAPI, Error RPCError ] >@> HathSettings-login = do-  x <- ehRPC emptyRPCParams { act = Just "client_login" }-  parseSettings <$> parseRPCResponse' x--downloadCertificates-  :: [ EHentaiAPI, Error RPCError, Reader ClientConfig ] >@> ( CertificateChain, PrivKey )-downloadCertificates = do-  bytes <- ehRPC emptyRPCParams { act = Just "get_cert" }-  cfg <- ask @ClientConfig-  fromPkcs12 cfg bytes-  where-    fromPkcs12 (encodeUtf8 . key -> passcode) bytes = case maybeCred of-      Left err       -> throw $ CertificateFailure $ show err-      Right Nothing  -> throw $ CertificateFailure "no credential"-      Right (Just c) -> pure c-      where-        maybeCred = do-          p12 <- readP12FileFromMemory bytes-          ( _, pkcs12 ) <- recoverAuthenticated passcode p12-          recover (toProtectionPassword passcode) (toCredential pkcs12)--{-# INLINE getSettings #-}-getSettings :: [ EHentaiAPI, Error RPCError ] >@> HathSettings-getSettings = do-  x <- ehRPC emptyRPCParams { act = Just "client_settings" }-  parseSettings <$> parseRPCResponse' x--{-# INLINE nextGalleryTask #-}--- | Left says why there is nothing to download (NO_PENDING_DOWNLOADS, or a refused list).-nextGalleryTask :: [ EHentaiAPI, Reader ClientConfig ] >@> Either Text GalleryMetadata-nextGalleryTask = do-  cfg <- ask @ClientConfig-  parseMetadata (encodeUtf8 (key cfg)) <$> ehGallery emptyRPCParams { act = Just "fetchqueue" }--{-# INLINE completeGalleryTask #-}-completeGalleryTask :: GalleryMetadata -> EHentaiAPI -@> ()-completeGalleryTask metadata-  = void-  $ ehGallery-    emptyRPCParams-    { act = Just "fetchqueue", add = Just [i|#{galleryID metadata};#{galleryMinXRes metadata}|] }--downloadGalleryFile-  :: GalleryMetadata-  -> GalleryFile-  -> [ EHentaiAPI, Reader ClientConfig, Error RPCError, Embed IO ] >@> Maybe ByteString-downloadGalleryFile metadata file = do-  ( failures, maybeContent ) <- runState @[ Text ] [] (operate 0)-  case maybeContent of-    Just content -> return $ Just content-    Nothing      -> do-      reportFailures failures-      return Nothing-  where-    download url = case parseRequest url of-      Nothing  -> pure Nothing-      Just req -> embed (try @IO @HttpException $ LBS.toStrict . responseBody <$> httpLbs @IO req)-        >>= \case-          Left _      -> pure Nothing-          Right bytes -> do-            if maybe False (/= hash bytes) (galleryFileHash file)-              then do-                modify-                  @[ Text ]-                  ([i|#{host req}-#{galleryFileIndex file}-#{galleryFileXRes file}|] :)-                pure Nothing-              else return $ Just bytes--    operate (retries :: Int)-      | retries > 3 = return Nothing-      | otherwise = do-        urls <- parseRPCResponse'-          =<< ehRPC-            emptyRPCParams-            { act = Just "dlfetch"-            , add = Just-                [i|#{galleryID metadata};#{galleryFilePage file};#{galleryFileIndex file};#{galleryFileXRes file};#{retries}|]-            }--        if null urls-          then operate (retries + 1) -- we did not get a valid URL, might be a network error-          else do-            -- `traverse` would execute every fallback download eagerly.-            -- We need monadic short-circuiting to stop after the first success.-            firstJustM (download . BSC.unpack) urls >>= maybe (operate (retries + 1)) (pure . Just)--{-# INLINE reportFailures #-}-reportFailures :: [ Text ] -> EHentaiAPI -@> ()-reportFailures reports-  = void $ ehRPC emptyRPCParams { act = Just "dlfails", add = Just $ T.intercalate ";" reports }---- | Ask the RPC server where a static-range file lives and open the first source--- that answers 200 with the expected length, as the official ProxyFileDownloader--- does. The caller streams the body and must close the response.-openResource :: FileURI-             -> ( ByteString, ByteString )-             -> [ EHentaiAPI, Reader ClientConfig, Error RPCError, Embed IO ] >@> Maybe (Response BodyReader)-openResource fileURI ( fileIndex, xres ) = do-  urls <- parseRPCResponse'-    =<< ehRPC-      emptyRPCParams { act = Just "srfetch", add = Just [i|#{fileIndex};#{xres};#{fileURI}|] }-  cfg <- ask @ClientConfig-  let token = [i|#{clientId cfg}-#{hash @ByteString $ encodeUtf8 (key cfg) <> renderFileURIBytes fileURI}|]-  embed $ do-    mgr <- getGlobalManager-    firstJustM (open mgr token . BSC.unpack) urls-  where-    open mgr token url = case parseRequest url of-      Nothing  -> pure Nothing-      Just req -> try @IO @HttpException-        (responseOpen req { requestHeaders = ( "Hath-Request", token ) : hentaiHeader } mgr)-        >>= \case-          Right resp-            | HT.statusCode (responseStatus resp) == 200-            , List.lookup hContentLength (responseHeaders resp) == Just (show (fileSize fileURI)) ->-              pure (Just resp)-            | otherwise -> responseClose resp $> Nothing-          -- connection error or wrong answer: firstJustM tries the next source-          Left _ -> pure Nothing--fetchBlacklist :: Int -> [ EHentaiAPI, Error RPCError ] >@> [ Text ]-fetchBlacklist deltaTime = do-  res <- ehRPC emptyRPCParams { act = Just "get_blacklist", add = Just $ show deltaTime }-  fmap decodeUtf8 <$> parseRPCResponse' res
− src/LegacyCiphers.hs
@@ -1,172 +0,0 @@--- |--- Module      : LegacyCiphers--- License     : BSD-3-Clause--- Maintainer  : Vincent Hanquez--- Stability   : experimental--- Portability : unknown------ This module contains legacy cipher definitions borrowed from the--- tls package (version 1.9.0) by Vincent Hanquez.------ Original source: https://hackage.haskell.org/package/tls-1.9.0/docs/src/Network.TLS.Extra.Cipher.html------ Copyright (c) Vincent Hanquez------ Redistribution and use in source and binary forms, with or without--- modification, are permitted provided that the following conditions are met:------ 1. Redistributions of source code must retain the above copyright notice,---    this list of conditions and the following disclaimer.------ 2. Redistributions in binary form must reproduce the above copyright notice,---    this list of conditions and the following disclaimer in the documentation---    and/or other materials provided with the distribution.------ 3. Neither the name of the copyright holder nor the names of its contributors---    may be used to endorse or promote products derived from this software without---    specific prior written permission.------ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"--- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE--- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE--- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE--- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR--- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF--- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS--- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN--- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)--- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE--- POSSIBILITY OF SUCH DAMAGE.------ This file is part of the hs-hath project, which is licensed under GPL-3.0.--- The BSD-3-Clause license applies to this specific file as noted above.-module LegacyCiphers-  ( cipher_ECDHE_ECDSA_AES128CBC_SHA-  , cipher_ECDHE_RSA_AES128CBC_SHA-  , cipher_ECDHE_RSA_AES256CBC_SHA-  ) where--import           Crypto.Cipher.AES   ( AES128, AES256 )-import           Crypto.Cipher.Types ( BlockCipher-                                     , IV-                                     , cbcDecrypt-                                     , cbcEncrypt-                                     , cipherInit-                                     , makeIV-                                     )-import           Crypto.Error        ( CryptoFailable, throwCryptoError )--import qualified Data.ByteString     as B--import           Network.TLS         ( Version(..) )-import           Network.TLS.Cipher  ( Bulk(..)-                                     , BulkBlock-                                     , BulkDirection(..)-                                     , BulkFunctions(..)-                                     , BulkKey-                                     , Cipher(..)-                                     , CipherKeyExchangeType(..)-                                     , Hash(..)-                                     )--import           Relude--cipher_ECDHE_ECDSA_AES128CBC_SHA :: Cipher-cipher_ECDHE_ECDSA_AES128CBC_SHA-  = Cipher { cipherID          = 0xC009-           , cipherName        = "ECDHE-ECDSA-AES128CBC-SHA"-           , cipherBulk        = bulk_aes128-           , cipherHash        = SHA1-           , cipherPRFHash     = Nothing-           , cipherKeyExchange = CipherKeyExchange_ECDHE_ECDSA-           , cipherMinVer      = Just TLS10-           }--cipher_ECDHE_RSA_AES128CBC_SHA :: Cipher-cipher_ECDHE_RSA_AES128CBC_SHA-  = Cipher { cipherID          = 0xC013-           , cipherName        = "ECDHE-RSA-AES128CBC-SHA"-           , cipherBulk        = bulk_aes128-           , cipherHash        = SHA1-           , cipherPRFHash     = Nothing-           , cipherKeyExchange = CipherKeyExchange_ECDHE_RSA-           , cipherMinVer      = Just TLS10-           }--cipher_ECDHE_RSA_AES256CBC_SHA :: Cipher-cipher_ECDHE_RSA_AES256CBC_SHA-  = Cipher { cipherID          = 0xC014-           , cipherName        = "ECDHE-RSA-AES256CBC-SHA"-           , cipherBulk        = bulk_aes256-           , cipherHash        = SHA1-           , cipherPRFHash     = Nothing-           , cipherKeyExchange = CipherKeyExchange_ECDHE_RSA-           , cipherMinVer      = Just TLS10-           }--bulk_aes128 :: Bulk-bulk_aes128-  = Bulk { bulkName       = "AES128"-         , bulkKeySize    = 16-         , bulkIVSize     = 16-         , bulkExplicitIV = 0-         , bulkAuthTagLen = 0-         , bulkBlockSize  = 16-         , bulkF          = BulkBlockF aes128cbc-         }--bulk_aes256 :: Bulk-bulk_aes256-  = Bulk { bulkName       = "AES256"-         , bulkKeySize    = 32-         , bulkIVSize     = 16-         , bulkExplicitIV = 0-         , bulkAuthTagLen = 0-         , bulkBlockSize  = 16-         , bulkF          = BulkBlockF aes256cbc-         }--noFail :: CryptoFailable a -> a-noFail = throwCryptoError--makeIV_ :: BlockCipher a => B.ByteString -> IV a-makeIV_ = fromMaybe (error "makeIV_") . makeIV--takelast :: Int -> B.ByteString -> B.ByteString-takelast i b = B.drop (B.length b - i) b--aes128cbc :: BulkDirection -> BulkKey -> BulkBlock-aes128cbc BulkEncrypt key-  = let-      ctx = noFail (cipherInit key) :: AES128-    in-      (\iv input -> let-           output = cbcEncrypt ctx (makeIV_ iv) input-         in-           ( output, takelast 16 output ))-aes128cbc BulkDecrypt key-  = let-      ctx = noFail (cipherInit key) :: AES128-    in-      (\iv input -> let-           output = cbcDecrypt ctx (makeIV_ iv) input-         in-           ( output, takelast 16 input ))--aes256cbc :: BulkDirection -> BulkKey -> BulkBlock-aes256cbc BulkEncrypt key-  = let-      ctx = noFail (cipherInit key) :: AES256-    in-      (\iv input -> let-           output = cbcEncrypt ctx (makeIV_ iv) input-         in-           ( output, takelast 16 output ))-aes256cbc BulkDecrypt key-  = let-      ctx = noFail (cipherInit key) :: AES256-    in-      (\iv input -> let-           output = cbcDecrypt ctx (makeIV_ iv) input-         in-           ( output, takelast 16 input ))
− src/Metrics/Counter.hs
@@ -1,27 +0,0 @@-module Metrics.Counter ( inc, add, read ) where--import           Metrics.Types--import           Relude---- | Atomically increment counter by 1-inc :: Counter -> IO ()-inc = add 1---- | Atomically add value to counter-add :: Int -> Counter -> IO ()-add n counter = atomicModifyIORef' (counterRef counter) $ \old -> ( old + fromIntegral n, () )--{-# INLINE add #-}---- | Read current counter value--- This is used for snapshot generation and should not be used in hot paths-read :: Counter -> IO Int64-read = readIORef . counterRef--{-# INLINE read #-}---- Note: We use atomicModifyIORef' instead of modifyIORef' because:--- 1. It's truly atomic (lock-free at CPU level)--- 2. No MVar blocking--- 3. Performance: ~50ns per operation vs ~500ns with MVar
− src/Metrics/Gauge.hs
@@ -1,41 +0,0 @@-module Metrics.Gauge ( inc, dec, set, add, sub, read ) where--import           Metrics.Types--import           Relude---- | Atomically increment gauge by 1-inc :: Gauge -> IO ()-inc = add 1--{-# INLINE inc #-}---- | Atomically decrement gauge by 1-dec :: Gauge -> IO ()-dec = sub 1--{-# INLINE dec #-}---- | Set gauge to exact value (non-atomic write is sufficient for gauges)-set :: Int64 -> Gauge -> IO ()-set n gauge = writeIORef (gaugeRef gauge) n--{-# INLINE set #-}---- | Atomically add value to gauge-add :: Int -> Gauge -> IO ()-add n gauge = atomicModifyIORef' (gaugeRef gauge) $ \old -> ( old + fromIntegral n, () )--{-# INLINE add #-}---- | Atomically subtract value from gauge-sub :: Int -> Gauge -> IO ()-sub n = add (negate n)--{-# INLINE sub #-}---- | Read current gauge value-read :: Gauge -> IO Int64-read = readIORef . gaugeRef--{-# INLINE read #-}
− src/Metrics/Prometheus.hs
@@ -1,84 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE RankNTypes #-}--module Metrics.Prometheus ( render ) where--import qualified Data.Map.Strict as Map-import qualified Data.Text       as Text--import           Mason.Builder   ( BuilderFor-                                 , StrictByteStringBackend-                                 , byteString-                                 , int64Dec-                                 , intersperse-                                 , textUtf8-                                 , toStrictByteString-                                 )--import           Metrics.Types--import           Relude          hiding ( intersperse )--type MBuilder = BuilderFor StrictByteStringBackend---- | Render a list of metric snapshots to Prometheus text format-render :: [ MetricSnapshot ] -> ByteString-render = toStrictByteString . mconcat . map renderMetric---- | Render a single metric snapshot-renderMetric :: MetricSnapshot -> MBuilder-renderMetric = \case-  CounterSnapshot metricId help val -> renderMetricLines "counter" metricId help val-  GaugeSnapshot metricId help val   -> renderMetricLines "gauge" metricId help val---- | Render the complete metric output (HELP, TYPE, and value lines)-renderMetricLines :: ByteString -> MetricId -> Text -> Int64 -> MBuilder-renderMetricLines typ (MetricId name labels) help val-  = mconcat-    [ "# HELP "-    , textUtf8 name-    , " "-    , textUtf8 help-    , "\n"-    , "# TYPE "-    , textUtf8 name-    , " "-    , byteString typ-    , "\n"-    , textUtf8 name-    , if Map.null labels-        then mempty-        else renderLabels labels-    , " "-    , int64Dec val-    , "\n"-    ]---- | Render label set: {label1="value1",label2="value2"}-renderLabels :: Labels -> MBuilder-renderLabels labels-  | Map.null labels = mempty-  | otherwise = mconcat [ "{", intersperse "," $ map renderLabel $ Map.toList labels, "}" ]---- | Render a single label: key="value"-renderLabel :: ( Text, Text ) -> MBuilder-renderLabel ( k, v ) = textUtf8 k <> "=\"" <> textUtf8 (escapeValue v) <> "\""---- | Escape special characters in label values according to Prometheus spec--- See: https://prometheus.io/docs/instrumenting/exposition_formats/-escapeValue :: Text -> Text-escapeValue = Text.replace "\\" "\\\\" . Text.replace "\"" "\\\"" . Text.replace "\n" "\\n"---- Example output:------ # HELP hath_cache_sent_bytes_total Total bytes sent from cache--- # TYPE hath_cache_sent_bytes_total counter--- hath_cache_sent_bytes_total 1234567------ # HELP hath_errors_total Total errors by type--- # TYPE hath_errors_total counter--- hath_errors_total{type="network"} 42--- hath_errors_total{type="timeout"} 13------ Note: Labels with the same metric name but different label values--- are currently registered as separate Counter instances.
− src/Metrics/Registry.hs
@@ -1,59 +0,0 @@-module Metrics.Registry-  ( newRegistry-  , registerCounter-  , registerGauge-  , registerCounterWith-  , registerGaugeWith-  ) where--import           Metrics.Types--import           Relude---- | Create a new empty registry-newRegistry :: IO Registry-newRegistry = do-  counters <- newIORef []-  gauges <- newIORef []-  cachedSnapshot <- newTVarIO ""-  pure $ Registry counters gauges cachedSnapshot---- | Register a counter without labels-registerCounter :: Text -> Text -> Registry -> IO Counter-registerCounter name help = registerCounterWith name help mempty---- | Register a counter with labels-registerCounterWith :: Text -> Text -> Labels -> Registry -> IO Counter-registerCounterWith name help labels reg = do-  ref <- newIORef 0-  let metricId = MetricId name labels-      counter  = Counter metricId help ref-  -- Atomically prepend to counter list-  atomicModifyIORef' (registryCounters reg) $ \cs -> ( counter : cs, () )-  pure counter---- | Register a gauge without labels-registerGauge :: Text -> Text -> Registry -> IO Gauge-registerGauge name help = registerGaugeWith name help mempty---- | Register a gauge with labels-registerGaugeWith :: Text -> Text -> Labels -> Registry -> IO Gauge-registerGaugeWith name help labels reg = do-  ref <- newIORef 0-  let metricId = MetricId name labels-      gauge    = Gauge metricId help ref-  -- Atomically prepend to gauge list-  atomicModifyIORef' (registryGauges reg) $ \gs -> ( gauge : gs, () )-  pure gauge---- Note: Registration is typically done at startup, so performance is not critical.--- Using atomicModifyIORef' ensures thread-safety if multiple threads register metrics.------ Label usage example (for future):---   errorCounter <- registerCounterWith---     "hath_errors_total"---     "Total errors by type"---     (Map.fromList [("type", "network")])---     registry------ This creates a counter with fixed labels at registration time.
− src/Metrics/Snapshot.hs
@@ -1,53 +0,0 @@-module Metrics.Snapshot ( generateSnapshot, startSnapshotWorker, getCachedSnapshot ) where--import           Control.Concurrent ( forkIO, threadDelay )--import qualified Metrics.Counter    as Counter-import qualified Metrics.Gauge      as Gauge-import qualified Metrics.Prometheus as Prometheus-import           Metrics.Types--import           Relude---- | Generate a snapshot of all metrics in the registry--- This reads all counter and gauge values and renders them to Prometheus format-generateSnapshot :: Registry -> IO ByteString-generateSnapshot reg = do-  counters <- readIORef (registryCounters reg)-  gauges <- readIORef (registryGauges reg)--  -- Read all counter values-  counterSnapshots <- forM counters $ \c -> do-    val <- Counter.read c-    pure $ CounterSnapshot (counterMetricId c) (counterHelp c) val--  -- Read all gauge values-  gaugeSnapshots <- forM gauges $ \g -> do-    val <- Gauge.read g-    pure $ GaugeSnapshot (gaugeMetricId g) (gaugeHelp g) val--  -- Render to Prometheus text format-  pure $ Prometheus.render (counterSnapshots <> gaugeSnapshots)---- | Start a background worker that periodically updates the cached snapshot--- This decouples metrics reads from the /metrics endpoint, eliminating blocking-startSnapshotWorker :: Registry -> Int -> IO ()-startSnapshotWorker reg intervalMicros = void $ forkIO $ forever $ do-  snapshot <- generateSnapshot reg-  atomically $ writeTVar (registryCachedSnapshot reg) snapshot-  threadDelay intervalMicros---- | Read the cached snapshot (zero-cost operation)--- This is what the /metrics endpoint should call-getCachedSnapshot :: Registry -> IO ByteString-getCachedSnapshot = readTVarIO . registryCachedSnapshot--{-# INLINE getCachedSnapshot #-}---- Performance notes:--- - generateSnapshot: O(n) where n = number of metrics (~10-100 typically)---   Takes ~1-10ms depending on metric count--- - startSnapshotWorker: Runs in background, default 5s interval---   CPU overhead: negligible (~0.02% with 10 metrics)--- - getCachedSnapshot: O(1), ~100ns---   This is what makes /metrics endpoint non-blocking
− src/Metrics/Types.hs
@@ -1,40 +0,0 @@-{-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE StrictData #-}--module Metrics.Types-  ( Labels-  , MetricId(..)-  , Counter(..)-  , Gauge(..)-  , Registry(..)-  , MetricSnapshot(..)-  ) where--import           GHC.Generics--import           Relude---- | Label support for metrics (key-value pairs)-type Labels = Map Text Text---- | Metric identifier combining name and labels-data MetricId = MetricId { metricName :: !Text, metricLabels :: !Labels }-  deriving ( Eq, Ord, Show, Generic )---- | Counter metric (monotonically increasing)-data Counter-  = Counter { counterMetricId :: !MetricId, counterHelp :: !Text, counterRef :: !(IORef Int64) }---- | Gauge metric (can increase or decrease)-data Gauge = Gauge { gaugeMetricId :: !MetricId, gaugeHelp :: !Text, gaugeRef :: !(IORef Int64) }---- | Registry holding all registered metrics-data Registry-  = Registry { registryCounters       :: !(IORef [ Counter ])-             , registryGauges         :: !(IORef [ Gauge ])-             , registryCachedSnapshot :: !(TVar ByteString)-             }---- | Snapshot of a metric value (for rendering)-data MetricSnapshot = CounterSnapshot !MetricId !Text !Int64 | GaugeSnapshot !MetricId !Text !Int64-  deriving ( Show )
− src/Middleware.hs
@@ -1,196 +0,0 @@-module Middleware-  ( rateLimitMiddleware-  , tracingConnections-  , normalizeAcceptMiddleware-  , tracingTimeUsage-  , withHentaiHeaders-  , IPMap-  , KeystampMap-  , IP(..)-  , IPRecord(..)-  , timeWindow-  , checkRateLimit-  , isLocalIP-  , parseIPv4-  ) where--import           Control.Exception   ( finally )--import           Data.Bits           ( shiftR, (.&.) )--import qualified Data.ByteString     as BS-import qualified Data.HashMap.Strict as HashMap--import           GHC.Clock           ( getMonotonicTime )--import qualified Metrics.Gauge       as Gauge--import           Network.HTTP.Types  ( hAccept )-import           Network.Socket      ( HostAddress-                                     , HostAddress6-                                     , SockAddr(..)-                                     , hostAddress6ToTuple-                                     , hostAddressToTuple-                                     , tupleToHostAddress-                                     )-import           Network.Wai         ( Middleware-                                     , mapResponseHeaders-                                     , rawPathInfo-                                     , remoteHost-                                     , requestHeaders-                                     )--import           Relude              hiding ( Reader, ask, get, runReader )--import           Stats               ( StatsEnv(..) )--import           System.IO.Unsafe    ( unsafePerformIO )--import           Types               ( hentaiHeader )--import           Utils               ( tooManyRequestsResponse )---- | Mirrors the official client's flood control (HTTPServer.FloodControlEntry):--- every request adds one hit, hits drain at one per second, and more than--- 'floodLimit' hits blocks the IP for 'blockSeconds'.-floodLimit :: Int-floodLimit = 10--blockSeconds :: Double-blockSeconds = 60---- | Entries idle for this long carry no state worth keeping (any block has expired).-timeWindow :: Double-timeWindow = 60--data IPRecord-  = IPRecord { hitCount    :: !Int     -- ^ Hits left after draining-             , lastHit     :: !Double  -- ^ Monotonic time of the last counted hit-             , bannedUntil :: !Double  -- ^ When the block expires-             }-  deriving ( Show )---- Change the Map key from SockAddr to a custom IP type-data IP = IPv4 {-# UNPACK #-} !HostAddress | IPv6 {-# UNPACK #-} !HostAddress6-  deriving ( Eq, Ord, Show, Generic )--instance Hashable IP--type IPMap = HashMap IP IPRecord--{-# INLINE getIP #-}--- Helper function to extract just the IP from a SockAddr-getIP :: SockAddr -> Maybe IP-getIP (SockAddrInet _ ha) = Just $ IPv4 ha-getIP (SockAddrInet6 _ _ ha _) = Just $ IPv6 ha-getIP _ = Nothing--{-# NOINLINE globalIPMap #-}-globalIPMap :: IORef IPMap-globalIPMap = unsafePerformIO $ newIORef HashMap.empty---- | Flood control for /h/. Local addresses and those the predicate exempts--- (the RPC servers, as in the official client) are never limited.-rateLimitMiddleware :: (IP -> IO Bool) -> Middleware-rateLimitMiddleware exempt app req k-  | "/h/" `BS.isPrefixOf` rawPathInfo req = do-    now <- getMonotonicTime-    let sockAddr = remoteHost req-    case getIP sockAddr of-      Nothing -> app req k  -- If we can't get IP, just allow the request-      Just ip-        | isLocalIP ip -> app req k-        | otherwise -> do-          skip <- exempt ip-          allowed <- if skip then pure True else checkRateLimit ip now-          if allowed-            then app req k-            else k tooManyRequestsResponse-  | otherwise = app req k  -- Skip rate limiting for non-resource paths---- | "a.b.c.d" or "::ffff:a.b.c.d", the form rpc_server_ip uses.--- ponytail: IPv4 only; the RPC servers have no IPv6 addresses.-parseIPv4 :: ByteString -> Maybe IP-parseIPv4 s = case traverse (readMaybe @Int . decodeUtf8) (BS.split 46 v4) of-  Just [ a, b, c, d ]-    | all (\x -> x >= 0 && x <= 255) [ a, b, c, d ] -> Just-      (IPv4 (tupleToHostAddress ( fromIntegral a, fromIntegral b, fromIntegral c, fromIntegral d )))-  _ -> Nothing-  where-    v4 = fromMaybe s (BS.stripPrefix "::ffff:" s)---- | Loopback, private and link-local addresses, which the official client--- exempts from flood control (HTTPServer.localNetworkPattern).-isLocalIP :: IP -> Bool-isLocalIP (IPv4 ha) = localV4 (hostAddressToTuple ha)-isLocalIP (IPv6 ha) = case hostAddress6ToTuple ha of-  ( 0, 0, 0, 0, 0, 0, 0, 1 ) -> True-  -- IPv4-mapped (::ffff:a.b.c.d), as seen on dual-stack sockets-  ( 0, 0, 0, 0, 0, 0xffff, hi, lo ) -> localV4 ( byte hi 8, byte hi 0, byte lo 8, byte lo 0 )-  ( w, _, _, _, _, _, _, _ ) -> (w .&. 0xfe00) == 0xfc00 -- fc00::/7-  where-    byte w n = fromIntegral (w `shiftR` n)--localV4 :: ( Word8, Word8, Word8, Word8 ) -> Bool-localV4 = \case-  ( 127, _, _, _ ) -> True-  ( 10, _, _, _ ) -> True-  ( 192, 168, _, _ ) -> True-  ( 172, b, _, _ ) -> b >= 16 && b <= 31-  ( 169, 254, _, _ ) -> True-  _ -> False---- | Count a hit for the IP; False if the request should be blocked.-checkRateLimit :: IP -> Double -> IO Bool-checkRateLimit ip now = atomicModifyIORef' globalIPMap $ \m0 ->-  let-    -- ponytail: O(n) prune once the map is large; fine for the few hundred IPs a client sees per minute-    m = if HashMap.size m0 > 1000-          then HashMap.filter ((> now - timeWindow) . lastHit) m0-          else m0-  in -    case HashMap.lookup ip m of-      Just r-        | bannedUntil r > now -> ( m, False ) -- blocked requests are not counted, as in the official client-      old -> let-          r0      = fromMaybe (IPRecord 0 now 0) old-          hits    = max 0 (hitCount r0 - floor (now - lastHit r0)) + 1-          blocked = hits > floodLimit-          r       = IPRecord { hitCount    = hits-                             , lastHit     = now-                             , bannedUntil = if blocked then now + blockSeconds else bannedUntil r0-                             }-        in -          ( HashMap.insert ip r m, not blocked )--type KeystampMap = HashMap ByteString ( Int, Double )---- | Track active connections-tracingConnections :: StatsEnv -> Middleware-tracingConnections statsEnv app req k = do-  Gauge.inc (statsActiveConnectionsGauge statsEnv)-  finally (app req k) (Gauge.dec (statsActiveConnectionsGauge statsEnv))--normalizeAcceptMiddleware :: Middleware-normalizeAcceptMiddleware app req = app req { requestHeaders = normalizedHeaders }-  where-    normalizedHeaders = map normalizeHeader (requestHeaders req)--    normalizeHeader ( name, value )-      | name == hAccept = ( name, "*/*" )-      | otherwise = ( name, value )--tracingTimeUsage :: StatsEnv -> Middleware-tracingTimeUsage statsEnv app req k = do-  startTime <- getMonotonicTime-  finally (app req k) (do-                         endTime <- getMonotonicTime-                         let latency = endTime - startTime-                         Gauge.set (round (latency * 1000000)) (statsLatency statsEnv))--withHentaiHeaders :: Middleware-withHentaiHeaders app req k = app req (k . mapResponseHeaders addHeaders)-  where-    addHeaders headers@(map fst -> names)-      = filter ((`notElem` names) . fst) hentaiHeader ++ headers-
− src/Server.hs
@@ -1,831 +0,0 @@-{-# LANGUAGE DataKinds #-}--{-# LANGUAGE PartialTypeSignatures #-}-{-# LANGUAGE TypeOperators #-}--module Server-  ( startServer-  , ServerAction(..)-  , makeApplication-  , mkServerLoopContext-  , CacheRunner(..)-  , IPMap-  , KeystampMap-  , GalleryTask(..)-  ) where--import           Colog                                ( Message-                                                      , Severity(Info, Warning)-                                                      , richMessageAction-                                                      )-import           Colog.Polysemy                       ( Log, runLogAction )--import           Control.Concurrent                   ( forkIO, threadDelay )-import           Control.Concurrent.Suspend           ( mDelay )-import           Control.Concurrent.Timer             ( repeatedTimer, stopTimer )-import           Control.Exception                    ( finally, try )--import qualified Data.ByteString                      as BS-import qualified Data.ByteString.Char8                as BSC-import           Data.ByteString.Builder              ( byteString )-import qualified Data.ByteString.Lazy                 as LBS-import qualified Data.ByteString.Lazy.Char8           as LBS8-import qualified Data.HashMap.Strict                  as HashMap-import qualified Data.Map.Strict                      as Map-import           Data.String.Interpolate              ( i )-import qualified Data.Text                            as Text-import           Data.Time.Clock.System               ( SystemTime(systemSeconds), getSystemTime )-import           Data.X509                            ( CertificateChain, PrivKey )--import           Database.SQLite.Simple               ( withConnection )--import           HathNetwork.Genesis-import           HathNetwork.RPC                      ( RPC-                                                      , notifyGalleryCompletion-                                                      , runRPC-                                                      , runRPCIO-                                                      , stillAlive-                                                      )--import           Interface.API                        ( API-                                                      , EHentaiAPI-                                                      , ServerCommand(..)-                                                      , api-                                                      , downloadGalleryFile-                                                      , fetchBlacklist-                                                      , nextGalleryTask-                                                      , renderServerCommand-                                                      , runEHentaiAPI-                                                      , runEHentaiAPIIO-                                                      , startListening-                                                      , stopListening-                                                      )--import           LegacyCiphers--import qualified Mason.Builder                        as BD--import           Middleware--import qualified Network.HTTP.Client                  as HC-import           Network.HTTP.Client                  ( Manager-                                                      , brRead-                                                      , newManager-                                                      , parseRequest-                                                      , withResponse-                                                      )-import           Network.HTTP.Client.TLS              ( tlsManagerSettings )-import           Network.HTTP.Simple                  ( getResponseBody, getResponseStatusCode )-import qualified Network.HTTP.Types                   as HT-import           Network.HTTP.Types                   ( hContentLength-                                                      , hContentType-                                                      , status200-                                                      , status404-                                                      )-import           Network.TLS                          ( Credentials(Credentials), Version(..) )-import           Network.TLS.Extra                    ( ciphersuite_all )-import           Network.Wai                          ( Middleware-                                                      , Request(requestMethod)-                                                      , responseLBS-                                                      , responseStream-                                                      )-import           Network.Wai.Handler.Warp             ( defaultSettings, setPort )-import           Network.Wai.Handler.WarpTLS          ( OnInsecure(AllowInsecure)-                                                      , TLSSettings(..)-                                                      , defaultTlsSettings-                                                      , runTLS-                                                      )-import           Network.Wai.Middleware.RealIp        ( realIpHeader )-import           Network.Wai.Middleware.RequestLogger ( logStdout )--import           Polysemy                             ( Embed-                                                      , Members-                                                      , Sem-                                                      , embed-                                                      , embedToFinal-                                                      , runFinal-                                                      )-import           Polysemy.Async                       ( Async, asyncToIOFinal )-import           Polysemy.Error                       ( Error, errorToIOFinal, mapError, throw )-import           Polysemy.KVStore                     ( KVStore )-import           Polysemy.Reader                      ( Reader, ask, runReader )--import           Relude                               hiding ( Reader, ask, get, runReader )--import           Servant-import           Servant.Client                       ( ClientError )-import qualified Servant.Types.SourceT                as Source--import           SettingM                             ( SettingM(..)-                                                      , getSettings-                                                      , runSettingM-                                                      , updateSettings-                                                      )--import           Stats                                ( Stats-                                                      , StatsEnv(..)-                                                      , addDlBytes-                                                      , addDownload-                                                      , addUpload-                                                      , incDlFile-                                                      , incDlTask-                                                      , incServed-                                                      , newStatsEnv-                                                      , readPrometheus-                                                      , runStats-                                                      )--import           Storage.Database                     ( FileRecord(fileRecordBytes)-                                                      , initializeDB-                                                      , runCache-                                                      )-import           Storage.Filesystem                   ( runCacheFilesystem )-import           Storage.Locate-import           Storage.R2                           ( mkR2Connection, runCacheR2 )--import           System.Directory                     ( createDirectoryIfMissing, doesFileExist )-import           System.FilePath                      ( takeDirectory )-import           System.IO                            ( hPutStrLn )-import           System.IO.Unsafe                     ( unsafePerformIO )--import           Types                                ( CacheBackend(..)-                                                      , ClientConfig-                                                      , ClientConfig(..)-                                                      , FileURI(fileExt, fileSize)-                                                      , GalleryFile(..)-                                                      , GalleryMetadata(..)-                                                      , HathSettings(..)-                                                      , RPCError-                                                      , StorageResult(..)-                                                      , hentaiHeader-                                                      , parseFileURI-                                                      )--import           UnliftIO                             ( TChan-                                                      , getMonotonicTime-                                                      , newTChanIO-                                                      , race-                                                      , readTChan-                                                      , replicateConcurrently-                                                      , timeout-                                                      , withAsync-                                                      , writeTChan-                                                      )--import           Utils                                ( bufferSending-                                                      , hash-                                                      , log-                                                      , lookupParam-                                                      , parseURLParams-                                                      )--maxTimeDrift :: Int64-maxTimeDrift = 600---- | Official Settings.MAX_KEY_TIME_DRIFT, used for servercmd.-maxCommandTimeDrift :: Int64-maxCommandTimeDrift = 300---- | Whole-download cap for one threaded_proxy_test request.--- ponytail: one total timeout; the official client uses 10 s connect + 60 s per read.-testTimeoutMicros :: Int-testTimeoutMicros = 60 * 1000000--data ServerAction = Reload | Cert | GracefulShutdown--data GalleryTask = GalleryTask---- | Server runtime environment containing all channels for inter-thread communication.--- This bundles channels used by the server loop to coordinate actions and tasks.-data ServerChannels-  = ServerChannels-  { serverActionChan :: !(TChan ServerAction)-    -- ^ Channel for receiving server control commands (reload, cert refresh, shutdown)-  , galleryTaskChan  :: !(TChan GalleryTask)-    -- ^ Channel for coordinating gallery download tasks-  }---- | Server runtime state containing all mutable variables.--- These TVars are shared across threads and updated during server operation.-data ServerState-  = ServerState { serverSettings :: !(TVar HathSettings)-                  -- ^ Dynamic server settings that can be updated via RPC-                , serverStatsEnv :: !StatsEnv-                  -- ^ Prometheus metrics environment (gauges, counters, histograms)-                }---- | Server configuration flags controlling middleware behavior.-data ServerFlags-  = ServerFlags { disableRateLimit  :: !Bool-                  -- ^ If True, bypass IP-based rate limiting middleware-                , trustProxyHeaders :: !Bool-                  -- ^ If True, trust X-Forwarded-For headers for real client IP-                }---- | Server loop context bundling all runtime dependencies.--- This groups all parameters needed by the server loop to reduce parameter count--- and improve code clarity.-data ServerLoopContext-  = ServerLoopContext-  { loopConfig      :: !ClientConfig-    -- ^ Static client configuration (client ID, key, cache backend, etc.)-  , loopChannels    :: !ServerChannels-    -- ^ Communication channels for server actions and gallery tasks-  , loopState       :: !ServerState-    -- ^ Mutable server state (settings, rate limits, metrics)-  , loopFlags       :: !ServerFlags-    -- ^ Configuration flags controlling middleware behavior-  , loopCacheRunner :: !CacheRunner-    -- ^ Cache backend interpreter (SQLite, R2, or Filesystem)-  }--mkServerLoopContext-  :: ClientConfig-  -> TVar HathSettings-  -> TChan ServerAction-  -> TChan GalleryTask-  -> Bool-  -> Bool-  -> CacheRunner-  -> StatsEnv-  -> ServerLoopContext-mkServerLoopContext cfg settings action gallery disableRL trustProxy cacheRunner statsEnv-  = ServerLoopContext-  { loopConfig      = cfg-  , loopChannels    = ServerChannels-      { serverActionChan = action-      , galleryTaskChan  = gallery-      }-  , loopState       = ServerState-      { serverSettings = settings-      , serverStatsEnv = statsEnv-      }-  , loopFlags       = ServerFlags-      { disableRateLimit  = disableRL-      , trustProxyHeaders = trustProxy-      }-  , loopCacheRunner = cacheRunner-  }---- Server implementation-server :: Members-         '[ Embed IO-          , Error ServerError-          , Error RPCError-          , EHentaiAPI-          , Reader ClientConfig-          , RPC-          , Locate-          , Reader (TChan ServerAction)-          , Reader (TChan GalleryTask)-          , Stats-          , Log Message-          , Genesis-          , SettingM-          , Async-          ]-         r-       => ServerT API (Sem r)-server-  = faviconHandler-  :<|> robotsHandler-  :<|> resourceHandler-  :<|> statsHandler-  :<|> serverCmdHandler-  :<|> testHandler-  :<|> adminSettingsHandler-  :<|> Tagged rawHandler-  where-    faviconHandler-      = throw $ err301 { errHeaders = [ ( "Location", "https://e-hentai.org/favicon.ico" ) ] }--    robotsHandler = return "User-agent: *\nDisallow: /"--    resourceHandler-      (encodeUtf8 -> fileid)-      (parseURLParams . encodeUtf8 -> opts)-      (encodeUtf8 -> filename)-      req-      respond-      -- answer HEAD like any other path instead of opening an upstream body nobody reads-      | requestMethod req == "HEAD" = embed $ respond $ responseLBS status200 hentaiHeader ""-      | otherwise = do-      currentTime <- embed getSystemTime-      cfg <- ask @ClientConfig-      when (abs (timestamp - systemSeconds currentTime) > maxTimeDrift)-        $ throw err403 { errBody = "Your time is out of sync. Please update your system time." }-      when (answer /= challange cfg) $ throw err403 { errBody = "Invalid key." }-      locateResource-        LocateURI { locateURIFilename = filename, locateURI = uri, locateURIOptions = opts }-        >>= \case-          Nothing -> throw err404-          Just (Stored (Redirect url)) -> throw-            $ err302-            { errHeaders = [ ( "Location", url ), ( "Cache-Control", "public, max-age=600" ) ] }-          Just (Stored (Record (fileRecordBytes -> bs))) -> do-            incServed-            addUpload (BS.length bs)-            embed $ respond $ responseLBS status200 (fileHeaders (BS.length bs)) (LBS.fromStrict bs)-          -- cache miss: relay upstream bytes as they arrive, like the official proxy mode-          Just (Streamed relay) -> do-            incServed-            addUpload (fileSize uri)-            embed-              $ respond-              $ responseStream status200 (fileHeaders (fileSize uri))-              $ \write flush -> relay (\chunk -> write (byteString chunk) >> flush)-      where-        fileHeaders :: Int -> [ HT.Header ]-        fileHeaders len = [ ( hContentType, mimeType ), ( hContentLength, show len ) ]--        mimeType = case fileExt uri of-          "jpg" -> "image/jpeg"-          "png" -> "image/png"-          "gif" -> "image/gif"-          "mp4" -> "video/mp4"-          "wbm" -> "video/webm"-          "wbp" -> "image/webp"-          "avf" -> "image/avif"-          "jxl" -> "image/jxl"-          _     -> "application/octet-stream"--        uri = parseFileURI fileid--        ( timestamp :: Int64, answer )-          = let-              rs        = fromMaybe "" (Map.lookup "keystamp" opts)-              ( t, rk ) = BSC.span (/= '-') rs-            in -              ( fromIntegral $ maybe 0 fst $ BSC.readInteger t, BS.drop 1 rk )--        {-# INLINE challange #-}-        -- [i|#{timestamp}-#{fileid}-#{key cfg}-hotlinkthis|]-        challange :: ClientConfig -> ByteString-        challange cfg-          = BS.take 10-          $ hash-          $ BD.toStrictByteString-            (BD.int64Dec timestamp-             <> "-"-             <> BD.byteString fileid-             <> "-"-             <> BD.textUtf8 (key cfg)-             <> "-hotlinkthis")--    serverCmdHandler command rawAdditional commandTime commandKey-      = checkCommandKey-      >> case command of-        StillAlive        -> plainText "I feel FANTASTIC and I'm still alive"-        ThreadedProxyTest -> let-            args = do-              hostname <- lookupParam "hostname" additional-              protocol <- lookupParam "protocol" additional <|> return "http"-              port <- lookupParam "port" additional-              testSize <- readInt @Int64 =<< lookupParam "testsize" additional-              testCount <- readInt =<< lookupParam "testcount" additional-              testTime <- readInt @Int64 =<< lookupParam "testtime" additional-              testKey <- lookupParam "testkey" additional-              return-                ( testCount-                , testSize-                  -- [i|#{protocol}://#{hostname}:#{port}/t/#{testSize}/#{testTime}/#{testKey}/randomkeyhere|]-                , LBS8.unpack-                  $ BD.toLazyByteString-                    (BD.byteString protocol-                     <> "://"-                     <> BD.byteString hostname-                     <> ":"-                     <> BD.byteString port-                     <> "/t/"-                     <> BD.int64Dec testSize-                     <> "/"-                     <> BD.int64Dec testTime-                     <> "/"-                     <> BD.byteString testKey-                     <> "/0")-                )-          in -            case args of-              Nothing -> throw err403-              Just ( testCount, testSize, url ) -> do-                mgr <- embed $ newManager tlsManagerSettings-                case parseRequest url of-                  Nothing  -> throw err403-                  Just req -> do-                    results <- embed $ replicateConcurrently testCount $ runTest testSize req mgr-                    let succeeded = catMaybes results-                    addUpload (testCount * fromIntegral testSize)-                    -- official format: OK:<successful tests>-<total millis of successful tests>-                    plainText [i|OK:#{length succeeded}-#{sum succeeded}|]-        SpeedTest         -> let-            testSize-              = maybe-                1000000-                (fromIntegral . fst)-                (BSC.readInteger =<< lookupParam "testsize" additional)-          in -            do-              addDownload testSize-              return-                $ addHeader @"Content-Length" testSize-                $ Source.fromStepT-                $ bufferSending testSize-        RefreshSettings   -> do-          log Info "Refreshing settings"-          updateSettings =<< fetchSettings-          plainText ""-        StartDownloader   -> do-          chan <- ask @(TChan GalleryTask)-          log Info "Starting gallery downloader"-          void $ embed $ forkIO $ do-            atomically $ writeTChan chan GalleryTask-          plainText ""-        RefreshCerts      -> do-          log Info "Refreshing certificates"-          chan <- ask @(TChan ServerAction)-          void $ embed $ forkIO $ do-            threadDelay (1000000 * 2)-            atomically $ writeTChan chan Cert-          plainText ""-      where-        additional = parseURLParams (encodeUtf8 rawAdditional)--        -- same check as the official client: only the RPC server knows our key-        checkCommandKey = do-          currentTime <- embed getSystemTime-          cfg <- ask @ClientConfig-          let expected-                = hash @Text-                $ encodeUtf8 @Text-                  [i|hentai@home-servercmd-#{renderServerCommand command}-#{rawAdditional}-#{clientId cfg}-#{commandTime}-#{key cfg}|]-          when (abs (fromIntegral commandTime - systemSeconds currentTime) > maxCommandTimeDrift-                || commandKey /= expected)-            $ throw err403--        -- A test succeeds only if the whole body arrives with status 200 within the-        -- timeout (official: FileDownloader + getContentLength() >= testsize).-        runTest :: Int64 -> HC.Request -> Manager -> IO (Maybe Int64)-        runTest testSize req mgr = do-          start <- getMonotonicTime-          ok <- timeout testTimeoutMicros-            $ try @SomeException-            $ withResponse req mgr-            $ \resp -> let-                drain n-                  | n <= 0 = pure True-                  | otherwise = do-                    chunk <- brRead (getResponseBody resp)-                    -- an empty chunk is EOF: the body was shorter than promised-                    if BS.null chunk-                      then pure False-                      else drain (n - fromIntegral (BS.length chunk))-              in-                if getResponseStatusCode resp == 200-                  then drain testSize-                  else pure False-          end <- getMonotonicTime-          pure $ case ok of-            Just (Right True) -> Just (truncate ((end - start) * 1000))-            _ -> Nothing--        {-# INLINE readInt #-}-        {-# SPECIALISE readInt :: ByteString -> Maybe Int #-}-        {-# SPECIALISE readInt :: ByteString -> Maybe Int64 #-}-        readInt :: Num a => ByteString -> Maybe a-        readInt = fmap (fromIntegral . fst) . BSC.readInteger--        {-# INLINE plainText #-}-        plainText t-          = pure-          $ addHeader @"Content-Length" (BS.length t)-          $ Source.fromStepT-          $ Source.Yield t Source.Stop--    statsHandler = readPrometheus--    testHandler testSize testTime (encodeUtf8 @_ @ByteString -> testKey) _ = do-      currentTime <- embed getSystemTime-      cfg <- ask @ClientConfig-      when (abs (testTime - systemSeconds currentTime) > maxTimeDrift) $ throw err403-      when (testKey /= challange cfg) $ throw err403-      addUpload testSize-      return $ addHeader @"Content-Length" testSize $ Source.fromStepT $ bufferSending testSize-      where-        {-# INLINE challange #-}-        -- [i|hentai@home-speedtest-#{testSize}-#{testTime}-#{clientId cfg}-#{key cfg}|]-        challange :: ClientConfig -> ByteString-        challange cfg-          = hash-          $ BD.toStrictByteString-            ("hentai@home-speedtest-"-             <> BD.intDec testSize-             <> "-"-             <> BD.int64Dec testTime-             <> "-"-             <> BD.textUtf8 (clientId cfg)-             <> "-"-             <> BD.textUtf8 (key cfg))--    adminSettingsHandler = show <$> getSettings--    rawHandler req k = case requestMethod req of-      -- Handle HEAD requests with 200 OK and empty body-      "HEAD" -> k $ responseLBS status200 hentaiHeader ""--      _      -> k $ responseLBS status404 [] ""---- | Set up periodic timers for various housekeeping tasks-tictok :: ClientConfig -> IO (IO ())-tictok config = do-  -- heartbeat-  heartbeatHandle <- repeatedTimer (void $ runRPCIO config stillAlive) (mDelay 1)--  -- resource blacklist-  blacklistHandle <- repeatedTimer (do-                                      res <- runRPCIO config (fetchBlacklist 43200)-                                      print res) (mDelay 36)--  return $ do-    stopTimer heartbeatHandle-    stopTimer blacklistHandle--notifyStart :: ClientConfig -> IO ()-notifyStart config = psi-  where-    psi = runRPCIO config startListening >>= \case-      Right (Right True) -> return ()-      _ -> threadDelay 1000000 >> psi---- | Abstract cache runner - wraps the cache effect interpreter--- This allows us to switch between SQLite and R2 backends at runtime-newtype CacheRunner-  = CacheRunner-  { runCacheWith-      :: forall r a. Members '[ Embed IO, Log Message, Reader ClientConfig, Error RPCError ] r-      => Sem (KVStore FileURI StorageResult : r) a-      -> Sem r a-  }---- Create the WAI application with rate limiting-makeApplication :: ServerLoopContext -> Application-makeApplication serverCxt-  = finalMiddleware-  $ normalizeAcceptMiddleware-  $ tracingConnections statsEnv-  $ tracingTimeUsage statsEnv-  $ withHentaiHeaders-  $ serve api (hoistServer api interpretServer server)-  where-    ServerLoopContext-      { loopConfig = cfg-      , loopChannels = ServerChannels { serverActionChan = action, galleryTaskChan = gallery }-      , loopState = ServerState { serverSettings = settings, serverStatsEnv = statsEnv }-      , loopFlags = ServerFlags { disableRateLimit = disableRL, trustProxyHeaders = trustProxy }-      , loopCacheRunner = cacheRunner-      } = serverCxt--    applyRealIp :: Middleware-    applyRealIp-      = if trustProxy-        then realIpHeader "X-Forwarded-For"-        else id--    applyRateLimit :: Middleware-    applyRateLimit-      = if disableRL-        then id-        else rateLimitMiddleware isRpcServer--    -- dual-stack sockets would show RPC servers as ::ffff:a.b.c.d; Warp listens on IPv4 here-    isRpcServer ip = (\st -> Just ip `elem` (parseIPv4 <$> rpcServerIPs st)) <$> readTVarIO settings--    finalMiddleware :: Middleware-    finalMiddleware = applyRealIp . applyRateLimit--    interpretServer :: Sem _ a -> Handler a-    interpretServer-      = Handler-      . ExceptT-      . runFinal-      . asyncToIOFinal-      . embedToFinal @IO-      . errorToIOFinal @ServerError-      . mapError @RPCError (const err500)-      . mapError @ClientError (const err500)-      . mapError @SomeException (const err500)-      . runSettingM settings-      . runReader cfg-      . runReader action-      . runReader statsEnv-      . runReader gallery-      . runLogAction @IO @Message richMessageAction-      . runEHentaiAPI-      . runCacheWith cacheRunner-      . runStats-      . runLocate-      . runGenesis-      . runRPC---- | Core server loop implementing the main request-response cycle.------ This function encapsulates the common logic for running the TLS server,--- handling server actions (reload, cert refresh, shutdown), and coordinating--- graceful shutdown. The loop is parameterized by a recursion continuation--- to support different cache backends that may need different state (e.g.,--- SQLite connection handle).------ The server loop:--- 1. Creates a WAI application with all configured middleware--- 2. Notifies the RPC server that we're ready to serve--- 3. Races between:---    - Receiving control commands on the action channel---    - Running the TLS server to handle HTTP requests--- 4. Handles control commands:---    - 'GracefulShutdown': Stops listening and exits cleanly---    - 'Reload': Exits to allow systemd/supervisor to restart---    - 'Cert': Refreshes TLS certificates and recurses with new certs-------serverLoop :: ServerLoopContext-           -- ^ Server runtime context (config, channels, state, flags, cache)-           -> ( CertificateChain, PrivKey )-           -- ^ Current TLS certificates-           -> IO ( CertificateChain, PrivKey )-serverLoop ctx certs = do-  let ServerLoopContext { loopConfig = cfg-                        , loopChannels = ServerChannels { serverActionChan = chan }-                        , loopState = ServerState { serverSettings = settings }-                        } = ctx--  let app = makeApplication ctx--  st <- readTVarIO settings-  result <- withAsync (notifyStart cfg) $ \_ -> race (atomically $ readTChan chan)-    $ runTLS-      (defaultTlsSettings-       { tlsCredentials     = Just (Credentials [ certs ])-       , onInsecure         = AllowInsecure-       , tlsCiphers         = ciphersuite_all-           ++ [ cipher_ECDHE_ECDSA_AES128CBC_SHA-              , cipher_ECDHE_RSA_AES128CBC_SHA-              , cipher_ECDHE_RSA_AES256CBC_SHA-              ]-       , tlsAllowedVersions = [ TLS10, TLS11, TLS12, TLS13 ]-       })-      (setPort (clientPort st) defaultSettings)-    $ logStdout app--  case result of-    Left GracefulShutdown -> gracefulShutdown cfg-    Left Reload -> exitSuccess-    Left Cert -> refreshCerts cfg-    Right _ -> error "Server terminated unexpectedly"---- | Perform graceful shutdown by notifying RPC server and exiting.------ This function repeatedly calls 'stopListening' until the RPC server--- acknowledges the shutdown request, then exits the process cleanly.-gracefulShutdown :: ClientConfig -> IO a-gracefulShutdown cfg = go (3 :: Int)-  where-    -- The official client sends client_stop once. Never spin when the RPC server is unreachable.-    go n = runEHentaiAPIIO cfg stopListening >>= \case-      Left _-        | n > 1 -> threadDelay 1000000 >> go (n - 1)-      _ -> exitSuccess---- | Refresh TLS certificates with exponential backoff retry logic.------ Attempts to fetch new certificates from the Genesis RPC endpoint.--- On failure, retries up to 3 times with exponential backoff (2^n seconds).--- After 3 failures, triggers graceful shutdown.------ This is called when the server receives a 'Cert' action, typically--- triggered by the periodic certificate refresh timer or manual server command.-refreshCerts :: ClientConfig -> IO ( CertificateChain, PrivKey )-refreshCerts cfg = retry 0-  where-    retry :: Int -> IO ( CertificateChain, PrivKey )-    retry retries = runGenesisIO cfg fetchCertificate >>= \case-      Right (Right newCerts) -> return newCerts-      e -> do-        putStrLn $ "Failed to refresh certs: " <> show e-        if retries >= 3-          then do-            putStrLn "Giving up after 3 retries"-            gracefulShutdown cfg-          else do-            let delaySeconds = 2 ^ retries :: Int-            putStrLn-              $ "Retrying in "-              <> show delaySeconds-              <> " seconds (attempt "-              <> show (retries + 1)-              <> "/3)"-            threadDelay (1000000 * delaySeconds)-            retry (retries + 1)--startServer :: ClientConfig-            -> HathSettings-            -> ( CertificateChain, PrivKey )-            -> TChan ServerAction-            -> Bool-            -> Bool-            -> IO ()-startServer config settings certs chan disableRateLimit trustProxyHeaders = do-  ssVar <- newTVarIO settings-  galleryHandler <- newTChanIO-  statsEnv <- newStatsEnv-  void $ forkIO $ forever $ do-    GalleryTask <- atomically $ readTChan galleryHandler-    result <- try @SomeException-      $ runFinal-      $ embedToFinal @IO-      $ errorToIOFinal @ClientError-      $ errorToIOFinal @RPCError-      $ runReader config-      $ runReader statsEnv-      $ runStats-      $ runLogAction @IO @Message richMessageAction-      $ runEHentaiAPI-      $ runRPC startsDownloader-    case result of-      Left e  -> hPutStrLn stderr $ "Gallery downloader error: " <> show e-      Right _ -> return ()-  print =<< runRPCIO config (fetchBlacklist 259200)-  let ctx runner-        = ServerLoopContext-        { loopConfig      = config-        , loopChannels    = ServerChannels-            { serverActionChan = chan, galleryTaskChan = galleryHandler }-        , loopState       = ServerState { serverSettings = ssVar, serverStatsEnv = statsEnv }-        , loopFlags       = ServerFlags-            { disableRateLimit = disableRateLimit, trustProxyHeaders = trustProxyHeaders }-        , loopCacheRunner = runner-        }-  case cacheBackend config of-    CacheBackendSQLite     -> do-      withConnection (Text.unpack $ cachePath config) $ \conn -> do-        initializeDB conn-        let cacheRunner = CacheRunner { runCacheWith = runCache conn }-        stopTictok <- tictok config-        finally (serverLoopMain (ctx cacheRunner) certs) stopTictok-    CacheBackendR2         -> do-      case r2Config config of-        Nothing    -> error "cacheBackend=r2 but r2Config is missing"-        Just r2Cfg -> do-          r2ConnResult <- mkR2Connection r2Cfg-          case r2ConnResult of-            Left err     -> error $ "Failed to initialize R2: " <> err-            Right r2Conn -> do-              let cacheRunner = CacheRunner { runCacheWith = runCacheR2 r2Conn }-              stopTictok <- tictok config-              finally (serverLoopMain (ctx cacheRunner) certs) stopTictok-    CacheBackendFilesystem -> do-      let root = Text.unpack $ cachePath config-      -- Ensure cache root exists-      createDirectoryIfMissing True root-      let cacheRunner = CacheRunner { runCacheWith = runCacheFilesystem root }-      stopTictok <- tictok config-      finally (serverLoopMain (ctx cacheRunner) certs) stopTictok-  where-    startsDownloader = do-      metadata <- nextGalleryTask-      incDlTask-      case metadata of-        Left "NO_PENDING_DOWNLOADS" -> log Info "No gallery task available"-        Left reason -> log Warning [i|Gallery download refused: #{reason}|]-        Right md -> do-          log Info [i|Starting download for gallery #{galleryID md}|]-          complete <- mapM (verifyAndDownload md) (galleryFileList md)-          when (and complete) (notifyGalleryCompletion md)--    verifyAndDownload md f = do-      let filePath :: FilePath-            = [i|download/#{galleryID md}/#{galleryFileName f}.#{galleryFileExt f}|]-      embed $ createDirectoryIfMissing True (takeDirectory filePath)-      existingBytes <- embed $ do-        existing <- doesFileExist filePath-        if existing-          then BS.readFile filePath-          else return BS.empty-      -- a file generated on demand has no known hash; like the official client, keep it-      if not (BS.null existingBytes) && maybe True (== hash existingBytes) (galleryFileHash f)-        then log Info [i|#{galleryFileName f}.#{galleryFileExt f} already verified, skipping|]-          >> pure True-        else downloadGalleryFile md f >>= \case-          Nothing    -> do-            log Warning [i|Failed to download #{galleryFileName f}.#{galleryFileExt f}|]-            pure False-          Just bytes -> do-            log Info [i|Downloaded #{galleryFileName f}.#{galleryFileExt f}|]-            incDlFile-            addDlBytes (BS.length bytes)-            embed $ BS.writeFile filePath bytes-            pure True--    serverLoopMain :: ServerLoopContext -> ( CertificateChain, PrivKey ) -> IO ()-    serverLoopMain ctx cert = do-      newCert <- serverLoop ctx cert-      serverLoopMain ctx newCert-
− src/SettingM.hs
@@ -1,26 +0,0 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE TemplateHaskell #-}--module SettingM ( SettingM(..), getSettings, updateSettings, runSettingM ) where--import           Polysemy-import           Polysemy.Operators--import           Relude             hiding ( Reader )--import           Types              ( HathSettings )--data SettingM m a where-  -- | Get current H@H settings-  GetSettings :: SettingM m HathSettings-  -- | Update H@H settings (hot update without restart)-  UpdateSettings :: HathSettings -> SettingM m ()--makeSem ''SettingM--{-# INLINE runSettingM #-}--- | Run the Settings effect with a TVar storing the current settings-runSettingM :: Member (Embed IO) r => TVar HathSettings -> SettingM : r @> a -> r @> a-runSettingM tvar = interpret $ \case-  GetSettings        -> embed $ readTVarIO @IO tvar-  UpdateSettings new -> embed $ atomically @IO $ writeTVar tvar new
− src/Stats.hs
@@ -1,173 +0,0 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE TemplateHaskell #-}--module Stats-  ( StatsEnv(..)-  , newStatsEnv-  , Stats(..)-  , addUpload-  , addDownload-  , incServed-  , incFetched-  , updateUptime-  , incActiveConnections-  , decActiveConnections-  , incDlTask-  , incDlFile-  , addDlBytes-  , readPrometheus-  , runStats-  ) where--import           GHC.Clock          ( getMonotonicTime )--import qualified Metrics.Counter    as Counter-import qualified Metrics.Gauge      as Gauge-import           Metrics.Registry   ( newRegistry, registerCounter, registerGauge )-import           Metrics.Snapshot   ( getCachedSnapshot, startSnapshotWorker )--- Custom metrics implementation (replaces prometheus library)-import           Metrics.Types      ( Counter, Gauge, Registry )--import           Polysemy-import           Polysemy.Operators-import           Polysemy.Reader    ( Reader, ask )--import           Relude             hiding ( Reader, ask )--data StatsEnv-  = StatsEnv-  { statsStart :: !Double-  , statsRegistry :: !Registry-  , statsUploadBytesCounter :: !Counter-  , statsDownloadBytesCounter :: !Counter-  , statsServedCounter :: !Counter-  , statsFetchedCounter :: !Counter-  , statsActiveConnectionsGauge :: !Gauge-  , statsUptimeGauge :: !Gauge-  , statsDlTaskCounter :: !Counter-  , statsDlFileCounter :: !Counter-  , statsDlBytesCounter :: !Counter-  , statsErrorCounter :: !Counter-  , statsLatency :: !Gauge-  }--newStatsEnv :: IO StatsEnv-newStatsEnv = do-  t0 <- getMonotonicTime-  registry <- newRegistry-  uploadBytesCounter-    <- registerCounter "hath_cache_sent_bytes_total" "Total bytes sent from cache" registry-  downloadBytesCounter-    <- registerCounter "hath_cache_received_bytes_total" "Total bytes received into cache" registry-  servedCounter <- registerCounter "hath_cache_sent_total" "Total number of cache hits" registry-  fetchedCounter-    <- registerCounter "hath_cache_received_total" "Total number of cache misses" registry-  activeConnectionsGauge-    <- registerGauge "hath_active_connections" "Number of active connections" registry-  uptimeGauge <- registerGauge "hath_uptime_seconds" "Server uptime in seconds" registry-  dlTaskCounter-    <- registerCounter "hath_download_task_count_total" "Total gallery download tasks" registry-  dlFileCounter-    <- registerCounter "hath_download_file_count_total" "Total gallery files downloaded" registry-  dlBytesCounter <- registerCounter-    "hath_download_size_bytes_total"-    "Total bytes downloaded from galleries"-    registry-  errorCounter <- registerCounter "hath_errors_total" "Total errors" registry-  latencyGauge <- registerGauge "hath_latency_seconds" "Request latency in seconds" registry--  -- Start background snapshot worker (updates every 5 seconds)-  startSnapshotWorker registry (5 * 1000000)--  pure-    $ StatsEnv-      t0-      registry-      uploadBytesCounter-      downloadBytesCounter-      servedCounter-      fetchedCounter-      activeConnectionsGauge-      uptimeGauge-      dlTaskCounter-      dlFileCounter-      dlBytesCounter-      errorCounter-      latencyGauge-data Stats m a where-  AddUpload :: Int -> Stats m ()-  AddDownload :: Int -> Stats m ()-  IncServed :: Stats m ()-  IncFetched :: Stats m ()-  ReadPrometheus :: Stats m Text--  -- New uptime metric-  UpdateUptime :: Stats m ()--  -- Active connections-  IncActiveConnections :: Stats m ()-  DecActiveConnections :: Stats m ()--  -- Gallery Downloader metrics-  IncDlTask :: Stats m ()-  IncDlFile :: Stats m ()-  AddDlBytes :: Int -> Stats m ()--  -- Error metrics with labels-  IncError :: Text -> Stats m ()  -- error type label--makeSem ''Stats--runStats :: Members '[ Embed IO, Reader StatsEnv ] r => Stats : r @> a -> r @> a-runStats = interpret $ \case-  AddUpload n -> do-    env <- ask-    embed $ Counter.add n (statsUploadBytesCounter env)-  AddDownload n -> do-    env <- ask-    embed $ Counter.add n (statsDownloadBytesCounter env)-  IncServed -> do-    env <- ask-    embed $ Counter.inc (statsServedCounter env)-  IncFetched -> do-    env <- ask-    embed $ Counter.inc (statsFetchedCounter env)--  -- New uptime metric-  UpdateUptime -> do-    env <- ask-    now <- embed getMonotonicTime-    let elapsed = now - statsStart env-    embed $ Gauge.set (round elapsed) (statsUptimeGauge env)--  -- Active connections-  IncActiveConnections -> do-    env <- ask-    embed $ Gauge.inc (statsActiveConnectionsGauge env)--  DecActiveConnections -> do-    env <- ask-    embed $ Gauge.dec (statsActiveConnectionsGauge env)--  -- Gallery Downloader metrics-  IncDlTask -> do-    env <- ask-    embed $ Counter.inc (statsDlTaskCounter env)-  IncDlFile -> do-    env <- ask-    embed $ Counter.inc (statsDlFileCounter env)-  AddDlBytes n -> do-    env <- ask-    embed $ Counter.add n (statsDlBytesCounter env)--  -- Error metrics with labels-  IncError _errorType -> do-    env <- ask-    embed $ Counter.inc (statsErrorCounter env)--  ReadPrometheus -> do-    env <- ask-    snapshot <- embed $ getCachedSnapshot (statsRegistry env)-    pure $ decodeUtf8 snapshot-
− src/Storage/Database.hs
@@ -1,81 +0,0 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE QuasiQuotes #-}-{-# LANGUAGE TypeOperators #-}--module Storage.Database ( FileRecord(..), initializeDB, runCache, runCachePure ) where--import           Data.String.Interpolate ( i )--import           Database.SQLite.Simple--import           Polysemy-import           Polysemy.KVStore        ( KVStore(..), runKVStorePurely )-import           Polysemy.Operators--import           Relude                  hiding ( Reader, State, ask, evalState, get, modify, put )--import           Types                   ( FileRecord(..)-                                         , FileURI-                                         , StorageResult(..)-                                         , renderFileURIBytes-                                         , renderFileURIText-                                         )--{-# INLINE initializeDB #-}--- | Initialize database with required schema-initializeDB :: Connection -> IO ()-initializeDB conn = do-  execute_-    conn-    [i|CREATE TABLE IF NOT EXISTS files (-        lru_counter INTEGER NOT NULL DEFAULT 0,-        s4 TEXT NOT NULL,-        file_id TEXT PRIMARY KEY,-        file_name TEXT,-        bytes BLOB NOT NULL-    ) strict|]-  execute_ conn "pragma journal_mode=delete"-  execute_ conn "pragma synchronous=normal"-  execute_ conn "pragma temp_store=memory"--    -- execute_ conn "pragma cache_size=100000"-    -- execute_ conn "pragma mmap_size=65536"--- | Run the cache with SQLite-runCache-  :: Members '[ Embed IO ] r => Connection -> KVStore FileURI StorageResult : r @> a -> r @> a-runCache conn = interpret $ \case-  LookupKV uri -> do-    let fid = renderFileURIText uri-    embed-      $ execute conn "UPDATE files SET lru_counter = lru_counter + 1 WHERE file_id = ?" (Only fid)-    results <- embed-      $ query-        conn-        "SELECT lru_counter, s4, file_id, file_name, bytes FROM files WHERE file_id = ?"-        (Only fid)-    case listToMaybe results of-      Just record -> return $ Just (Record record)-      Nothing     -> return Nothing--  UpdateKV _uri (Just (Redirect _url))-    -> error "impossible: cannot store Redirect in Database backend"--  UpdateKV _ (Just (Record record)) -> do-    embed-      $ execute-        conn-        [i|INSERT OR REPLACE INTO files-                    (lru_counter, s4, file_id, file_name, bytes)-                    VALUES (?, ?, ?, ?, ?)|]-        record--  UpdateKV uri Nothing -> let-      fid = renderFileURIBytes uri-    in -      embed $ execute conn "DELETE FROM files WHERE file_id = ?" (Only fid)--{-# INLINE runCachePure #-}-runCachePure :: Map FileURI FileRecord -> KVStore FileURI FileRecord : r @> a -> r @> a-runCachePure initial = fmap snd . runKVStorePurely initial
− src/Storage/Filesystem.hs
@@ -1,103 +0,0 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE GADTs #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeOperators #-}--module Storage.Filesystem ( runCacheFilesystem ) where--import           Colog              ( Message, Severity(Error) )-import           Colog.Polysemy     ( Log )--import           Control.Exception  ( try )--import qualified Data.ByteString    as BS-import           Data.Cache.LRU.IO-import qualified Data.Cache.LRU.IO  as LRU-import qualified Data.Text          as T--import           Polysemy-import           Polysemy.Error     ( Error, throw )-import           Polysemy.KVStore   ( KVStore(..) )-import           Polysemy.Operators-import           Polysemy.Reader    ( Reader, asks )--import           Relude             hiding ( Reader, ask, asks )--import           Storage.Database   ( FileRecord(..) )--import           System.Directory   ( createDirectoryIfMissing, doesFileExist, removeFile )-import           System.FilePath    ( (</>), takeDirectory, takeFileName )--import           Types              ( ClientConfig-                                   , FileURI(..)-                                   , RPCError(..)-                                   , StorageResult(..)-                                   , renderFileURIText-                                   )-import qualified Types              as Ty--import           Utils              ( log )---- | Generate filesystem path from root and FileURI--- Format: root/s4/fileid-fileURIToPath :: FilePath -> FileURI -> FilePath-fileURIToPath root uri = root </> s4 </> fileId-  where-    -- Sanitize to prevent directory traversal-    fileId = takeFileName (toString $ renderFileURIText uri)--    s4     = T.unpack $ T.take 4 (T.pack fileId)---- | Run the cache with Filesystem backend-runCacheFilesystem :: Members '[ Embed IO, Log Message, Reader ClientConfig, Error RPCError ] r-                   => FilePath -- ^ Cache root directory-                   -> KVStore FileURI StorageResult : r @> a-                   -> r @> a-runCacheFilesystem root m = do-  lruSize <- asks Ty.lruCacheSize-  let entries = Just (fromIntegral (lruSize `div` 300000)) -- Approximate number of entries based on avg file size ~300 KiB-  memCache <- embed $ newAtomicLRU entries-  interpret (phi memCache) m-  where-    phi :: forall r r1 x. Members '[ Embed IO, Log Message, Error RPCError ] r1-        => AtomicLRU FileURI FileRecord-        -> KVStore FileURI StorageResult (Sem r) x-        -> Sem r1 x-    phi cache (LookupKV uri) = do-      let path = fileURIToPath root uri-      embed (LRU.lookup uri cache) >>= \case-        Just res -> pure $ Just $ Record res-        Nothing  -> do-          result <- embed $ try @SomeException $ BS.readFile path-          case result of-            Left _      -> pure Nothing -- File not found or read error-            Right bytes -> do-              let recd = Ty.reconstructRecord uri bytes-              embed $ LRU.insert uri recd cache-              pure $ Just $ Record recd--    phi _cache (UpdateKV _uri (Just (Redirect _url)))-      = error "impossible: cannot store Redirect in Filesystem backend"--    phi cache (UpdateKV uri (Just (Record record))) = do-      let path    = fileURIToPath root uri-          content = fileRecordBytes record-      result <- embed $ try @SomeException $ do-        createDirectoryIfMissing True (takeDirectory path)-        BS.writeFile path content-      case result of-        Left err -> do-          log Error ("Filesystem write failed for " <> T.pack path <> ": " <> T.pack (show err))-          throw $ Ty.StorageError (T.pack $ show err)-        Right _  -> embed $ LRU.insert uri record cache--    phi cache (UpdateKV uri Nothing) = do-      let path = fileURIToPath root uri-      result <- embed $ try @SomeException $ do-        exists <- doesFileExist path-        when exists $ removeFile path-      case result of-        Left err -> do-          log Error ("Filesystem delete failed for " <> T.pack path <> ": " <> T.pack (show err))-          throw $ Ty.StorageError (T.pack $ show err)-        Right _  -> embed $ void $ LRU.delete uri cache
− src/Storage/Locate.hs
@@ -1,141 +0,0 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE TemplateHaskell #-}--module Storage.Locate-  ( LocateURI(..)-  , Locate(..)-  , Located(..)-  , locateResource-  , runLocate-  , streamUpstream-  ) where--import           Colog                   ( Message, Severity(Info, Warning) )-import           Colog.Polysemy          ( Log )--import           Control.Concurrent      ( forkIO )-import           Control.Exception       ( finally )--import qualified Data.ByteString         as BS-import qualified Data.ByteString.Short   as SBS-import qualified Data.Map                as Map-import           Data.String.Interpolate ( i )--import           Interface.API           ( EHentaiAPI, openResource )--import qualified Network.HTTP.Client     as HC--import           Polysemy-import           Polysemy.Error          ( Error, catch )-import           Polysemy.Final          ( withWeavingToFinal )-import           Polysemy.KVStore        ( KVStore, lookupKV, updateKV )-import           Polysemy.Operators-import           Polysemy.Reader         ( Reader )--import           Relude                  hiding ( Reader, ask )--import           SettingM                ( SettingM, getSettings )--import           Stats                   ( Stats, addDownload, incFetched )--import           Types--import           Utils                   ( hash, log )--data LocateURI-  = LocateURI { locateURIFilename :: !ByteString-              , locateURI         :: !FileURI-              , locateURIOptions  :: !(Map ByteString ByteString)-              }--data Located-  = Stored !StorageResult-    -- | Cache miss being relayed from upstream: call it with a chunk writer.-    -- It closes the upstream connection and caches the file once it is complete.-  | Streamed !((ByteString -> IO ()) -> IO ())--data Locate m a where-  -- | Fetch a specific resource from the server-  LocateResource :: LocateURI -> Locate m (Maybe Located)--makeSem ''Locate--runLocate-  :: Members-    '[ KVStore FileURI StorageResult-     , SettingM-     , Error RPCError-     , Reader ClientConfig-     , EHentaiAPI-     , Embed IO-     , Final IO-     , Stats-     , Log Message-     ]-    r-  => Locate : r @> a-  -> r @> a-runLocate = interpret $ \case-  LocateResource uri -> do-    settings <- getSettings-    let fileURI = locateURI uri-        s4      = SBS.toShort $ BS.take 4 $ renderFileURIBytes fileURI-        record content-          = Record-            FileRecord-            { fileRecordLRUCounter = 1-            , fileRecordS4         = decodeUtf8 s4-            , fileRecordFileId     = renderFileURIText fileURI-            , fileRecordFileName   = Just $ decodeUtf8 $ locateURIFilename uri-            , fileRecordBytes      = content-            }-    -- if HashSet.member s4 (staticRanges settings) <-- we have a bug in setting parsing. wait for fix-    if True-      then lookupKV fileURI >>= \case-        Just x -> return $ Just $ Stored x-        Nothing -> case ( Map.lookup "fileindex" (locateURIOptions uri)-                        , Map.lookup "xres" (locateURIOptions uri)-                        ) of-          ( Just fileIndex, Just xres ) -> log Info [i|Fetching resource: #{fileURI}|]-            >> openResource fileURI ( fileIndex, xres )-            >>= \case-              Just resp -> do-                incFetched-                -- runs after the handler has returned, once the body has been relayed-                cacheIt <- deferToIO $ \content -> do-                  addDownload (BS.length content)-                  -- a failed cache write only costs us the next hit-                  catch @RPCError (updateKV fileURI $ Just $ record content)-                    $ \e -> log Warning [i|Cache write failed for #{fileURI}: #{e}|]-                pure $ Just $ Streamed $ streamUpstream fileURI resp (void . forkIO . cacheIt)-              Nothing   -> do-                log Warning [i|Failed to fetch resource, sorry for you|]-                pure Nothing-          _ -> do-            log Info [i|Not enough information to fetch resource: #{fileURI}|]-            pure Nothing-      else do-        log Info [i|Resource not in our ranges, rejecting: #{s4}|]-        pure Nothing---- | Relay an upstream body chunk by chunk. If it arrives complete and its SHA-1--- matches, hand the bytes to the continuation (the cache write). The upstream--- connection is closed in every case, including the client going away.-streamUpstream :: FileURI-               -> HC.Response HC.BodyReader-               -> (ByteString -> IO ())-               -> (ByteString -> IO ())-               -> IO ()-streamUpstream fileURI resp whenComplete write = go [] `finally` HC.responseClose resp-  where-    go acc = HC.brRead (HC.responseBody resp) >>= \chunk -> if BS.null chunk-      then let-          content = BS.concat (reverse acc)-        in-          when (BS.length content == fileSize fileURI && hash content == fileHash fileURI)-            $ whenComplete content-      else write chunk >> go (chunk : acc)---- | Lower a Sem action into IO so it can run after the request handler has returned.-deferToIO :: Member (Final IO) r => (x -> Sem r ()) -> Sem r (x -> IO ())-deferToIO k = withWeavingToFinal $ \s lower _ -> pure ((\x -> void (lower (k x <$ s))) <$ s)
− src/Storage/R2.hs
@@ -1,179 +0,0 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE GADTs #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeOperators #-}--module Storage.R2 ( R2Connection(..), mkR2Connection, runCacheR2 ) where--import           Colog                   ( Message, Severity(Error, Warning) )-import           Colog.Polysemy          ( Log )--import qualified Conduit                 as C--import           Control.Exception       ( try )--import qualified Data.ByteArray          as BA-import qualified Data.ByteString         as BS-import           Data.Cache.LRU.IO-import qualified Data.Cache.LRU.IO       as LRU-import qualified Data.Text               as T--import           GHC.Clock               ( getMonotonicTime )--import           Network.HTTP.Client     ( newManager )-import           Network.HTTP.Client.TLS ( tlsManagerSettings )-import           Network.Minio           ( AccessKey(..)-                                         , CredentialValue(..)-                                         , SecretKey(..)-                                         , defaultPutObjectOptions-                                         , presignedGetObjectUrl-                                         , removeObject-                                         , setCreds-                                         , setRegion-                                         )-import qualified Network.Minio           as Minio-import           Network.Minio.S3API     ( headObject )--import           Polysemy-import           Polysemy.Error          ( Error, throw )-import           Polysemy.KVStore        ( KVStore(..) )-import           Polysemy.Operators--import           Relude                  hiding ( Reader, ask, asks )--import           Storage.Database        ( FileRecord(..) )--import           Types                   ( FileURI(..)-                                         , RPCError(..)-                                         , StorageResult(..)-                                         , renderFileURIText-                                         )-import qualified Types                   as Ty--import           Utils                   ( log )---- | R2 connection configuration built at runtime-data R2Connection-  = R2Connection-  { r2MinioConn  :: !Minio.MinioConn-  , r2ConnBucket :: !Text-  , r2UrlCache   :: !(AtomicLRU FileURI ( Double, ByteString ))-    -- ^ Presigned links. Lives here because runCacheR2 runs once per request.-  }---- | Build R2 connection from config + environment variables--- Reads R2_ACCESS_KEY and R2_SECRET_KEY from environment-mkR2Connection :: Ty.R2Config -> IO (Either Text R2Connection)-mkR2Connection cfg = do-  accessKeyMay <- lookupEnv "R2_ACCESS_KEY"-  secretKeyMay <- lookupEnv "R2_SECRET_KEY"-  case ( accessKeyMay, secretKeyMay ) of-    ( Nothing, _ ) -> pure $ Left "R2_ACCESS_KEY environment variable not set"-    ( _, Nothing ) -> pure $ Left "R2_SECRET_KEY environment variable not set"-    ( Just accessKey, Just secretKey ) -> do-      let creds-            = CredentialValue-            { cvAccessKey    = AccessKey $ T.pack accessKey-            , cvSecretKey    = SecretKey $ BA.convert (encodeUtf8 @String @ByteString secretKey)-            , cvSessionToken = Nothing-            }-          -- Use IsString instance to parse endpoint URL-          baseConnInfo = fromString $ T.unpack $ Ty.r2Endpoint cfg-      -- Set credentials and region-      minioConn <- liftIO $ do-        manager <- newManager tlsManagerSettings-        Minio.mkMinioConn (setCreds creds $ setRegion "auto" baseConnInfo) manager-      -- ponytail: fixed capacity; links expire after 10 min, so it only has to hold ~10 min of distinct hits-      urlCache <- newAtomicLRU (Just 100000)-      pure-        $ Right-          R2Connection-          { r2MinioConn = minioConn, r2ConnBucket = Ty.r2Bucket cfg, r2UrlCache = urlCache }---- | Generate R2 object key from FileURI--- Format: s4/hash-size-xres-yres-ext-fileURIToKey :: FileURI -> Text-fileURIToKey uri = s4 <> "/" <> fileId-  where-    fileId = renderFileURIText uri--    s4     = T.take 4 fileId---- | Run the cache with Cloudflare R2--- Read operations silently degrade to Nothing on failure (with warning log)--- Write/delete operations propagate errors-runCacheR2 :: Members '[ Embed IO, Log Message, Error RPCError ] r-           => R2Connection-           -> KVStore FileURI StorageResult : r @> a-           -> r @> a-runCacheR2 conn = interpret (phi (r2UrlCache conn))-  where-    bucket = r2ConnBucket conn--    phi :: forall r r1 x. Members '[ Embed IO, Log Message, Error RPCError ] r1-        => AtomicLRU FileURI ( Double, ByteString )-        -> KVStore FileURI StorageResult (Sem r) x-        -> Sem r1 x-    phi cache (LookupKV uri@(fileURIToKey -> key))         = do-      now <- embed getMonotonicTime-      embed (LRU.lookup uri cache) >>= \case-        Just ( expireTime, link )-          | now < expireTime -> pure $ Just $ Redirect link-        _ -> do-          -- First, check if object exists using HEAD request-          headResult <- embed-            $ try @SomeException-            $ Minio.runMinioWith (r2MinioConn conn)-            $ headObject bucket key []-          case headResult of-            Right (Right _info) -> do-              -- Object exists, try to generate presigned URL (1200 seconds expiry = 20 mins)-              maybeLink <- embed-                (Minio.runMinioWith-                   (r2MinioConn conn)-                   (presignedGetObjectUrl bucket key 1200 [] []))-              case maybeLink of-                Left _minioErr -> pure Nothing-                Right link     -> do-                  let expireTime = now + 600 -- 10 minutes cache TTL to ensure fresh links-                  embed $ LRU.insert uri ( expireTime, link ) cache-                  pure $ Just $ Redirect link-            _ -> do-              -- Object does not exist in R2 storage-              log Warning ("Object not found in R2 storage: " <> key)-              pure Nothing--    phi _cache (UpdateKV _uri (Just (Redirect _url)))-      = error "impossible: cannot store Redirect in R2 backend"--    phi _cache (UpdateKV uri (Just (Record record)))       = do-      let key     = fileURIToKey uri-          content = fileRecordBytes record-          siz     = fromIntegral (BS.length content) :: Int64-          src     = C.yield content-      result <- embed-        $ try @SomeException-        $ Minio.runMinioWith (r2MinioConn conn)-        $ Minio.putObject bucket key src (Just siz) defaultPutObjectOptions-      case result of-        Left err -> do-          log Error ("R2 write failed for " <> key <> ": " <> T.pack (show err))-          throw $ Ty.StorageError (T.pack $ show err)-        Right (Left minioErr) -> do-          log Error ("R2 write failed for " <> key <> ": " <> T.pack (show minioErr))-          throw $ Ty.StorageError (T.pack $ show minioErr)-        Right (Right _) -> pure ()--    phi cache (UpdateKV uri@(fileURIToKey -> key) Nothing) = do-      result <- embed-        $ try @SomeException-        $ Minio.runMinioWith (r2MinioConn conn)-        $ removeObject bucket key-      case result of-        Left err -> do-          log Error ("R2 delete failed for " <> key <> ": " <> T.pack (show err))-          throw $ Ty.StorageError (T.pack $ show err)-        Right (Left minioErr) -> do-          log Error ("R2 delete failed for " <> key <> ": " <> T.pack (show minioErr))-          throw $ Ty.StorageError (T.pack $ show minioErr)-        Right (Right _) -> embed $ void $ LRU.delete uri cache
− src/Types.hs
@@ -1,405 +0,0 @@-{-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE DerivingStrategies #-}-{-# LANGUAGE Strict #-}--module Types-  (  -- * Types-    ClientProxy(..)-  , ClientConfig(..)-  , GalleryMetadata(..)-  , HathSettings(..)-  , GalleryFile(..)-  , RPCError(..)-  , FileURI(..)-  , RPCResponse(..)-  , CacheBackend(..)-  , R2Config(..)-  , FileRecord(..)-  , StorageResult(..)-    -- * Globals-  , hentaiHeader-  , clientBuild-    -- * Default values-  , defaultHathSettings-  , defaultClientConfig-  , emptyFileURI-    -- * Parsing-  , parseSettings-  , parseMetadata-  , readClientConfig-  , parseFileURI-  , renderFileURIBytes-  , renderFileURIText-  , parseRPCResponse-  , parseRPCResponse'-  , verifyRPCResponse-  , reconstructRecord-    -- * Selectors-  , getPayload-  ) where--import qualified Data.ByteString.Char8  as BS-import           Data.Char              ( isAsciiLower, isAsciiUpper, isDigit )-import qualified Data.ByteString.Short  as SBS-import qualified Data.HashSet           as HashSet-import qualified Data.Text              as T--import qualified Mason.Builder          as BD--import           Database.SQLite.Simple ( FromRow, ToRow )--import           Dhall                  ( FromDhall(..), ToDhall(..), auto, input )--import           Network.HTTP.Types     ( HeaderName )--import           Polysemy               ( Member, Sem )-import           Polysemy.Error         ( Error )-import qualified Polysemy.Error         as Error--import           Prelude                ( Show(show) )--import           Relude                 hiding ( show )--import           Utils                  ( hash )--{-# SPECIALISE hentaiHeader :: [ ( HeaderName, Text ) ] #-}-{-# SPECIALISE hentaiHeader :: [ ( HeaderName, ByteString ) ] #-}-hentaiHeader :: IsString a => [ ( HeaderName, a ) ]-hentaiHeader-  = [ ( "Connection", "close" )-    , ( "User-Agent", "Hentai@Home 1.6.6" )-    , ( "Cache-Control", "public, max-age=31536000" )-    , ( "Server", "Genetic Lifeform and Distributed Open Server 1.6.6" )-    , ( "X-Content-Type-Options", "nosniff" )-    ]---- | The protocol level this client implements (official 1.6.6), sent as clientbuild.-clientBuild :: Text-clientBuild = "181"--data ClientProxy-  = ClientProxy { host :: {-# UNPACK #-} !Text-                , port :: {-# UNPACK #-} !Integer-                , auth :: {-# UNPACK #-} !(Maybe ( Text, Text ))-                }-  deriving ( Show, Generic )--instance FromDhall ClientProxy--instance ToDhall ClientProxy---- | Cache backend selection-data CacheBackend = CacheBackendSQLite | CacheBackendR2 | CacheBackendFilesystem-  deriving ( Show, Eq, Generic )--instance FromDhall CacheBackend--instance ToDhall CacheBackend---- | R2 configuration (endpoint and bucket from config, secrets from env)-data R2Config = R2Config { r2Endpoint :: {-# UNPACK #-} !Text, r2Bucket :: {-# UNPACK #-} !Text }-  deriving ( Show, Generic )--instance FromDhall R2Config--instance ToDhall R2Config--data ClientConfig-  = ClientConfig-  { clientId     :: {-# UNPACK #-} !Text-  , key          :: {-# UNPACK #-} !Text-  , version      :: {-# UNPACK #-} !Text -- ^ unused, kept so existing configs load; see 'clientBuild'-  , proxy        :: {-# UNPACK #-} !(Maybe ClientProxy)-  , downloadDir  :: {-# UNPACK #-} !Text-  , cachePath    :: {-# UNPACK #-} !Text-  , cacheBackend :: !CacheBackend-  , r2Config     :: !(Maybe R2Config)-  , lruCacheSize :: {-# UNPACK #-} !Int64-  }-  deriving ( Show, Generic )--instance FromDhall ClientConfig--instance ToDhall ClientConfig--defaultClientConfig :: ClientConfig-defaultClientConfig-  = ClientConfig-  { clientId     = ""-  , key          = ""-  , version      = ""-  , proxy        = Nothing-  , downloadDir  = ""-  , cachePath    = ""-  , cacheBackend = CacheBackendSQLite-  , r2Config     = Nothing-  , lruCacheSize = 100 * 1024 * 1024 -- 100 MiB, roughly 100 MiB / 300 KiB per file = 341 entries-  }--readClientConfig :: Text -> IO ClientConfig-readClientConfig = input auto--data HathSettings-  = HathSettings-  { rpcBaseURL         :: {-# UNPACK #-} !ShortByteString-  , clientHost         :: {-# UNPACK #-} !ShortByteString-  , clientPort         :: {-# UNPACK #-} !Int-  , throttleBytes      :: {-# UNPACK #-} !Int64-  , diskLimitBytes     :: {-# UNPACK #-} !Int64-  , diskRemainingBytes :: {-# UNPACK #-} !Int64-  , cacheNeedsRescan   :: {-# UNPACK #-} !Bool-  , cacheNeedsVerify   :: {-# UNPACK #-} !Bool-  , useLessMemory      :: {-# UNPACK #-} !Bool-  , checkIPOrigin      :: {-# UNPACK #-} !Bool-  , floodControl       :: {-# UNPACK #-} !Bool-  , staticRanges       :: {-# UNPACK #-} !(HashSet ShortByteString)-  , rpcServerIPs       :: ![ ByteString ]-    -- ^ As sent by the server, e.g. "::ffff:192.0.2.10"-  }-  deriving ( Show )--defaultHathSettings :: HathSettings-defaultHathSettings-  = HathSettings-  { rpcBaseURL         = "/15/rpc"-  , clientHost         = ""-  , clientPort         = 0-  , throttleBytes      = 0-  , diskLimitBytes     = 0-  , diskRemainingBytes = 0-  , cacheNeedsRescan   = False-  , cacheNeedsVerify   = False-  , useLessMemory      = False-  , checkIPOrigin      = True-  , floodControl       = True-  , staticRanges       = mempty-  , rpcServerIPs       = []-  }--parseSettings :: [ ByteString ] -> HathSettings-parseSettings-  = foldl' (\s kv -> let-                ( k, rest ) = BS.span (/= '=') kv-                v           = BS.drop 1 rest-              in -                case ( k, readMaybe @Int64 (BS.unpack v) ) of-                  ( "host", _ ) -> s { clientHost = SBS.toShort v }-                  ( "port", Just p ) -> s { clientPort = fromIntegral p }-                  ( "throttle_bytes", Just bytes ) -> s { throttleBytes = bytes }-                  ( "disklimit_bytes", Just bytes ) -> s { diskLimitBytes = bytes }-                  ( "diskremaining_bytes", Just bytes ) -> s { diskRemainingBytes = bytes }-                  ( "static_ranges", _ ) -> s-                    { staticRanges = HashSet.fromList (SBS.toShort <$> BS.split ';' v) }-                  ( "rpc_server_ip", _ ) -> s { rpcServerIPs = BS.split ';' v }-                  _ -> s) defaultHathSettings--data GalleryFile-  = GalleryFile { galleryFilePage  :: {-# UNPACK #-} !Int-                , galleryFileIndex :: {-# UNPACK #-} !Int-                , galleryFileName  :: {-# UNPACK #-} !ByteString-                , galleryFileXRes  :: {-# UNPACK #-} !ByteString-                , galleryFileHash  :: !(Maybe ByteString) -- ^ Nothing: generated on demand, unverifiable-                , galleryFileExt   :: {-# UNPACK #-} !ByteString-                }-  deriving ( Eq )--data GalleryMetadata-  = GalleryMetadata { galleryID        :: {-# UNPACK #-} !Int-                    , galleryFileCount :: {-# UNPACK #-} !Int-                    , galleryMinXRes   :: {-# UNPACK #-} !ByteString-                    , galleryTitle     :: {-# UNPACK #-} !ByteString-                    , galleryFileList  :: {-# UNPACK #-} ![ GalleryFile ]-                    }-  deriving ( Eq )---- | Parse a /16/dl fetchqueue reply (hathdl format). Left carries the reason it was--- refused: NO_PENDING_DOWNLOADS and other non-lists, or a list failing the 1.6.6 checks.--- The file list must carry DLHASH = sha1(client key ++ "\n" ++ line, per file line),--- and names/types must be ones the gallery system can produce, which keeps a forged--- list from writing outside the download directory.-parseMetadata :: ByteString -> ByteString -> Either Text GalleryMetadata-parseMetadata clientKey bytes = case break (== "FILELIST") (BS.split '\n' bytes) of-  ( _, [] ) -> Left $ decodeUtf8 $ BS.strip $ BS.take 64 bytes-  ( header, _ : rest ) -> do-    let fileLines = filter (not . BS.null) (takeWhile (/= "INFORMATION") rest)-        field k = listToMaybe [ BS.drop 1 v | l <- header, let ( f, v ) = BS.break (== ' ') l, f == k ]-    gid <- maybeToRight "invalid GID" $ mfilter (> 0) $ readIntExact =<< field "GID"-    minXRes <- maybeToRight "invalid MINXRES" $ mfilter validXRes $ field "MINXRES"-    when (field "DLHASH" /= Just (hash (clientKey <> foldMap ("\n" <>) fileLines)))-      $ Left "DLHASH does not match the file list"-    files <- traverse parseFile fileLines-    pure-      GalleryMetadata-      { galleryID        = gid-      , galleryFileCount = length files-      , galleryMinXRes   = minXRes-      , galleryTitle     = fromMaybe "" (field "TITLE")-      , galleryFileList  = files-      }-  where-    parseFile line = case BS.split ' ' line of-      [ page, index, xres, sha1, ext, name ]-        | Just p <- readIntExact page-        , Just ix <- readIntExact index-        , validName name-        , ext `elem` [ "jpg", "gif", "png", "webp" ] -> Right-          GalleryFile-          { galleryFilePage  = p-          , galleryFileIndex = ix-          , galleryFileName  = name-          , galleryFileXRes  = xres-          , galleryFileHash  = if sha1 == "unknown" then Nothing else Just sha1-          , galleryFileExt   = ext-          }-      _ -> Left $ "invalid file line: " <> decodeUtf8 line--    validName n = not (BS.null n) && BS.length n <= 250 && not (".." `BS.isInfixOf` n)-      && BS.all (\c -> isAsciiUpper c || isAsciiLower c || isDigit c || c `elem` ("._-" :: String)) n--    validXRes x = x == "org" || (not (BS.null x) && BS.all isDigit x)--    readIntExact s = case BS.readInt s of-      Just ( n, "" ) -> Just n-      _ -> Nothing--data FileURI-  = FileURI { fileHash :: {-# UNPACK #-} !ByteString-            , fileSize :: {-# UNPACK #-} !Int-            , fileXRes :: {-# UNPACK #-} !Int-            , fileYRes :: {-# UNPACK #-} !Int-            , fileExt  :: {-# UNPACK #-} !ByteString-            }-  deriving ( Ord, Eq )--renderFileURIBytes :: FileURI -> ByteString-renderFileURIBytes (FileURI { fileHash, fileSize, fileXRes, fileYRes, fileExt })-  = BD.toStrictByteString-    (BD.byteString fileHash-     <> "-"-     <> BD.intDec fileSize-     <> "-"-     <> BD.intDec fileXRes-     <> "-"-     <> BD.intDec fileYRes-     <> "-"-     <> BD.byteString fileExt)--renderFileURIText :: FileURI -> Text-renderFileURIText = decodeUtf8 . renderFileURIBytes--instance Show FileURI where-  show = toString . renderFileURIText--emptyFileURI :: FileURI-emptyFileURI = FileURI { fileHash = "", fileSize = 0, fileXRes = 0, fileYRes = 0, fileExt = "" }--parseFileURI :: ByteString -> FileURI-parseFileURI bytes = loop 0 0 0 0 0 0-  where-    !len = BS.length bytes--    loop !idx !cnt !i1 !i2 !i3 !i4-      | idx >= len-        = if cnt == (4 :: Int)-          then mkURI i1 i2 i3 i4-          else emptyFileURI-      | BS.index bytes idx == '-' = case cnt of-        0 -> loop (idx + 1) 1 idx 0 0 0-        1 -> loop (idx + 1) 2 i1 idx 0 0-        2 -> loop (idx + 1) 3 i1 i2 idx 0-        3 -> loop (idx + 1) 4 i1 i2 i3 idx-        _ -> emptyFileURI-      | otherwise = loop (idx + 1) cnt i1 i2 i3 i4--    mkURI i1 i2 i3 i4-      = let-          hash = BS.take i1 bytes-          size = BS.take (i2 - i1 - 1) (BS.drop (i1 + 1) bytes)-          xres = BS.take (i3 - i2 - 1) (BS.drop (i2 + 1) bytes)-          yres = BS.take (i4 - i3 - 1) (BS.drop (i3 + 1) bytes)-          ext  = BS.drop (i4 + 1) bytes-        in -          FileURI { fileHash = hash-                  , fileSize = conv size-                  , fileXRes = conv xres-                  , fileYRes = conv yres-                  , fileExt  = ext-                  }--    conv = maybe 0 fst . BS.readInt--data RPCResponse-  = RPCResponse { statusCode :: {-# UNPACK #-} !ByteString, payload :: ![ ByteString ] }-  deriving ( Show, Eq, Generic )--data RPCError-  = EmptyResponse-  | RequestFailure {-# UNPACK #-} !Text  -- Contains the error status code-  | CertificateFailure {-# UNPACK #-} !Text-  | StorageError {-# UNPACK #-} !Text    -- Storage write/delete failure-  deriving ( Show, Eq, Generic )--instance Exception RPCError--{-# INLINE parseRPCResponse #-}--- | Parse an RPC response from a lazy ByteString--- The first line is the status code, followed by the payload lines-parseRPCResponse :: ByteString -> Either RPCError RPCResponse-parseRPCResponse bytes = case BS.lines bytes of-  [] -> Left EmptyResponse-  status : rest -> Right $ RPCResponse { statusCode = status, payload = rest }--{-# INLINE getPayload #-}--- | Get the payload if the response was successful-getPayload :: RPCResponse -> Either RPCError [ ByteString ]-getPayload response-  | statusCode response == "OK" = Right $ payload response-  | otherwise = Left $ RequestFailure $ decodeUtf8 $ statusCode response---- | Check the 1.6.6 RPC signature. A signed reply starts with OK-<time>-<sha1>, the sha1--- covering "<time>-<client key>" then "\n<line>" for every later line. A valid reply--- comes back with that first line reduced to OK; a forged or unsigned OK becomes--- RESPONSE_HASH_FAILED; any other status (FAIL, KEY_EXPIRED, ...) passes through.-verifyRPCResponse :: ByteString -> ByteString -> ByteString-verifyRPCResponse clientKey bytes = case BS.split '\n' bytes of-  status : rest-    | Just ( time, sig ) <- BS.break (== '-') <$> BS.stripPrefix "OK-" status-    , sig == "-" <> hash (time <> "-" <> clientKey <> foldMap ("\n" <>) body)-      -> BS.intercalate "\n" ("OK" : body)-    | "OK" `BS.isPrefixOf` status -> "RESPONSE_HASH_FAILED"-    where-      -- Java's split drops trailing empty lines, and the hash is over what it keeps-      body = reverse $ dropWhile BS.null $ reverse rest-  _ -> bytes--{-# INLINE parseRPCResponse' #-}--- | Parse RPC responses effectfully-parseRPCResponse' :: Member (Error RPCError) r => ByteString -> Sem r [ ByteString ]-parseRPCResponse' bytes = Error.fromEither (getPayload =<< parseRPCResponse bytes)--data FileRecord-  = FileRecord { fileRecordLRUCounter :: {-# UNPACK #-} !Int64-               , fileRecordS4         :: {-# UNPACK #-} !Text-               , fileRecordFileId     :: {-# UNPACK #-} !Text-               , fileRecordFileName   :: {-# UNPACK #-} !(Maybe Text)-               , fileRecordBytes      :: !BS.ByteString-               }-  deriving ( Show, Generic, Eq )--instance FromRow FileRecord--instance ToRow FileRecord---- | Reconstruct FileRecord from FileURI and bytes-reconstructRecord :: FileURI -> ByteString -> FileRecord-reconstructRecord uri bytes-  = FileRecord { fileRecordLRUCounter = 0-               , fileRecordS4         = T.take 4 fileId-               , fileRecordFileId     = fileId-               , fileRecordFileName   = Nothing-               , fileRecordBytes      = bytes-               }-  where-    fileId = renderFileURIText uri--data StorageResult = Record FileRecord | Redirect ByteString
− src/Utils.hs
@@ -1,94 +0,0 @@--module Utils ( module Utils ) where--import           Colog                   ( Message, Msg(..), Severity )-import           Colog.Polysemy          ( Log )-import qualified Colog.Polysemy          as Co--import           Crypto.Hash             ( SHA1 )-import qualified Crypto.Hash             as Crypto--import           Data.ByteArray.Encoding ( Base(Base16), convertToBase )-import qualified Data.ByteString         as BS-import qualified Data.ByteString.Char8   as BS8-import qualified Data.Map.Strict         as Map--import           Network.HTTP.Types      ( mkStatus )-import           Network.Wai             ( Response, responseLBS )--import           Polysemy.Operators--import           Relude--import qualified Servant.Types.SourceT   as Source-import           Servant.Types.SourceT   ( StepT )--type URLParams = Map.Map ByteString ByteString--{-# INLINE parseURLParams #-}--- | Parse URL parameters in the format "key1=value1;key2=value2;key3;key4"------ >>> parseURLParams "key1=value1;key2=value2;key3;key4"-parseURLParams :: ByteString -> URLParams-parseURLParams = foldl' insertPair Map.empty . BS8.split ';'-  where-    insertPair acc pair = case BS8.split '=' pair of-      []           -> acc-      [ key ]      -> Map.insert key mempty acc  -- Parameter without value becomes empty string-      [ key, val ] -> Map.insert key val acc-      key : val    -> error $ "Invalid URL parameter: " <> show key <> "=" <> show val--{-# INLINE lookupParam #-}--- | Lookup a parameter value-lookupParam :: ByteString -> URLParams -> Maybe ByteString-lookupParam = Map.lookup--{-# INLINE hasParam #-}--- | Check if a parameter exists-hasParam :: ByteString -> URLParams -> Bool-hasParam = Map.member--{-# INLINE firstJustM #-}--- | Run effects from left to right and stop at the first successful result.-firstJustM :: Monad m => (a -> m (Maybe b)) -> [ a ] -> m (Maybe b)-firstJustM _ [] = pure Nothing-firstJustM f (x : xs) = f x >>= maybe (firstJustM f xs) (pure . Just)--bufferSending :: Int -> StepT m ByteString-bufferSending 0 = Source.Stop-bufferSending n-  | n >= tcpBufferSize = Source.Yield preallocated (bufferSending (n - tcpBufferSize))-  | otherwise = Source.Yield (BS.take n preallocated) Source.Stop-  where-    tcpBufferSize :: Int-    tcpBufferSize = 1460--    preallocated :: ByteString-    preallocated = BS.replicate tcpBufferSize 54---- msg :: sev -> Text -> Msg sev--- msg sev m = withFrozenCallStack (Msg { msgSeverity = sev, msgStack = callStack, msgText = m })-{-# INLINE log #-}-log :: Severity -> Text -> Log Message -@> ()-log sev msg-  = withFrozenCallStack (Co.log Msg { msgSeverity = sev, msgStack = callStack, msgText = msg })--class HathHash a where-  hash :: ByteString -> a--instance HathHash Text where-  {-# INLINABLE hash #-}-  hash = decodeUtf8 @Text @ByteString . convertToBase Base16 . Crypto.hash @ByteString @SHA1--instance HathHash ByteString where-  {-# INLINABLE hash #-}-  hash = convertToBase Base16 . Crypto.hash @ByteString @SHA1--{-# INLINE tooManyRequestsResponse #-}--- Standard 429 response-tooManyRequestsResponse :: Response-tooManyRequestsResponse-  = responseLBS-    (mkStatus 429 "Too Many Requests")-    [ ( "Content-Type", "text/plain" ) ]-    "Too Many Requests"
− src/Version.hs
@@ -1,39 +0,0 @@-{-# LANGUAGE TemplateHaskell #-}--module Version ( versionInfo ) where--import           Control.Exception   ( catch )--import qualified Data.Text           as T-import           Data.Time.Clock     ( getCurrentTime )-import           Data.Time.Format    ( defaultTimeLocale, formatTime )--import           Language.Haskell.TH ( runIO, stringE )--import           Relude--import           System.Process      ( readProcess )---- | Version information string-versionInfo :: T.Text-versionInfo-  = T.unlines-    [ "hs-hath " <> commitStr <> " (" <> dateStr <> ")"-    , "Build Commit: " <> commitStr-    , "Build Date:   " <> dateStr-    ]-  where-    commitStr :: T.Text-    commitStr-      = T.strip $ T.pack $( runIO (do-                                     c <- catch-                                       (readProcess "git" [ "rev-parse", "--short=7", "HEAD" ] "")-                                       (\(_ :: SomeException) -> return "unknown")-                                     stringE c) )--    dateStr :: T.Text-    dateStr-      = T.strip $ T.pack $( runIO (do-                                     t <- getCurrentTime-                                     let d = formatTime defaultTimeLocale "%Y-%m-%d %H:%M:%S UTC" t-                                     stringE d) )
+ test/APISpec.hs view
@@ -0,0 +1,115 @@+module APISpec ( apiSpecs ) where++import qualified Data.Map.Strict    as Map++import           Hath.Server.API    ( ProxyTest(..)+                                    , Route(..)+                                    , ServerCommand(..)+                                    , parseServerCommand+                                    , route+                                    )++import           Network.HTTP.Types ( Status, status400, status403, status404, statusCode )++import           Relude++import           Test.Hspec++-- | What a path comes to, shown so that failures can be read.+routed :: ByteString -> Text+routed path = case route path of+  Favicon -> "favicon"+  Robots -> "robots"+  File fileId options -> "file " <> decodeUtf8 fileId <> " " <> decodeUtf8 options+  ServerCmd command additional time commandKey+    -> unwords [ "servercmd", command, additional, show time, commandKey ]+  Test size time testKey -> unwords [ "t", show size, show time, decodeUtf8 testKey ]+  Metrics -> "metrics"+  AdminSettings -> "admin"+  Refuse status -> refused status++refused :: Status -> Text+refused status = "refuse " <> show (statusCode status)++apiSpecs :: Spec+apiSpecs = describe "API" $ do+  describe "route" $ do+    it "finds the official paths and ours"+      $ map routed [ "/favicon.ico", "/robots.txt", "/metrics", "/admin/settings" ]+      `shouldBe` [ "favicon", "robots", "metrics", "admin" ]++    it "takes /h/<file id>/<options>, the file name or not"+      $ map+        routed+        [ "/h/id/keystamp=1-a;xres=org/name.jpg", "/h/id/keystamp=1-a", "/h/id/opts/name/more" ]+      `shouldBe` [ "file id keystamp=1-a;xres=org", "file id keystamp=1-a", "file id opts" ]++    it "decodes %3d, as the official client, and nothing else"+      $ map+        routed+        [ "/h/id/keystamp%3d1-a%3Bxres%3dorg", "/servercmd/speed_test/testsize%3d5/7/k" ]+      `shouldBe` [ "file id keystamp=1-a%3Bxres=org", "servercmd speed_test testsize=5 7 k" ]++    it "takes /servercmd/<command>/<additional>/<time>/<key>, additional empty or not"+      $ map routed [ "/servercmd/still_alive//1700000000/k", "/servercmd/STILL_ALIVE/a=1/-5/k/x" ]+      `shouldBe` [ "servercmd still_alive  1700000000 k", "servercmd STILL_ALIVE a=1 -5 k" ]++    it "takes /t/<size>/<time>/<key>, the last part or not"+      $ map routed [ "/t/1000/1700000000/k/0", "/t/1000/1700000000/k" ]+      `shouldBe` [ "t 1000 1700000000 k", "t 1000 1700000000 k" ]++    it "refuses what the official client refuses, as it does" $ do+      map routed [ "/h/id", "/h/id/", "/h", "/t/1000/1700000000", "/t/size/1700000000/k" ]+        `shouldBe` replicate 5 (refused status400)+      map routed [ "/servercmd/still_alive/a/1700000000", "/servercmd/still_alive/a/soon/k" ]+        `shouldBe` replicate 2 (refused status403)+      map routed [ "/", "", "/anything", "/favicon.ico/x", "//h/id/opts", "/H/id/opts" ]+        `shouldBe` replicate 6 (refused status404)++  describe "parseServerCommand" $ do+    let params    = Map.fromList+        proxyTest+          = params+            [ ( "hostname", "192.0.2.1" )+            , ( "port", "443" )+            , ( "testsize", "1000" )+            , ( "testcount", "3" )+            , ( "testtime", "1700000000" )+            , ( "testkey", "k" )+            ]++    it "reads a command name in any case"+      $ map+        (`parseServerCommand` mempty)+        [ "still_alive", "STILL_ALIVE", "Refresh_Settings", "start_downloader", "refresh_certs" ]+      `shouldBe` map+        Just+        [ StillAlive, StillAlive, RefreshSettings, StartDownloader, RefreshCerts ]++    it "knows no other command" $ parseServerCommand "stillalive" mempty `shouldBe` Nothing++    it "sends 1000000 bytes for a speed_test without testsize" $ do+      parseServerCommand "speed_test" mempty `shouldBe` Just (SpeedTest 1000000)+      parseServerCommand "speed_test" (params [ ( "testsize", "25" ) ])+        `shouldBe` Just (SpeedTest 25)++    it "does not run a speed_test whose testsize is not a number"+      $ forM_ [ "", "lots", "25x" ]+      $ \size -> parseServerCommand "speed_test" (params [ ( "testsize", size ) ])+      `shouldBe` Nothing++    it "aims a threaded_proxy_test at the other client's /t/, by http unless told otherwise" $ do+      parseServerCommand "threaded_proxy_test" proxyTest+        `shouldBe` Just+          (ThreadedProxyTest (ProxyTest 3 1000 "http://192.0.2.1:443/t/1000/1700000000/k/0"))+      (fmap proxyTestURL . testOf+       <$> parseServerCommand "threaded_proxy_test" (Map.insert "protocol" "https" proxyTest))+        `shouldBe` Just (Just "https://192.0.2.1:443/t/1000/1700000000/k/0")++    it "does not run a threaded_proxy_test with a parameter missing or not a number"+      $ forM_ [ Map.delete "port" proxyTest, Map.insert "testcount" "x" proxyTest ]+      $ \given -> parseServerCommand "threaded_proxy_test" given `shouldBe` Nothing+  where+    testOf = \case+      ThreadedProxyTest test -> Just test+      _ -> Nothing
+ test/ConnectionLimitSpec.hs view
@@ -0,0 +1,54 @@+module ConnectionLimitSpec ( connectionLimitSpecs ) where++import           Control.Concurrent          ( threadDelay )++import           Hath.Server.ConnectionLimit ( Admission(..)+                                             , admission+                                             , connectionLimit+                                             , overloadDue+                                             )+import           Hath.Settings               ( HathSettings(..), defaultHathSettings )++import           Network.Socket              ( SockAddr(SockAddrInet), tupleToHostAddress )++import           Relude++import           Test.Hspec++connectionLimitSpecs :: Spec+connectionLimitSpecs = describe "Connection limit" $ do+  describe "admission" $ do+    it "admits a connection while no more than the limit are open before it" $ do+      admission 20 20 `shouldBe` AdmittedNearLimit+      admission 20 21 `shouldBe` Refused++    it "is near the limit past 80 % of it" $ do+      admission 20 16 `shouldBe` Admitted+      admission 20 17 `shouldBe` AdmittedNearLimit++  describe "overloadDue" $ do+    it "is due the first time" $ overloadDue 100 Nothing `shouldBe` True++    it "is due again more than 30 s after the last notice" $ do+      overloadDue 100 (Just 70) `shouldBe` False+      overloadDue 100.5 (Just 70) `shouldBe` True++  describe "the Warp hooks"+    $ it+      "refuse readers past the limit, but not the RPC server or local addresses, and notify once"+    $ do+      settings <- newTVarIO+        defaultHathSettings { maxConnectionsOverride = 5, rpcServerIPs = [ "192.0.2.10" ] }+      notices <- newIORef (0 :: Int)+      ( onOpen, onClose ) <- connectionLimit settings (atomicModifyIORef'_ notices (+ 1))+      let peer a b c d = SockAddrInet 443 (tupleToHostAddress ( a, b, c, d ))+          visitor      = peer 198 51 100 7+      -- the sixth finds five open, more than 80 %; the seventh six, more than 5+      replicateM 7 (onOpen visitor) `shouldReturn` replicate 6 True <> [ False ]+      onOpen (peer 192 0 2 10) `shouldReturn` True+      onOpen (peer 127 0 0 1) `shouldReturn` True+      -- back to five open: near the limit again, but within 30 s of the notice+      replicateM_ 4 (onClose visitor)+      onOpen visitor `shouldReturn` True+      threadDelay 100000+      readIORef notices `shouldReturn` 1
− test/FileURIRenderSpec.hs
@@ -1,27 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE OverloadedStrings #-}--module FileURIRenderSpec ( fileURIRenderSpecs, main ) where--import           Relude--import           Test.Hspec--import           Types            ( FileURI, parseFileURI, renderFileURIBytes, renderFileURIText )--main :: IO ()-main = hspec fileURIRenderSpecs--fileURIRenderSpecs :: Spec-fileURIRenderSpecs = describe "FileURI rendering" $ do-  it "round-trips a canonical file id through parse and byte rendering" $ do-    let fid = "abcdef1234567890-12345-1024-768-jpg"-    renderFileURIBytes (parseFileURI fid) `shouldBe` fid--  it "keeps text rendering consistent with byte rendering" $ do-    let uri = parseFileURI "deadbeefcafebabe-42-640-480-png"-    renderFileURIText uri `shouldBe` decodeUtf8 (renderFileURIBytes uri)--  it "keeps Show consistent with canonical text rendering" $ do-    let uri = parseFileURI "feedface12345678-2048-1920-1080-webp"-    show uri `shouldBe` toString (renderFileURIText uri)
+ test/FileURISpec.hs view
@@ -0,0 +1,58 @@+module FileURISpec ( fileURISpecs ) where++import qualified Data.ByteString.Char8 as BS++import           Hath.FileURI          ( FileURI(..), parseFileURI, renderFileURIBytes )++import           Relude++import           Test.Hspec+import           Test.Hspec.QuickCheck ( prop )+import           Test.QuickCheck++-- | A file the official client would accept, with or without a resolution.+fileURI :: Gen FileURI+fileURI = do+  digest <- BS.pack <$> vectorOf 40 (elements ([ '0' .. '9' ] <> [ 'a' .. 'f' ]))+  size <- choose ( 0, 2147483647 )+  ( xres, yres ) <- oneof [ pure ( 0, 0 ), (,) <$> choose ( 1, 99999 ) <*> choose ( 0, 99999 ) ]+  FileURI digest size xres yres+    <$> elements [ "jpg", "png", "gif", "mp4", "wbm", "wbp", "avf", "jxl" ]++hash :: ByteString+hash = "0123456789abcdef0123456789abcdef01234567"++fileURISpecs :: Spec+fileURISpecs = describe "FileURI" $ do+  prop "parses back what it renders" $ forAll fileURI $ \uri+    -> parseFileURI (renderFileURIBytes uri) === Just uri++  it "reads both forms of a file id"+    $ map+      parseFileURI+      [ hash <> "-1000-1280-1707-jpg", hash <> "-1000-wbm", hash <> "-0001000-0-0-png" ]+    `shouldBe` [ Just (FileURI hash 1000 1280 1707 "jpg")+               , Just (FileURI hash 1000 0 0 "wbm")+               , Just (FileURI hash 1000 0 0 "png")+               ]++  it "renders an id without a resolution in three parts"+    $ renderFileURIBytes (FileURI hash 1000 0 0 "mp4") `shouldBe` hash <> "-1000-mp4"++  it "refuses what the official client refuses (HVFile.isValidHVFileid)"+    $ forM_+      [ "0123456789ABCDEF0123456789abcdef01234567-1000-1280-1707-jpg" -- upper-case hash+      , BS.drop 1 hash <> "-1000-1280-1707-jpg"                       -- 39 characters+      , hash <> "-1000-1280-1707-txt"                                 -- unknown type+      , hash <> "-1000-1280-1707-JPG"+      , hash <> "-12345678901-1280-1707-jpg"                          -- 11 digits+      , hash <> "-2147483648-1280-1707-jpg"                           -- more than a Java int+      , hash <> "-1000-128000-1707-jpg"                               -- 6 digits+      , hash <> "-1000-1280-jpg"                                      -- four parts+      , hash <> "-1000--1707-jpg"+      , hash <> "-+1000-1280-1707-jpg"+      , hash <> "-big-1280-1707-jpg"+      , hash <> "-1000-1280-1707-jpg-"+      , ""+      ]+    $ \fileId -> parseFileURI fileId `shouldBe` Nothing
test/FilesystemSpec.hs view
@@ -1,140 +1,109 @@ {-# LANGUAGE DataKinds #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeApplications #-}-{-# LANGUAGE TypeOperators #-}  module FilesystemSpec ( filesystemSpecs ) where -import           Colog                ( LogAction(..), Message )-import           Colog.Polysemy       ( Log, runLogAction )+import           Colog                 ( LogAction(..), Message )+import           Colog.Polysemy        ( Log, runLogAction ) -import           Storage.Database     ( FileRecord(..) )+import qualified Data.ByteString       as BS+import           Data.Time.Clock.POSIX ( posixSecondsToUTCTime ) -import           Polysemy-import           Polysemy.Error       ( Error, runError )-import           Polysemy.KVStore     ( KVStore, lookupKV, updateKV )-import           Polysemy.Reader      ( Reader, runReader )+import           Hath.Cache            ( Cache, CacheHit(..), deleteFile, lookupFile, storeFile )+import           Hath.Cache.Filesystem ( FsCache, newFsCache, runCacheFilesystem, sweep )+import           Hath.Error            ( HathError )+import           Hath.FileURI          ( FileURI(..) )+import           Hath.Protocol         ( sha1Hex ) -import           Storage.Filesystem   ( runCacheFilesystem )+import           Polysemy+import           Polysemy.Error        ( Error, runError ) -import           Relude               hiding ( Reader, runReader )+import           Relude -import           System.Directory     ( doesFileExist )-import           System.FilePath      ( (</>) )-import           System.IO.Temp       ( withSystemTempDirectory )+import           System.Directory      ( createDirectoryIfMissing+                                       , doesFileExist+                                       , setModificationTime+                                       )+import           System.FilePath       ( (</>) )+import           System.IO.Temp        ( withSystemTempDirectory )  import           Test.Hspec -import           Types                ( ClientConfig-                                      , FileURI-                                      , RPCError-                                      , StorageResult(..)-                                      , defaultClientConfig-                                      , parseFileURI-                                      )+uriFor :: ByteString -> FileURI+uriFor content+  = FileURI { fileHash = sha1Hex content+            , fileSize = BS.length content+            , fileXRes = 800+            , fileYRes = 600+            , fileExt  = "jpg"+            } --- | Sample file record for testing-sampleRecord :: FileRecord-sampleRecord = FileRecord-  {-    fileRecordLRUCounter = 0-  ,-    fileRecordS4         = "test"-  ,-    fileRecordFileId     = "testhash1234-1024-800-600-jpg"-  ,-    fileRecordFileName   = Just "test.jpg"-  ,-    fileRecordBytes      = "test content for Filesystem storage"-  }+runFs :: FsCache -> Sem '[ Cache, Log Message, Error HathError, Embed IO ] a -> IO a+runFs fs program = do+  result <- runM+    . runError @HathError+    . runLogAction @IO @Message (LogAction $ const pass)+    . runCacheFilesystem fs+    $ program+  either (\e -> fail $ "unexpected error: " <> show e) pure result --- | Parse a file URI from the sample record-sampleURI :: FileURI-sampleURI = parseFileURI "testhash1234-1024-800-600-jpg"+store :: FsCache -> ByteString -> IO ()+store fs content = runFs fs $ storeFile (uriFor content) content --- | Silence logs for cleaner test output-silentLogAction :: LogAction IO Message-silentLogAction = LogAction $ \_ -> pure ()+served :: FsCache -> ByteString -> IO (Maybe FilePath)+served fs content = runFs fs (lookupFile (uriFor content)) <&> \case+  Just (LocalFile path) -> Just path+  _ -> Nothing --- | Run a program with Filesystem cache effect (silent logging)-runWithFilesystemSilent :: FilePath-                        -> Sem '[KVStore FileURI StorageResult, Log Message, Reader ClientConfig, Error RPCError, Embed IO] a -                        -> IO (Either RPCError a)-runWithFilesystemSilent root program = -  runM-  . runError @RPCError-  . runReader defaultClientConfig-  . runLogAction @IO @Message silentLogAction-  . runCacheFilesystem root-  $ program+withCache :: (FilePath -> FsCache -> IO a) -> IO a+withCache k = withSystemTempDirectory "hath-test" $ \tmp -> do+  let root = tmp </> "cache"+  newFsCache root >>= k root --- | Lookup with explicit type-lookupFile :: Member (KVStore FileURI StorageResult) r => FileURI -> Sem r (Maybe StorageResult)-lookupFile = lookupKV+filesystemSpecs :: Spec+filesystemSpecs = describe "Filesystem cache" $ do+  let content = "test content for Filesystem storage" --- | Update with explicit type-updateFile :: Member (KVStore FileURI StorageResult) r => FileURI -> Maybe StorageResult -> Sem r ()-updateFile = updateKV+  it "stores a file and serves hits straight from disk" $ withCache $ \_ fs -> do+    store fs content+    served fs content >>= \case+      Nothing   -> expectationFailure "expected a hit"+      Just path -> readFileBS path >>= (`shouldBe` content) -filesystemSpecs :: Spec-filesystemSpecs = describe "Filesystem Cache Effect" $ do-  -  it "should store and retrieve a file record" $ do-    withSystemTempDirectory "hath-test" $ \tempDir -> do-      let root = tempDir </> "cache"-      -      -- Store-      storeResult <- runWithFilesystemSilent root $ updateFile sampleURI (Just $ Record sampleRecord)-      storeResult `shouldBe` Right ()-      -      -- Verify file exists on disk (impl detail check)-      -- Path: root/s4/fileid-      -- s4 for sampleURI: "test" (first 4 of hash? No, show uri)-      -- show uri: testhash1234-1024-800-600-jpg-      -- s4: test-      let expectedPath = root </> "test" </> "testhash1234-1024-800-600-jpg"-      exists <- doesFileExist expectedPath-      exists `shouldBe` True+  it "misses a file it never stored" $ withCache $ \_ fs -> served fs content+    >>= (`shouldBe` Nothing) -      -- Retrieve-      retrieveResult <- runWithFilesystemSilent root $ lookupFile sampleURI-      case retrieveResult of-        Left err -> expectationFailure $ "Error: " <> show err-        Right Nothing -> expectationFailure "Expected file record but got Nothing"-        Right (Just (Record record)) -> do-          fileRecordBytes record `shouldBe` fileRecordBytes sampleRecord-        Right (Just (Redirect _)) -> expectationFailure "Expected cached record but got redirect"+  it "drops a truncated file instead of serving it" $ withCache $ \_ fs -> do+    store fs content+    Just path <- served fs content+    writeFileBS path (BS.take 5 content)+    served fs content >>= (`shouldBe` Nothing)+    doesFileExist path >>= (`shouldBe` False) -  it "should return Nothing for non-existent file" $ do-    withSystemTempDirectory "hath-test" $ \tempDir -> do-      let root = tempDir </> "cache"-      let nonExistentURI = parseFileURI "nonexistent999-1024-800-600-jpg"-      -      result <- runWithFilesystemSilent root $ lookupFile nonExistentURI-      case result of-        Left err -> expectationFailure $ "Error: " <> show err-        Right Nothing -> pure ()-        Right (Just _) -> expectationFailure "Expected Nothing but got a record"+  it "deletes a file; deleting one it does not have is no error" $ withCache $ \_ fs -> do+    store fs content+    Just path <- served fs content+    runFs fs $ deleteFile (uriFor content)+    doesFileExist path >>= (`shouldBe` False)+    served fs content >>= (`shouldBe` Nothing)+    runFs fs $ deleteFile (uriFor content) -  it "should delete a file record" $ do-    withSystemTempDirectory "hath-test" $ \tempDir -> do-      let root = tempDir </> "cache"-      -      -- Store first-      _ <- runWithFilesystemSilent root $ updateFile sampleURI (Just $ Record sampleRecord)-      -      -- Delete-      deleteResult <- runWithFilesystemSilent root $ updateFile sampleURI Nothing-      deleteResult `shouldBe` Right ()-      -      -- Verify deleted on disk-      let expectedPath = root </> "test" </> "testhash1234-1024-800-600-jpg"-      exists <- doesFileExist expectedPath-      exists `shouldBe` False-      -      -- Verify lookup returns Nothing-      lookupResult <- runWithFilesystemSilent root $ lookupFile sampleURI-      case lookupResult of-        Left err -> expectationFailure $ "Error: " <> show err-        Right Nothing -> pure ()-        Right (Just _) -> expectationFailure "Expected Nothing after delete"+  it "evicts the least recently served files, and only its own" $ withCache $ \root fs -> do+    let ( a, b, c ) = ( BS.replicate 1000 97, BS.replicate 1000 98, BS.replicate 1000 99 )+    paths <- forM [ a, b, c ] $ \x -> store fs x >> fromMaybe (error "stored") <$> served fs x+    forM_ (zip paths [ 100, 200, 300 ]) $ \( p, t )+      -> setModificationTime p (posixSecondsToUTCTime t)+    -- serving a makes it the newest, so b is now the least recently served+    void $ served fs a+    createDirectoryIfMissing True (root </> "abcd")+    writeFileBS (root </> "notes.txt") "not ours"+    writeFileBS (root </> "abcd" </> "other") "not ours"+    -- 3000 bytes against a 2500 byte limit (evicting down to 2250): one file has to go+    sweep fs 2500 >>= (`shouldBe` ( 1, 1000 ))+    mapM doesFileExist paths >>= (`shouldBe` [ True, False, True ])+    mapM doesFileExist [ root </> "notes.txt", root </> "abcd" </> "other" ]+      >>= (`shouldBe` [ True, True ])++  it "evicts nothing under the limit, or without one" $ withCache $ \_ fs -> do+    store fs content+    sweep fs 0 >>= (`shouldBe` ( 0, 0 ))+    sweep fs 1000000 >>= (`shouldBe` ( 0, 0 ))
+ test/FloodControlSpec.hs view
@@ -0,0 +1,72 @@+module FloodControlSpec ( floodControlSpecs ) where++import qualified Data.Map.Strict          as Map++import           Hath.Server.FloodControl ( IP(..)+                                          , IPMap+                                          , checkRateLimit+                                          , isLocalIP+                                          , newFloodControl+                                          , parseIPv4+                                          , registerHit+                                          )++import           Network.Socket           ( tupleToHostAddress, tupleToHostAddress6 )++import           Relude++import           Test.Hspec+import           Test.Hspec.QuickCheck    ( prop )+import           Test.QuickCheck++visitor :: IP+visitor = IPv4 (tupleToHostAddress ( 203, 0, 113, 7 ))++hits :: [ Double ] -> ( IPMap, [ Bool ] )+hits = foldl' hit ( Map.empty, [] )+  where+    hit ( records, verdicts ) time+      = let+          ( records', allowed ) = registerHit time visitor records+        in +          ( records', verdicts <> [ allowed ] )++floodControlSpecs :: Spec+floodControlSpecs = describe "Flood control" $ do+  it "matches the official client" $ do+    floodControl <- newFloodControl+    let other = IPv4 (tupleToHostAddress ( 203, 0, 113, 8 ))+        t0    = 1000 :: Double+    firstTen <- forM [ 0 .. 9 ] $ \k -> checkRateLimit floodControl visitor (t0 + 0.1 * k)+    firstTen `shouldBe` replicate 10 True+    checkRateLimit floodControl visitor (t0 + 1.0) `shouldReturn` False+    -- blocked for 60 s, then counting starts over+    checkRateLimit floodControl visitor (t0 + 59) `shouldReturn` False+    checkRateLimit floodControl visitor (t0 + 62) `shouldReturn` True+    -- one hit per second drains as fast as it fills+    steady <- forM [ 0 .. 29 ] $ \k -> checkRateLimit floodControl other (t0 + k)+    steady `shouldBe` replicate 30 True++  it "treats loopback, private and IPv4-mapped local addresses as local" $ do+    isLocalIP (IPv4 (tupleToHostAddress ( 127, 0, 0, 1 ))) `shouldBe` True+    isLocalIP (IPv4 (tupleToHostAddress ( 172, 20, 1, 1 ))) `shouldBe` True+    isLocalIP (IPv4 (tupleToHostAddress ( 172, 32, 1, 1 ))) `shouldBe` False+    isLocalIP visitor `shouldBe` False+    isLocalIP (IPv6 (tupleToHostAddress6 ( 0, 0, 0, 0, 0, 0xffff, 0x7f00, 1 ))) `shouldBe` True+    isLocalIP (IPv6 (tupleToHostAddress6 ( 0, 0, 0, 0, 0, 0xffff, 0xcb00, 0x7107 )))+      `shouldBe` False+    isLocalIP (IPv6 (tupleToHostAddress6 ( 0xfd12, 0, 0, 0, 0, 0, 0, 1 ))) `shouldBe` True++  it "parses rpc_server_ip entries" $ do+    parseIPv4 "::ffff:192.0.2.10" `shouldBe` Just (IPv4 (tupleToHostAddress ( 192, 0, 2, 10 )))+    parseIPv4 "198.51.100.20" `shouldBe` Just (IPv4 (tupleToHostAddress ( 198, 51, 100, 20 )))+    parseIPv4 "300.1.1.1" `shouldBe` Nothing+    parseIPv4 "fe80::1" `shouldBe` Nothing++  prop "never blocks a visitor who asks at most once a second"+    $ forAll (listOf (choose ( 1, 30 )))+    $ \gaps -> snd (hits (scanl (+) 1000 gaps)) === (True <$ (1000 : gaps))++  it "refuses a blocked visitor without counting the request" $ do+    let blocked = fst (hits (replicate 11 1000))+    registerHit 1030 visitor blocked `shouldBe` ( blocked, False )
+ test/GallerySpec.hs view
@@ -0,0 +1,197 @@+module GallerySpec ( gallerySpecs ) where++import qualified Data.ByteString.Char8   as BS+import qualified Data.Text               as T++import           Hath.Config             ( ClientConfig(..), defaultClientConfig )+import           Hath.Gallery            ( GalleryFile(..)+                                         , GalleryMetadata(..)+                                         , fallbackDirectoryName+                                         , galleryDirectoryName+                                         , parseMetadata+                                         )+import           Hath.Gallery.Downloader ( anotherRound, downloadQueue )+import           Hath.Protocol           ( sha1Hex )+import           Hath.RPC                ( failureReport )+import           Hath.Settings           ( HathSettings(..), defaultHathSettings, parseSettings )+import           Hath.Stats              ( newStatsEnv )++import           MockRPC++import           Relude++import           System.Directory        ( doesPathExist )+import           System.FilePath         ( (</>) )+import           System.IO.Temp          ( withSystemTempDirectory )++import           Test.Hspec++clientKey :: ByteString+clientKey = mockClientKey defaultMockRPC++-- | A download list: the given header lines, a DLHASH that signs the first list+-- of file lines, a FILELIST that holds the second, and 'information'.+galleryList :: [ ByteString ] -> [ ByteString ] -> [ ByteString ] -> ByteString+galleryList header signedLines fileLines+  = BS.intercalate "\n"+  $ header+  <> [ "DLHASH " <> sha1Hex (clientKey <> foldMap ("\n" <>) signedLines), "", "FILELIST" ]+  <> fileLines+  <> [ "", "INFORMATION" ]+  <> information+  <> [ "", "" ]++-- | What follows INFORMATION: not signed, empty lines included.+information :: [ ByteString ]+information = [ "Title: t", "", "Tags: x" ]++downloadList :: [ ByteString ] -> [ ByteString ] -> ByteString+downloadList = galleryList [ "GID 123", "FILECOUNT 2", "MINXRES org", "TITLE t" ]++-- | A list signed as the server signs it.+signedList :: [ ByteString ] -> ByteString+signedList fileLines = downloadList fileLines fileLines++goodLines :: [ ByteString ]+goodLines = [ "1 99 org " <> sha1Hex "x" <> " jpg 001", "2 98 780 unknown webp p_2" ]++refused :: ByteString -> Bool+refused = isLeft . parseMetadata clientKey++gallery :: Int -> ByteString -> Text -> GalleryMetadata+gallery gid minXRes title+  = GalleryMetadata+  { galleryID          = gid+  , galleryMinXRes     = minXRes+  , galleryTitle       = title+  , galleryInformation = ""+  , galleryFileList    = []+  }++gallerySpecs :: Spec+gallerySpecs = describe "Gallery downloader" $ do+  describe "download lists (1.6.6)" $ do+    it "reads a signed list; a file whose hash is unknown has none"+      $ fmap (\file -> ( galleryFileName file, galleryFileHash file )) . galleryFileList+      <$> parseMetadata clientKey (signedList goodLines)+      `shouldBe` Right [ ( "001", Just (sha1Hex "x") ), ( "p_2", Nothing ) ]++    it "keeps the INFORMATION section, as the official client writes it to galleryinfo.txt"+      $ galleryInformation+      <$> parseMetadata clientKey (signedList goodLines) `shouldBe` Right "Title: t\n\nTags: x\n"++    it "cleans up the title as the official client does: the quirks of its pattern included"+      $ galleryTitle+      <$> parseMetadata+        clientKey+        (galleryList+           [ "GID 1", "MINXRES org", "TITLE \t a*b\"c\\d<e>f  g:|?h ? : | :| x\vy\194\160 " ]+           goodLines+           goodLines)+      `shouldBe` Right "abcdef gh ? : | :| x y\160"++    it "refuses a list without a TITLE"+      $ galleryList [ "GID 1", "MINXRES org" ] goodLines goodLines `shouldSatisfy` refused++    it "refuses a list that differs from what was signed"+      $ downloadList goodLines (take 1 goodLines) `shouldSatisfy` refused++    it "refuses file names that leave the gallery directory" $ do+      signedList [ "1 99 org h jpg ../../.ssh/x" ] `shouldSatisfy` refused+      signedList [ "1 99 org h jpg .." ] `shouldSatisfy` refused++    it "refuses file types the gallery system does not produce"+      $ signedList [ "1 99 org h exe 001" ] `shouldSatisfy` refused++    it "gives the server's reason when there is no list"+      $ galleryID+      <$> parseMetadata clientKey "NO_PENDING_DOWNLOADS\n" `shouldBe` Left "NO_PENDING_DOWNLOADS"++  describe "the gallery's directory" $ do+    it "is named for the title and the gallery, and the resolution unless it is the original" $ do+      galleryDirectoryName 125 (gallery 123 "org" "t") `shouldBe` Just "t [123]"+      galleryDirectoryName 125 (gallery 123 "780" "t") `shouldBe` Just "t [123-780x]"++    it "cuts a long title short, to max_filename_length UTF-16 code units" $ do+      let named maxLength title = galleryDirectoryName maxLength (gallery 123 "org" title)+      named 125 (T.replicate 200 "a") `shouldBe` Just (T.replicate 116 "a" <> "... [123]")+      named 20 (T.replicate 200 "a") `shouldBe` Just "aaaaaaaaaaa... [123]"+      named 125 (T.replicate 119 "a") `shouldBe` Just (T.replicate 119 "a" <> " [123]")+      -- the official client keeps a character whole that the cut would split, one unit over+      named 125 (T.replicate 115 "a" <> "\128512" <> T.replicate 20 "b")+        `shouldBe` Just (T.replicate 115 "a" <> "\128512... [123]")+      named 125 (T.replicate 116 "a" <> "\128512" <> T.replicate 20 "b")+        `shouldBe` Just (T.replicate 116 "a" <> "... [123]")++    it "is not made for a title that would leave the download directory"+      $ galleryDirectoryName 125 (gallery 123 "org" "a/b") `shouldBe` Nothing++    it "falls back on the gallery and the resolution" $ do+      fallbackDirectoryName (gallery 123 "org" "t") `shouldBe` "123"+      fallbackDirectoryName (gallery 123 "780" "t") `shouldBe` "123-780x"++    it "takes max_filename_length from the settings"+      $ maxFilenameLength (parseSettings defaultHathSettings [ "max_filename_length=60" ])+      `shouldBe` 60++  describe "rounds" $ do+    it "go on for at most nine rounds" $ do+      anotherRound 10 8 0 `shouldBe` True+      anotherRound 10 9 0 `shouldBe` False++    it "stop once the failures reach twice the number of files" $ do+      anotherRound 3 1 5 `shouldBe` True+      anotherRound 3 1 6 `shouldBe` False++  describe "the failure report" $ do+    it "lists each failure once"+      $ failureReport [ "a-1-org", "b-2-org", "a-1-org" ] `shouldBe` Just "a-1-org;b-2-org"++    it "is not sent without failures, or with more than 50" $ do+      failureReport [] `shouldBe` Nothing+      failureReport (map show [ 1 .. 50 :: Int ]) `shouldSatisfy` isJust+      failureReport (map show [ 1 .. 51 :: Int ]) `shouldBe` Nothing++  it "works through the queue: each gallery is marked done, whether or not its files came"+    $ withSystemTempDirectory "hath-download"+    $ \downloads -> do+      seen <- newIORef []+      let content = "GALLERY_FILE"+          -- the first gallery's file comes; the second's has another hash+          queue   = \case+            ""      -> galleryList+              [ "GID 1", "MINXRES org", "TITLE A <Test>  Gallery" ]+              [ "1 11 org " <> sha1Hex content <> " jpg 001" ]+              [ "1 11 org " <> sha1Hex content <> " jpg 001" ]+            "1;org" -> galleryList+              [ "GID 2", "MINXRES 780", "TITLE B" ]+              [ "1 21 780 " <> sha1Hex "other" <> " jpg 001" ]+              [ "1 21 780 " <> sha1Hex "other" <> " jpg 001" ]+            _       -> "NO_PENDING_DOWNLOADS"+          config+            = defaultClientConfig { key = decodeUtf8 clientKey, downloadDir = toText downloads }+      stats <- newStatsEnv+      settings <- newTVarIO defaultHathSettings+      outcome <- withMockRPC+        defaultMockRPC+        { mockFileContent = content, mockQueue = queue, mockSeen = atomicModifyIORef'_ seen . (:) }+        (downloadQueue config stats settings)+      outcome `shouldBe` Right ()+      -- the second file fails twice, twice its gallery's number of files, and+      -- is reported once+      reverse+        <$> readIORef seen+        `shouldReturn` [ ( "fetchqueue", "" )+                       , ( "dlfetch", "1;1;11;org;1" )+                       , ( "fetchqueue", "1;org" )+                       , ( "dlfetch", "2;1;21;780;1" )+                       , ( "dlfetch", "2;1;21;780;2" )+                       , ( "dlfails", "localhost-21-780" )+                       , ( "fetchqueue", "2;780" )+                       ]+      readFileBS (downloads </> "A Test Gallery [1]" </> "001.jpg") `shouldReturn` content+      readFileBS (downloads </> "A Test Gallery [1]" </> "galleryinfo.txt")+        `shouldReturn` "Title: t\n\nTags: x\n"+      doesPathExist (downloads </> "B [2-780x]") `shouldReturn` True+      doesPathExist (downloads </> "B [2-780x]" </> "001.jpg") `shouldReturn` False+      doesPathExist (downloads </> "B [2-780x]" </> "galleryinfo.txt") `shouldReturn` False
test/Integration.hs view
@@ -1,1024 +1,742 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE QuasiQuotes #-}-{-# LANGUAGE ScopedTypeVariables #-}-{-# LANGUAGE TypeApplications #-}-{-# LANGUAGE TypeOperators #-}---- | Integration tests for hs-hath full lifecycle-module Integration ( integrationSpecs ) where--import           Control.Concurrent       ( ThreadId, forkIO, killThread, threadDelay )-import           Control.Concurrent.MVar  ( MVar, newEmptyMVar, putMVar, tryTakeMVar )-import           Control.Concurrent.STM   ( TVar, newTVarIO )-import           Control.Exception        ( bracket, catch, SomeException )--import qualified Data.Aeson               as A-import qualified Data.Aeson               as A-import qualified Data.ByteString          as BS-import qualified Data.ByteString.Char8    as BSC-import qualified Data.ByteString.Lazy     as LBS-import qualified Data.Map.Strict          as Map-import           Data.String.Interpolate  ( i )-import qualified Data.Text                as T-import           Data.Text.Encoding       ( decodeUtf8, encodeUtf8 )-import           Data.Time.Clock.System   ( SystemTime(systemSeconds), getSystemTime )-import           Data.X509                ( CertificateChain(..), PrivKey )-import           Data.X509.File           ( readSignedObject, readKeyFile )--import           Storage.Database         ( initializeDB )-import           Database.SQLite.Simple   ( withConnection )--import           Utils                    ( hash )--import           MockRPC--import           Network.HTTP.Client      ( defaultManagerSettings-                                          , httpLbs-                                          , method-                                          , newManager-                                          , parseRequest-                                          , redirectCount-                                          , responseBody-                                          , responseHeaders-                                          , responseStatus-                                          )-import           Network.HTTP.Types       ( status200, status301, status403, status404, status429 )-import qualified Network.Wai.Handler.Warp as Warp--import           Relude                   hiding ( Reader-                                                 , decodeUtf8-                                                 , encodeUtf8-                                                 , newEmptyMVar-                                                 , newTVarIO-                                                 , runReader-                                                 )--import           Server                   ( CacheRunner(..)-                                          , ServerAction(..)-                                          , GalleryTask(..)-                                          , makeApplication-                                          , mkServerLoopContext-                                          , startServer-                                          )----import           Storage.Database         ( runCache )----import           Stats                    ( newStatsEnv )----import           System.Environment       ( setEnv )-import           System.Timeout           ( timeout )--import           Middleware               ( IP(..), checkRateLimit, isLocalIP, parseIPv4 )--import           Network.Socket           ( tupleToHostAddress, tupleToHostAddress6 )--import           Polysemy                 ( interpret )-import           Polysemy.Error           ( throw )-import           Polysemy.KVStore         ( KVStore(..) )----import           Test.Hspec----import           UnliftIO                 ( TChan----                                          , newTChanIO----                                          , tryReadTChan----                                          , writeTChan----                                          )--------import           Types                    ( CacheBackend(..)-                                          , RPCError(StorageError)---                                          , ClientConfig(..)-                                          , GalleryFile(..)-                                          , GalleryMetadata(..)-                                          , HathSettings(..)-                                          , defaultHathSettings-                                          , parseMetadata-                                          , verifyRPCResponse-                                          )---- | Test configuration-testConfig :: Int -> ClientConfig-testConfig port-  = ClientConfig-  { clientId     = "test-client-001"-  , key          = "test-secret-key-12345"-  , version      = "1.0.0-test"-  , proxy        = Nothing-  , downloadDir  = "/tmp/hs-hath-test"-  , cachePath    = "/tmp/hs-hath-test-" <> show port <> ".db"-  , cacheBackend = CacheBackendSQLite-  , r2Config     = Nothing-  , lruCacheSize = 100 * 1024 * 1024-  }---- | A servercmd URL signed the way the RPC server signs it.-serverCmdUrl :: Int -> ClientConfig -> Text -> Text -> Int64 -> String-serverCmdUrl port cfg cmd add ts = [i|http://localhost:#{port}/servercmd/#{cmd}/#{add}/#{ts}/#{sig}|]-  where-    sig = hash @Text $ encodeUtf8 ([i|hentai@home-servercmd-#{cmd}-#{add}-#{clientId cfg}-#{ts}-#{key cfg}|] :: Text)--withTestServerConfigured-  :: Bool-  -> MockRPCConfig-  -> ClientConfig-  -> HathSettings-  -> (Int -> IO a)-  -> IO a-withTestServerConfigured disableRateLimit mockCfg clientCfg settings action-  = withConnection ":memory:" $ \conn -> do-    initializeDB conn-    withRunnerTestServer-      disableRateLimit-      CacheRunner { runCacheWith = runCache conn }-      mockCfg-      clientCfg-      settings-      action--withRunnerTestServer-  :: Bool-  -> CacheRunner-  -> MockRPCConfig-  -> ClientConfig-  -> HathSettings-  -> (Int -> IO a)-  -> IO a-withRunnerTestServer disableRateLimit cacheRunner mockCfg clientCfg settings action = withMockRPC mockCfg $ do-  setEnv "HATH_RPC_HOST" ("localhost:" <> show (mockPort mockCfg))-  serverChan <- newTChanIO-  galleryHandler <- newTChanIO-  ssVar <- newTVarIO settings-  statsEnv <- newStatsEnv-  let ctx = mkServerLoopContext-        clientCfg-        ssVar-        serverChan-        galleryHandler-        disableRateLimit-        False-        cacheRunner-        statsEnv-      app = makeApplication ctx-  Warp.testWithApplication (pure app) action--withTestServer-  :: MockRPCConfig-  -> ClientConfig-  -> HathSettings-  -> (Int -> IO a)-  -> IO a-withTestServer = withTestServerConfigured True--withRateLimitedTestServer-  :: MockRPCConfig-  -> ClientConfig-  -> HathSettings-  -> (Int -> IO a)-  -> IO a-withRateLimitedTestServer = withTestServerConfigured False---- | Helper to load test certificates-loadTestCertificates :: IO (CertificateChain, [PrivKey])-loadTestCertificates = do-  certs <- readSignedObject "test-cert.pem"-  keys <- readKeyFile "test-key.pem"-  return (CertificateChain certs, keys)---- | Test with startServer (includes full initialization)-withStartServer-  :: MockRPCConfig-  -> ClientConfig-  -> HathSettings-  -> (Int -> TChan ServerAction -> IO a)-  -> IO a-withStartServer mockCfg clientCfg settings action = withMockRPC mockCfg $ do-  setEnv "HATH_RPC_HOST" ("localhost:" <> show (mockPort mockCfg))-  (certChain, keys) <- loadTestCertificates-  case keys of-    [] -> error "No private key found in test-key.pem"-    (privKey:_) -> do-      serverChan <- newTChanIO-      let certs = (certChain, privKey)-          port = clientPort settings-      -      -- Start server in background thread (skipVerify = True for tests)-      bracket-        (forkIO $ startServer clientCfg settings certs serverChan True False)-        killThread-        (\_ -> do-          -- Wait a bit for server to initialize-          threadDelay 1000000  -- 1 second-          -- Run the test action-          result <- action port serverChan-          -- Shutdown server gracefully-          atomically $ writeTChan serverChan GracefulShutdown-          threadDelay 500000  -- Wait for shutdown-          return result-        )---- | Integration specs-integrationSpecs :: Spec-integrationSpecs = do-  describe "Full Server Lifecycle" $ do-    it "serves requested file by fetching from upstream (MockRPC)" $ do-      let mockPortNum = 19801-          mockContent = "TEST_FILE_CONTENT_123"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 0-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200-        LBS.toStrict (responseBody response) `shouldBe` mockContent-        let parsedOpts = Map.fromList $ do-              t <- T.splitOn ";" opts-              let (optKey, v) = T.breakOn "=" t-              pure (encodeUtf8 optKey, encodeUtf8 (T.drop 1 v))-        Map.lookup "keystamp" parsedOpts `shouldSatisfy` isJust--    it "rejects requests with invalid keystamp" $ do-      let mockPortNum = 19802-          mockContent = "TEST_FILE_CONTENT_123"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 1-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            invalidKeystampText = "invalid" :: Text-            opts = [i|keystamp=#{invalidKeystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status403-        responseBody response `shouldSatisfy` const True--    it "flood control matches the official client" $ do-      let ip = IPv4 (tupleToHostAddress ( 203, 0, 113, 7 ))-          t0 = 1000 :: Double-      -- ten quick hits pass, the eleventh blocks-      firstTen <- forM [ 0 .. 9 ] $ \k -> checkRateLimit ip (t0 + 0.1 * k)-      firstTen `shouldBe` replicate 10 True-      checkRateLimit ip (t0 + 1.0) `shouldReturn` False-      -- blocked for 60 s, then counting starts over-      checkRateLimit ip (t0 + 59) `shouldReturn` False-      checkRateLimit ip (t0 + 62) `shouldReturn` True-      -- one hit per second drains as fast as it fills-      let ip2 = IPv4 (tupleToHostAddress ( 203, 0, 113, 8 ))-      steady <- forM [ 0 .. 29 ] $ \k -> checkRateLimit ip2 (t0 + k)-      steady `shouldBe` replicate 30 True--    it "treats loopback, private and IPv4-mapped local addresses as local" $ do-      isLocalIP (IPv4 (tupleToHostAddress ( 127, 0, 0, 1 ))) `shouldBe` True-      isLocalIP (IPv4 (tupleToHostAddress ( 172, 20, 1, 1 ))) `shouldBe` True-      isLocalIP (IPv4 (tupleToHostAddress ( 172, 32, 1, 1 ))) `shouldBe` False-      isLocalIP (IPv4 (tupleToHostAddress ( 203, 0, 113, 7 ))) `shouldBe` False-      isLocalIP (IPv6 (tupleToHostAddress6 ( 0, 0, 0, 0, 0, 0xffff, 0x7f00, 1 ))) `shouldBe` True-      isLocalIP (IPv6 (tupleToHostAddress6 ( 0, 0, 0, 0, 0, 0xffff, 0xcb00, 0x7107 ))) `shouldBe` False-      isLocalIP (IPv6 (tupleToHostAddress6 ( 0xfd12, 0, 0, 0, 0, 0, 0, 1 ))) `shouldBe` True--    it "parses rpc_server_ip entries" $ do-      parseIPv4 "::ffff:192.0.2.10" `shouldBe` Just (IPv4 (tupleToHostAddress ( 192, 0, 2, 10 )))-      parseIPv4 "198.51.100.20" `shouldBe` Just (IPv4 (tupleToHostAddress ( 198, 51, 100, 20 )))-      parseIPv4 "300.1.1.1" `shouldBe` Nothing-      parseIPv4 "fe80::1" `shouldBe` Nothing--    it "accepts only RPC replies signed with the client key (1.6.6)" $ do-      let k = "test-secret-key-12345"-          sig t ls = hash @ByteString (t <> "-" <> k <> foldMap ("\n" <>) ls)-      verifyRPCResponse k ("OK-7-" <> sig "7" [ "a=1", "b=2" ] <> "\na=1\nb=2\n")-        `shouldBe` "OK\na=1\nb=2"-      verifyRPCResponse k ("OK-7-" <> sig "7" [ "a=1" ] <> "\na=2") `shouldBe` "RESPONSE_HASH_FAILED"-      verifyRPCResponse k "OK\na=1" `shouldBe` "RESPONSE_HASH_FAILED"-      verifyRPCResponse k "KEY_EXPIRED" `shouldBe` "KEY_EXPIRED"--    it "accepts only signed gallery lists with safe file names (1.6.6)" $ do-      let k = "test-secret-key-12345"-          meta signedFor files = BS.intercalate "\n"-            $ [ "GID 123", "FILECOUNT 2", "MINXRES org"-              , "DLHASH " <> hash (k <> foldMap ("\n" <>) signedFor), "TITLE t", "", "FILELIST" ]-            <> files-            <> [ "", "INFORMATION", "not a file line, not signed" ]-          ok files = meta files files-          good = [ "1 99 org " <> hash @ByteString "x" <> " jpg 001", "2 98 780 unknown webp p_2" ]-          files = fmap (fmap (\f -> ( galleryFileName f, galleryFileHash f )) . galleryFileList)-      files (parseMetadata k (ok good))-        `shouldBe` Right [ ( "001", Just (hash "x") ), ( "p_2", Nothing ) ]-      isLeft (parseMetadata k (meta good (take 1 good))) `shouldBe` True  -- list tampered-      isLeft (parseMetadata k (ok [ "1 99 org h jpg ../../.ssh/x" ])) `shouldBe` True-      isLeft (parseMetadata k (ok [ "1 99 org h jpg .." ])) `shouldBe` True-      isLeft (parseMetadata k (ok [ "1 99 org h exe 001" ])) `shouldBe` True-      galleryID <$> parseMetadata k "NO_PENDING_DOWNLOADS\n" `shouldBe` Left "NO_PENDING_DOWNLOADS"--    it "exempts local addresses from flood control" $ do-      let mockPortNum = 19803-          mockContent = "RL_TEST_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 2-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withRateLimitedTestServer mockConfig clientCfg defaultHathSettings { throttleBytes = 1 } $ \serverPort -> do-        now <- getSystemTime-        let ts = systemSeconds now-            k  = key clientCfg-            goodKeystamp =-              let inner = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-              in [i|#{ts}-#{decodeUtf8 (BS.take 10 (hash @ByteString inner))}|] :: Text-            opts = [i|keystamp=#{goodKeystamp};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]--        manager <- newManager defaultManagerSettings-        req0 <- parseRequest url-        let burstRequests = replicate 13 req0--        responses <- forM burstRequests $ \req -> httpLbs req manager-        let statuses = responseStatus <$> responses-        statuses `shouldBe` replicate 13 status200--    it "returns stats snapshot with non-negative counters" $ do-      let mockPortNum = 19804-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 3--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        manager <- newManager defaultManagerSettings-        req <- parseRequest [i|http://localhost:#{serverPort}/metrics|]-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200-        LBS.toStrict (responseBody resp) `shouldSatisfy` (\body -> "hath_cache_sent_total" `BS.isInfixOf` body)--    it "returns 404 for unknown raw routes" $ do-      let mockPortNum = 19805-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 4--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        manager <- newManager defaultManagerSettings-        req <- parseRequest [i|http://localhost:#{serverPort}/unknown-path|]-        responseStatus <$> httpLbs req manager `shouldReturn` status404--    it "serves /robots.txt with disallow all" $ do-      let mockPortNum = 19806-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 5--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        manager <- newManager defaultManagerSettings-        req <- parseRequest [i|http://localhost:#{serverPort}/robots.txt|]-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200-        LBS.toStrict (responseBody resp) `shouldSatisfy` (\body -> "Disallow" `BS.isInfixOf` body)--    it "redirects /favicon.ico to e-hentai.org" $ do-      let mockPortNum = 19807-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 6--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        manager <- newManager defaultManagerSettings-        req0 <- parseRequest [i|http://localhost:#{serverPort}/favicon.ico|]-        let req = req0 { redirectCount = 0 }-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status301--    it "handles /servercmd/still_alive" $ do-      let mockPortNum = 19808-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 7--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        now <- getSystemTime-        let ts = systemSeconds now-        manager <- newManager defaultManagerSettings-        req <- parseRequest (serverCmdUrl serverPort clientCfg "still_alive" "additional" ts)-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200-        LBS.toStrict (responseBody resp) `shouldSatisfy` (\body -> "FANTASTIC" `BS.isInfixOf` body)--    it "handles /servercmd/speed_test" $ do-      let mockPortNum = 19809-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 8--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        now <- getSystemTime-        let ts = systemSeconds now-        manager <- newManager defaultManagerSettings-        req <- parseRequest (serverCmdUrl serverPort clientCfg "speed_test" "testsize=1000" ts)-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200--    it "handles /t speed test endpoint with valid key" $ do-      let mockPortNum = 19810-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 9-          testSize = 1000 :: Int--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        now <- getSystemTime-        let ts = systemSeconds now-            cid = clientId clientCfg-            k = key clientCfg-            challengeBS = encodeUtf8 $ T.pack $ "hentai@home-speedtest-" <> show testSize <> "-" <> show ts <> "-" <> T.unpack cid <> "-" <> T.unpack k-            testKeyText = hash @Text challengeBS-            urlStr = "http://localhost:" <> show serverPort <> "/t/" <> show testSize <> "/" <> show ts <> "/" <> T.unpack testKeyText <> "/nothing"-        manager <- newManager defaultManagerSettings-        req <- parseRequest urlStr-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200--    it "rejects /t speed test with invalid time drift" $ do-      let mockPortNum = 19811-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 10-          testSize = 1000 :: Int--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        let ts = 1000000 :: Int64-            testKeyStr = "invalid-key" :: String-            url = [i|http://localhost:#{serverPort}/t/#{testSize}/#{ts}/#{testKeyStr}/nothing|] :: String-        manager <- newManager defaultManagerSettings-        req <- parseRequest url-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status403--    it "handles HEAD request with 200" $ do-      let mockPortNum = 19812-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 11--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        manager <- newManager defaultManagerSettings-        req0 <- parseRequest [i|http://localhost:#{serverPort}/anything|]-        let req = req0 { method = "HEAD" }-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200-        responseBody resp `shouldBe` ""--    it "handles /servercmd/refresh_settings" $ do-      let mockPortNum = 19813-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 12--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        now <- getSystemTime-        let ts = systemSeconds now-        manager <- newManager defaultManagerSettings-        req <- parseRequest (serverCmdUrl serverPort clientCfg "refresh_settings" "additional" ts)-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200--    it "handles /servercmd/refresh_certs" $ do-      let mockPortNum = 19814-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 13--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        now <- getSystemTime-        let ts = systemSeconds now-        manager <- newManager defaultManagerSettings-        req <- parseRequest (serverCmdUrl serverPort clientCfg "refresh_certs" "additional" ts)-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200--    it "handles /servercmd/threaded_proxy_test with missing params" $ do-      let mockPortNum = 19815-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 14--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        now <- getSystemTime-        let ts = systemSeconds now-        manager <- newManager defaultManagerSettings-        req <- parseRequest (serverCmdUrl serverPort clientCfg "threaded_proxy_test" "additional" ts)-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status403--    it "rejects unsigned /servercmd" $ do-      let mockConfig = defaultMockConfig { mockPort = 19830 }-          clientCfg  = testConfig 30--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        ts <- systemSeconds <$> getSystemTime-        manager <- newManager defaultManagerSettings-        req <- parseRequest [i|http://localhost:#{serverPort}/servercmd/still_alive/additional/#{ts}/key|]-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status403--    it "reports successful tests for /servercmd/threaded_proxy_test, and does not hang on a failing target" $ do-      let mockConfig = defaultMockConfig { mockPort = 19831 }-          clientCfg  = testConfig 31--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        ts <- systemSeconds <$> getSystemTime-        manager <- newManager defaultManagerSettings-        let testKey = hash @Text $ encodeUtf8 ([i|hentai@home-speedtest-1000-#{ts}-#{clientId clientCfg}-#{key clientCfg}|] :: Text)-            proxyTest k = do-              -- the test server tests itself-              let add = [i|hostname=localhost;protocol=http;port=#{serverPort};testsize=1000;testcount=2;testtime=#{ts};testkey=#{k}|] :: Text-              req <- parseRequest (serverCmdUrl serverPort clientCfg "threaded_proxy_test" add ts)-              resp <- httpLbs req manager-              responseStatus resp `shouldBe` status200-              pure (LBS.toStrict (responseBody resp))--        ok <- proxyTest testKey-        ok `shouldSatisfy` ("OK:2-" `BS.isPrefixOf`)--        -- the target answers 403 with a short body; this used to spin forever-        failed <- timeout 10000000 (proxyTest "wrong-key")-        failed `shouldBe` Just "OK:0-0"--    it "still serves a fetched file when the cache write fails" $ do-      let mockContent  = "WRITE_FAILS_CONTENT"-          mockConfig   = defaultMockConfig { mockPort = 19832, mockFileContent = mockContent }-          clientCfg    = testConfig 32-          fileURIStr   = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"-          brokenWrites = CacheRunner-            { runCacheWith = interpret $ \case-                LookupKV _   -> pure Nothing-                UpdateKV _ _ -> throw (StorageError "write refused")-            }--      withRunnerTestServer True brokenWrites mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        ts <- systemSeconds <$> getSystemTime-        let inner    = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{key clientCfg}-hotlinkthis|]-            keystamp = [i|#{ts}-#{decodeUtf8 (BS.take 10 (hash @ByteString inner))}|] :: Text-        manager <- newManager defaultManagerSettings-        req <- parseRequest [i|http://localhost:#{serverPort}/h/#{fileURIStr}/keystamp=#{keystamp};fileindex=1;xres=1024/f.txt|]-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200-        responseBody resp `shouldBe` LBS.fromStrict mockContent--    it "serves file with jpg extension and correct mime type" $ do-      let mockPortNum = 19816-          mockContent = "FAKE_JPG_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 15-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-jpg"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.jpg|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200-        LBS.toStrict (responseBody response) `shouldBe` mockContent--    it "serves file with png extension" $ do-      let mockPortNum = 19817-          mockContent = "FAKE_PNG_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 16-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-png"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.png|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200--    it "serves file with gif extension" $ do-      let mockPortNum = 19818-          mockContent = "FAKE_GIF_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 17-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-gif"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.gif|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200--    it "serves file with mp4 extension" $ do-      let mockPortNum = 19819-          mockContent = "FAKE_MP4_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 18-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-mp4"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.mp4|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200--    it "handles /servercmd/start_downloader" $ do-      let mockPortNum = 19820-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 19--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        now <- getSystemTime-        let ts = systemSeconds now-        manager <- newManager defaultManagerSettings-        req <- parseRequest (serverCmdUrl serverPort clientCfg "start_downloader" "additional" ts)-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status200--    it "serves file with webm extension" $ do-      let mockPortNum = 19821-          mockContent = "FAKE_WEBM_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 20-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-wbm"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.webm|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200--    it "serves file with webp extension" $ do-      let mockPortNum = 19822-          mockContent = "FAKE_WEBP_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 21-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-wbp"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.webp|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200-        find ((== "Content-Type") . fst) (responseHeaders response) `shouldBe` Just ( "Content-Type", "image/webp" )--    it "serves file with avif extension" $ do-      let mockPortNum = 19823-          mockContent = "FAKE_AVIF_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 22-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-avf"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.avif|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200--    it "serves file with jxl extension" $ do-      let mockPortNum = 19824-          mockContent = "FAKE_JXL_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 23-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-jxl"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.jxl|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200--    it "rejects resource request with time drift at boundary (601 seconds)" $ do-      let mockPortNum = 19825-          mockContent = "TIMEDRIFT_TEST"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 24-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime - 601-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status403-        LBS.toStrict (responseBody response) `shouldSatisfy` (\body -> "time" `BS.isInfixOf` body)--    it "serves resource from cache when already stored" $ do-      let mockPortNum = 19826-          mockContent = "CACHED_CONTENT_TEST"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 25-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        -        -- First request fetches from upstream-        response1 <- httpLbs request manager-        responseStatus response1 `shouldBe` status200-        -        -- Second request should hit cache-        response2 <- httpLbs request manager-        responseStatus response2 `shouldBe` status200-        LBS.toStrict (responseBody response2) `shouldBe` mockContent--    it "rejects resource request with missing fileindex option" $ do-      let mockPortNum = 19827-          mockContent = "MISSING_PARAM_TEST"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 26-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status404--    it "returns 404 when resource not found in cache and no params" $ do-      let mockPortNum = 19828-          mockContent = "NOT_FOUND_TEST"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 27-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText}|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status404--    it "serves resource with default octet-stream mime type for unknown extension" $ do-      let mockPortNum = 19829-          mockContent = "UNKNOWN_EXT_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 28-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-xyz"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/test.unknown|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200-        LBS.toStrict (responseBody response) `shouldBe` mockContent--    it "accepts resource request with future timestamp within drift limit" $ do-      let mockPortNum = 19830-          mockContent = "FUTURE_TIME_TEST"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 29-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime + 500-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text-            url = [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]--        manager <- newManager defaultManagerSettings-        request <- parseRequest url-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200--    it "handles POST request to unknown route with 404" $ do-      let mockPortNum = 19831-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 30--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        manager <- newManager defaultManagerSettings-        req0 <- parseRequest [i|http://localhost:#{serverPort}/unknown-post|]-        let req = req0 { method = "POST" }-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status404--    it "handles GET request to unknown route with 404" $ do-      let mockPortNum = 19832-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 31--      withTestServer mockConfig clientCfg defaultHathSettings $ \serverPort -> do-        manager <- newManager defaultManagerSettings-        req <- parseRequest [i|http://localhost:#{serverPort}/random/path/here|]-        resp <- httpLbs req manager-        responseStatus resp `shouldBe` status404--  -- Tests using startServer to cover initialization code-  describe "Server Initialization with startServer" $ do-    it "starts server successfully and serves requests" $ do-      let mockPortNum = 19900-          mockContent = "INIT_TEST_CONTENT"-          mockConfig  = defaultMockConfig { mockPort = mockPortNum, mockFileContent = mockContent }-          clientCfg   = testConfig 100-          settings = defaultHathSettings { clientPort = 9900 }-          fileURIStr  = hash @Text mockContent <> "-" <> show (BS.length mockContent) <> "-1024-768-txt"--      withStartServer mockConfig clientCfg settings $ \serverPort _chan -> do-        currentTime <- getSystemTime-        let ts = systemSeconds currentTime-            k  = key clientCfg-            challengeInput = encodeUtf8 [i|#{ts}-#{fileURIStr}-#{k}-hotlinkthis|]-            challenge = BS.take 10 (hash @ByteString challengeInput)-            keystampText = [i|#{ts}-#{decodeUtf8 challenge}|] :: Text-            opts = [i|keystamp=#{keystampText};fileindex=1;xres=1024|] :: Text--        -- Use TLS-aware manager to connect to HTTPS server-        manager <- newManager defaultManagerSettings-        -- Try HTTP first (onInsecure = AllowInsecure means HTTP should work)-        request <- parseRequest $ [i|http://localhost:#{serverPort}/h/#{fileURIStr}/#{opts}/testfile.txt|]-        response <- httpLbs request manager--        responseStatus response `shouldBe` status200--    it "initializes database and fetches blacklist on startup" $ do-      let mockPortNum = 19901-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 101-          settings = defaultHathSettings { clientPort = 9901 }--      withStartServer mockConfig clientCfg settings $ \_serverPort _chan -> do-        -- The test passes if startServer successfully:-        -- 1. Calls fetchBlacklist (line 477 in Server.hs)-        -- 2. Initializes database (line 479)-        -- 3. Sets up tictok timers (line 480)-        -- 4. Starts the server loop-        threadDelay 2000000  -- Wait 2 seconds for initialization-        -- If we got here without exceptions, initialization succeeded-        return ()--    it "fetches blacklist via tictok timer" $ do-      let mockPortNum = 19903-          mockConfig  = defaultMockConfig { mockPort = mockPortNum }-          clientCfg   = testConfig 103-          settings = defaultHathSettings { clientPort = 9903 }--      withStartServer mockConfig clientCfg settings $ \_serverPort _chan -> do-        -- tictok sets up three timers:-        -- 1. IP map cleanup (every 1 minute)-        -- 2. Heartbeat stillAlive (every 1 minute)-        -- 3. Blacklist fetch (every 36 minutes)-        -- Wait a bit to ensure timers are set up-        threadDelay 3000000  -- 3 seconds-        -- The RPC mock will receive heartbeat calls-        return ()+module Integration ( integrationSpecs ) where++import           Control.Concurrent        ( forkIO, killThread, threadDelay )+import           Control.Exception         ( IOException, bracket, catch, throwIO, try )++import qualified Data.ByteString           as BS+import qualified Data.List                 as List+import           Data.String.Interpolate   ( i )+import qualified Data.Text                 as T+import           Data.Time.Clock.System    ( SystemTime(systemSeconds), getSystemTime )+import           Data.X509                 ( CertificateChain(..) )+import           Data.X509.File            ( readKeyFile, readSignedObject )++import           Database.SQLite.Simple    ( Connection, SQLError, close, open )++import           Hath.Cache                ( Cache(..), CacheRunner(..) )+import           Hath.Cache.Filesystem     ( newFsCache, runCacheFilesystem )+import           Hath.Cache.SQLite         ( initializeDB, runCacheSQLite )+import           Hath.Config               ( ClientConfig(..), defaultClientConfig )+import           Hath.Error                ( HathError(StorageError) )+import           Hath.Locate               ( heldBytes )+import           Hath.Protocol             ( sha1HexText )+import           Hath.Server               ( ServerAction(..)+                                           , ServerEnv(..)+                                           , ServerOptions(..)+                                           , makeApplication+                                           , startServer+                                           )+import           Hath.Server.FloodControl  ( newFloodControl )+import           Hath.Settings             ( HathSettings(..), defaultHathSettings, parseSettings )+import           Hath.Stats                ( newStatsEnv )++import           MockRPC++import           Network.HTTP.Client       ( HttpException+                                           , Response+                                           , defaultManagerSettings+                                           , httpLbs+                                           , method+                                           , newManager+                                           , parseRequest+                                           , redirectCount+                                           , requestHeaders+                                           , responseBody+                                           , responseHeaders+                                           , responseStatus+                                           )+import           Network.HTTP.Types        ( Header+                                           , HeaderName+                                           , mkStatus+                                           , status200+                                           , status301+                                           , status400+                                           , status403+                                           , status404+                                           , status502+                                           , statusCode+                                           )+import           Network.Socket            ( tupleToHostAddress )+import qualified Network.Socket            as Sock+import           Network.Socket.ByteString ( recv, sendAll )+import qualified Network.TLS               as TLS+import qualified Network.Wai.Handler.Warp  as Warp++import           Polysemy                  ( interpret )+import           Polysemy.Error            ( throw )++import           Relude++import           System.Directory          ( createDirectoryIfMissing )+import           System.FilePath           ( (</>) )+import           System.IO.Temp            ( withSystemTempDirectory )+import           System.Process            ( readProcess )+import           System.Timeout            ( timeout )++import           Test.Hspec++import           UnliftIO                  ( TChan, newTChanIO, writeTChan )++testConfig :: ClientConfig+testConfig+  = defaultClientConfig { clientId    = "test-client-001"+                        , key         = "test-secret-key-12345"+                        , downloadDir = "/tmp/hs-hath-test"+                        }++-- | Run the application on a free port, with the settings the mock RPC server+-- assigns at login: without flood control, not behind a proxy, and with+-- 'metricsToken' set, unless @adjust@ says otherwise.+withAppUsing :: (ServerEnv -> ServerEnv) -> CacheRunner -> MockRPC -> (Int -> IO a) -> IO a+withAppUsing adjust cache mock action = withMockRPC mock $ do+  actions <- newTChanIO+  downloadRequest <- newEmptyMVar+  settings <- newTVarIO (parseSettings defaultHathSettings (loginSettings mock))+  stats <- newStatsEnv+  let app+        = makeApplication+        $ adjust+          ServerEnv { envConfig = testConfig+                    , envSettings = settings+                    , envActions = actions+                    , envDownloadRequest = downloadRequest+                    , envStats = stats+                    , envCache = cache+                    , envFloodControl = Nothing+                    , envTrustProxyHeaders = False+                    , envMetricsToken = Just metricsToken+                    }+  Warp.testWithApplication (pure app) action++metricsToken :: ByteString+metricsToken = "let-me-see"++-- | Behind a proxy on this machine, which says where each request came from,+-- and with flood control on.+behindProxy :: MockRPC -> (Int -> IO a) -> IO a+behindProxy mock action = withMemoryCache $ \cache -> do+  floodControl <- newFloodControl+  withAppUsing+    (\env -> env { envFloodControl = Just floodControl, envTrustProxyHeaders = True })+    cache+    mock+    action++from :: ByteString -> [ Header ]+from address = [ ( "X-Forwarded-For", address ) ]++bearer :: ByteString -> [ Header ]+bearer token = [ ( "Authorization", "Bearer " <> token ) ]++withMock :: MockRPC -> (Int -> IO a) -> IO a+withMock mock action = withMemoryCache $ \cache -> withAppUsing id cache mock action++withApp :: ByteString -> (Int -> IO a) -> IO a+withApp content = withMock defaultMockRPC { mockFileContent = content }++-- | The application over a Filesystem cache that holds a file with this+-- content, given its id. Upstream would send something else, so a match proves+-- the bytes came from disk.+withFilesystemHit :: ByteString -> (Int -> Text -> IO a) -> IO a+withFilesystemHit content action = withSystemTempDirectory "hath-fs" $ \root -> do+  let fileId    = fileIdFor content "jpg"+      directory = root </> toString (T.take 4 fileId)+  createDirectoryIfMissing True directory+  writeFileBS (directory </> toString fileId) content+  fs <- newFsCache root+  withAppUsing+    id+    CacheRunner { runCacheWith = runCacheFilesystem fs }+    defaultMockRPC { mockFileContent = "UPSTREAM_CONTENT" }+    (`action` fileId)++withMemoryCache :: (CacheRunner -> IO a) -> IO a+withMemoryCache use = bracket (open ":memory:") closeWhenIdle $ \conn -> do+  initializeDB conn+  use CacheRunner { runCacheWith = runCacheSQLite conn }++-- | A cache miss is written back after the response has gone out, so a test can+-- finish while that write still holds the database; wait for it (up to a second).+closeWhenIdle :: Connection -> IO ()+closeWhenIdle conn = go (100 :: Int)+  where+    go n = close conn `catch` \(e :: SQLError) -> if n > 0+      then threadDelay 10000 >> go (n - 1)+      else throwIO e++now :: IO Int64+now = systemSeconds <$> getSystemTime++signedWithKey :: Text -> Text+signedWithKey = sha1HexText . encodeUtf8++fileIdFor :: ByteString -> Text -> Text+fileIdFor content ext+  = sha1HexText content <> "-" <> show (BS.length content) <> "-1024-768-" <> ext++filePath :: Text -> Int64 -> Text -> String+filePath fileId time options = [i|/h/#{fileId}/keystamp=#{time}-#{stamp}#{options}/file|]+  where+    stamp = T.take 10 $ signedWithKey [i|#{time}-#{fileId}-#{key testConfig}-hotlinkthis|]++-- | The options that let a client fetch a file it does not have.+fetchable :: Text+fetchable = ";fileindex=1;xres=1024"++commandPath :: Text -> Text -> Int64 -> String+commandPath command additional time+  = [i|/servercmd/#{command}/#{additional}/#{time}/#{commandKey}|]+  where+    commandKey+      = signedWithKey+        [i|hentai@home-servercmd-#{command}-#{additional}-#{clientId testConfig}-#{time}-#{key testConfig}|]++speedTestKeyFor :: Int -> Int64 -> Text+speedTestKeyFor size time+  = signedWithKey+    [i|hentai@home-speedtest-#{size}-#{time}-#{clientId testConfig}-#{key testConfig}|]++request :: ByteString -> Int -> String -> IO (Response LByteString)+request = requestWith []++requestWith :: [ Header ] -> ByteString -> Int -> String -> IO (Response LByteString)+requestWith headers verb port path = do+  manager <- newManager defaultManagerSettings+  req <- parseRequest [i|http://localhost:#{port}#{path}|]+  httpLbs req { method = verb, redirectCount = 0, requestHeaders = headers } manager++fetch :: Int -> String -> IO (Response LByteString)+fetch = request "GET"++fetchWith :: [ Header ] -> Int -> String -> IO (Response LByteString)+fetchWith headers = requestWith headers "GET"++body :: Response LByteString -> ByteString+body = toStrict . responseBody++header :: HeaderName -> Response LByteString -> Maybe ByteString+header name = List.lookup name . responseHeaders++integrationSpecs :: Spec+integrationSpecs = do+  describe "Files" $ do+    it "fetches a file it does not have and serves it" $ do+      let content = "TEST_FILE_CONTENT_123"+      withApp content $ \port -> do+        time <- now+        response <- fetch port (filePath (fileIdFor content "jpg") time fetchable)+        responseStatus response `shouldBe` status200+        body response `shouldBe` content++    it "serves it again once it is cached" $ do+      let content = "CACHED_CONTENT_TEST"+      withApp content $ \port -> do+        time <- now+        let path = filePath (fileIdFor content "jpg") time fetchable+        fetched <- fetch port path+        responseStatus fetched `shouldBe` status200+        cached <- fetch port path+        responseStatus cached `shouldBe` status200+        body cached `shouldBe` content++    it "sends the official client's headers, and no User-Agent" $ withApp "x" $ \port -> do+      time <- now+      response <- fetch port (filePath (fileIdFor "x" "jpg") time fetchable)+      map (`header` response) [ "Server", "Connection", "Cache-Control", "Content-Length" ]+        `shouldBe` map+          Just+          [ "Genetic Lifeform and Distributed Open Server 1.6.6"+          , "close"+          , "public, max-age=31536000"+          , "1"+          ]+      map (`header` response) [ "User-Agent", "X-Content-Type-Options" ]+        `shouldBe` [ Nothing, Nothing ]++    it "serves a Filesystem cache hit from disk"+      $ withFilesystemHit "ON_DISK_CONTENT"+      $ \port fileId -> do+        time <- now+        response <- fetch port (filePath fileId time fetchable)+        responseStatus response `shouldBe` status200+        body response `shouldBe` "ON_DISK_CONTENT"+        header "Content-Length" response `shouldBe` Just "15"+        header "Content-Type" response `shouldBe` Just "image/jpeg"++    it "does not serve even a cached file without its fileindex and xres, as the official client"+      $ withFilesystemHit "ON_DISK_CONTENT"+      $ \port fileId -> do+        time <- now+        response <- fetch port (filePath fileId time "")+        responseStatus response `shouldBe` status404++    it "serves a file id written with leading zeros" $ do+      let content = "LEADING_ZEROS"+          fileId  = sha1HexText content <> "-0013-01024-768-jpg"+      withApp content $ \port -> do+        time <- now+        response <- fetch port (filePath fileId time fetchable)+        responseStatus response `shouldBe` status200+        body response `shouldBe` content++    it "still serves a fetched file when the cache write fails" $ do+      let content      = "WRITE_FAILS_CONTENT"+          brokenWrites = CacheRunner { runCacheWith = interpret $ \case+                                         LookupFile _  -> pure Nothing+                                         StoreFile _ _ -> throw (StorageError "write refused")+                                         DeleteFile _  -> pass }+      withAppUsing id brokenWrites defaultMockRPC { mockFileContent = content } $ \port -> do+        time <- now+        response <- fetch port (filePath (fileIdFor content "jpg") time fetchable)+        responseStatus response `shouldBe` status200+        body response `shouldBe` content++    it "refuses an invalid keystamp" $ withApp "x" $ \port -> do+      response <- fetch port [i|/h/#{fileIdFor "x" "jpg"}/keystamp=invalid#{fetchable}/file|]+      responseStatus response `shouldBe` status403++    it "refuses a keystamp 900 seconds old, with the official error page" $ withApp "x" $ \port+      -> do+        time <- now+        response <- fetch port (filePath (fileIdFor "x" "jpg") (time - 900) fetchable)+        responseStatus response `shouldBe` status403+        body response `shouldBe` "An error has occurred. (403)"+        header "Content-Type" response `shouldBe` Just "text/html; charset=ISO-8859-1"++    -- a few seconds short of 900, for the time the request takes+    forM_ [ -890, 890 ]+      $ \drift -> it [i|accepts a keystamp #{drift} seconds from now|] $ withApp "x" $ \port -> do+        time <- now+        response <- fetch port (filePath (fileIdFor "x" "jpg") (time + drift) fetchable)+        responseStatus response `shouldBe` status200++    it "does not find a file it lacks when the request has no fileindex" $ withApp "x" $ \port -> do+      time <- now+      response <- fetch port (filePath (fileIdFor "x" "jpg") time ";xres=1024")+      responseStatus response `shouldBe` status404++    it "does not find a file it lacks when the request has no options" $ withApp "x" $ \port -> do+      time <- now+      response <- fetch port (filePath (fileIdFor "x" "jpg") time "")+      responseStatus response `shouldBe` status404++    it "does not ask the RPC server about a file id the official client refuses" $ do+      seen <- newIORef []+      withMock defaultMockRPC { mockSeen = atomicModifyIORef'_ seen . (:) . fst } $ \port -> do+        time <- now+        response <- fetch port (filePath (fileIdFor "x" "txt") time fetchable)+        responseStatus response `shouldBe` status404+      readIORef seen `shouldReturn` []++    it "does not find a file when srfetch fails"+      $ withMock defaultMockRPC { mockOutages = 3 }+      $ \port -> do+        time <- now+        response <- fetch port (filePath (fileIdFor "x" "jpg") time fetchable)+        responseStatus response `shouldBe` status404++    it "answers 502 when the source does not send the file" $ withApp "NOT_THE_FILE" $ \port -> do+      time <- now+      response <- fetch port (filePath (fileIdFor "x" "jpg") time fetchable)+      responseStatus response `shouldBe` status502+      body response `shouldBe` "An error has occurred. (502)"++    it "goes on fetching a file whose reader hung up, and caches it" $ do+      let content = encodeUtf8 (T.replicate 200000 "z")+      seen <- newIORef []+      withMock+        defaultMockRPC { mockFileContent  = content+                       , mockPauseHalfway = 500000+                       , mockSeen         = atomicModifyIORef'_ seen . (:) . fst+                       }+        $ \port -> do+          time <- now+          let path = filePath (fileIdFor content "jpg") time fetchable+          bracket (connectLocal port) Sock.close $ \sock -> do+            sendAll sock $ "GET " <> encodeUtf8 path <> " HTTP/1.1\r\nHost: localhost\r\n\r\n"+            -- the start of the reply, then hang up+            void $ recv sock 4096+          -- the pause, and time to reach the cache+          threadDelay 2000000+          body <$> fetch port path `shouldReturn` content+      readIORef seen `shouldReturn` [ "srfetch" ]++    it "drops the reader's connection when upstream sends less than it announced" $ do+      let content = encodeUtf8 (T.replicate 100000 "y")+      seen <- newIORef []+      withMock+        defaultMockRPC { mockFileContent   = content+                       , mockHangUpHalfway = True+                       , mockSeen          = atomicModifyIORef'_ seen . (:) . fst+                       }+        $ \port -> do+          time <- now+          let path = filePath (fileIdFor content "jpg") time fetchable+          -- http-client fails a body that ends short when the connection+          -- closes; left open, it would wait for the rest+          cutShort <- timeout 10000000 $ try @HttpException (fetch port path)+          cutShort `shouldSatisfy` maybe False isLeft+          -- nothing was cached: the next request asks for the file again+          void $ timeout 10000000 $ try @HttpException (fetch port path)+      readIORef seen `shouldReturn` [ "srfetch", "srfetch" ]++    it "holds no memory for a download that has ended, whole or broken off" $ do+      let content = encodeUtf8 (T.replicate 100000 "z")+          ended   = timeout 5000000 $ fix $ \again -> do+            taken <- heldBytes+            unless (taken == 0) $ threadDelay 50000 >> again+      forM_ [ False, True ] $ \broken -> withMock+        defaultMockRPC { mockFileContent = content, mockHangUpHalfway = broken }+        $ \port -> do+          time <- now+          void+            $ timeout 10000000+            $ try @HttpException+            $ fetch port (filePath (fileIdFor content "jpg") time fetchable)+          ended `shouldReturn` Just ()++    it "exempts local addresses from flood control"+      $ behindProxy defaultMockRPC { mockFileContent = "x" }+      $ \port -> do+        time <- now+        responses <- replicateM 13 $ fetch port (filePath (fileIdFor "x" "jpg") time fetchable)+        responseStatus <$> responses `shouldBe` replicate 13 status200++  describe "Flood control"+    $ it "blocks an address after ten requests, whatever they ask for, and no other"+    $ behindProxy defaultMockRPC+    $ \port -> do+      let statuses address+            = map (statusCode . responseStatus)+            <$> replicateM 11 (fetchWith (from address) port "/robots.txt")+      statuses "203.0.113.9" `shouldReturn` replicate 10 200 <> [ 429 ]+      responseStatus+        <$> fetchWith (from "203.0.113.9") port "/favicon.ico"+        `shouldReturn` mkStatus 429 "Too Many Requests"+      responseStatus+        <$> fetchWith (from "203.0.113.10") port "/robots.txt" `shouldReturn` status200++  describe "Server commands" $ do+    it "answers still_alive" $ withApp "x" $ \port -> do+      time <- now+      response <- fetch port (commandPath "still_alive" "additional" time)+      responseStatus response `shouldBe` status200+      body response `shouldBe` "I feel FANTASTIC and I'm still alive"+      header "Content-Type" response `shouldBe` Just "text/html; charset=ISO-8859-1"++    it "takes a command name in any case" $ withApp "x" $ \port -> do+      time <- now+      response <- fetch port (commandPath "Still_Alive" "" time)+      body response `shouldSatisfy` ("FANTASTIC" `BS.isInfixOf`)++    forM_+      [ ( "speed_test", "testsize=1000" )+      , ( "refresh_settings", "additional" )+      , ( "refresh_certs", "additional" )+      , ( "start_downloader", "additional" )+      ]+      $ \( command, additional ) -> it [i|accepts #{command}|] $ withApp "x" $ \port -> do+        time <- now+        response <- fetch port (commandPath command additional time)+        responseStatus response `shouldBe` status200++    it "answers refresh_settings with 200 and nothing when the refresh fails"+      $ withMock defaultMockRPC { mockOutages = 3 }+      $ \port -> do+        time <- now+        response <- fetch port (commandPath "refresh_settings" "" time)+        responseStatus response `shouldBe` status200+        body response `shouldBe` ""++    forM_+      [ ( "an unknown command", "stand_up", "" )+      , ( "threaded_proxy_test without its parameters", "threaded_proxy_test", "additional" )+      , ( "speed_test with a testsize that is not a number", "speed_test", "testsize=lots" )+      ]+      $ \( what :: Text, command, additional )+      -> it [i|answers #{what} with INVALID_COMMAND|] $ withApp "x" $ \port -> do+        time <- now+        response <- fetch port (commandPath command additional time)+        responseStatus response `shouldBe` status200+        body response `shouldBe` "INVALID_COMMAND"++    it "refuses a command without a valid key" $ withApp "x" $ \port -> do+      time <- now+      response <- fetch port [i|/servercmd/still_alive/additional/#{time}/key|]+      responseStatus response `shouldBe` status403++    it "refuses a command with a part missing" $ withApp "x" $ \port -> do+      time <- now+      response <- fetch port [i|/servercmd/still_alive/additional/#{time}|]+      responseStatus response `shouldBe` status403++    it "takes a command from an RPC server's address, and refuses one from another"+      $ behindProxy defaultMockRPC { mockRPCServerIP = "192.0.2.1" }+      $ \port -> do+        time <- now+        let commandFrom address+              = responseStatus+              <$> fetchWith (from address) port (commandPath "still_alive" "" time)+        commandFrom "192.0.2.1" `shouldReturn` status200+        commandFrom "203.0.113.9" `shouldReturn` status403++    it "judges a command from a local address by its key alone, as behind a tunnel"+      $ withMock defaultMockRPC { mockRPCServerIP = "192.0.2.1" }+      $ \port -> do+        time <- now+        response <- fetch port (commandPath "still_alive" "" time)+        responseStatus response `shouldBe` status200+        refused <- fetch port [i|/servercmd/still_alive//#{time}/key|]+        responseStatus refused `shouldBe` status403++    it "reports the successful tests of threaded_proxy_test, and does not hang on a failing target"+      $ withApp "x"+      $ \port -> do+        time <- now+        let proxyTest testKey = do+              -- the server tests itself+              let additional+                    = [i|hostname=localhost;protocol=http;port=#{port};testsize=1000;testcount=2;testtime=#{time};testkey=#{testKey}|]+              response <- fetch port (commandPath "threaded_proxy_test" additional time)+              responseStatus response `shouldBe` status200+              pure (body response)+        ok <- proxyTest (speedTestKeyFor 1000 time)+        ok `shouldSatisfy` ("OK:2-" `BS.isPrefixOf`)+        -- the target answers 403 with a short body; this used to spin forever+        failed <- timeout 10000000 (proxyTest "wrong-key")+        failed `shouldBe` Just "OK:0-0"++  describe "Speed tests" $ do+    forM_ [ "/nothing", "" ] $ \(rest :: Text)+      -> it [i|sends the bytes asked for with a valid key (#{rest})|] $ withApp "x" $ \port -> do+        time <- now+        response <- fetch port [i|/t/1000/#{time}/#{speedTestKeyFor 1000 time}#{rest}|]+        responseStatus response `shouldBe` status200+        BS.length (body response) `shouldBe` 1000++    it "refuses a test from another time" $ withApp "x" $ \port -> do+      response <- fetch port "/t/1000/1000000/invalid-key/nothing"+      responseStatus response `shouldBe` status403++    it "refuses a test 301 seconds old" $ withApp "x" $ \port -> do+      time <- subtract 301 <$> now+      response <- fetch port [i|/t/1000/#{time}/#{speedTestKeyFor 1000 time}|]+      responseStatus response `shouldBe` status403++    it "does not understand a test with a part missing" $ withApp "x" $ \port -> do+      time <- now+      response <- fetch port [i|/t/1000/#{time}|]+      responseStatus response `shouldBe` status400++  describe "Other routes" $ do+    it "reports metrics to whoever sends the token" $ withApp "x" $ \port -> do+      response <- fetchWith (bearer metricsToken) port "/metrics"+      responseStatus response `shouldBe` status200+      body response `shouldSatisfy` ("hath_cache_sent_total" `BS.isInfixOf`)+      -- a scrape must not be cached like a file+      header "Cache-Control" response `shouldBe` Nothing++    it "shows the settings to whoever sends the token" $ withApp "x" $ \port -> do+      response <- fetchWith (bearer metricsToken) port "/admin/settings"+      responseStatus response `shouldBe` status200+      body response `shouldSatisfy` ("rpcServerIPs" `BS.isInfixOf`)++    forM_ [ "/metrics", "/admin/settings" ] $ \path -> do+      it [i|refuses #{path} without the token|] $ withApp "x" $ \port -> do+        responseStatus <$> fetch port path `shouldReturn` status403+        responseStatus <$> fetchWith (bearer "let-me-se") port path `shouldReturn` status403+        responseStatus <$> fetchWith (bearer "") port path `shouldReturn` status403++      it [i|has no #{path} when no token is set|] $ withMemoryCache $ \cache+        -> withAppUsing (\env -> env { envMetricsToken = Nothing }) cache defaultMockRPC $ \port+        -> responseStatus <$> fetchWith (bearer "") port path `shouldReturn` status404++    it "disallows robots" $ withApp "x" $ \port -> do+      response <- fetch port "/robots.txt"+      responseStatus response `shouldBe` status200+      body response `shouldBe` "User-agent: *\nDisallow: /"+      header "Content-Type" response `shouldBe` Just "text/plain; charset=ISO-8859-1"++    it "redirects /favicon.ico to e-hentai.org" $ withApp "x" $ \port -> do+      response <- fetch port "/favicon.ico"+      responseStatus response `shouldBe` status301+      header "Location" response `shouldBe` Just "https://e-hentai.org/favicon.ico"++    forM_+      [ ( "GET", "/unknown-path", status404 )+      , ( "GET", "/random/path/here", status404 )+      , ( "GET", "/h/only-an-id", status400 )+      , ( "POST", "/unknown-post", status400 )+      , ( "PUT", "/robots.txt", status400 )+      ]+      $ \( verb, path, status )+      -> it [i|answers #{verb} #{path} with #{statusCode status}|] $ withApp "x" $ \port -> do+        response <- request verb port path+        responseStatus response `shouldBe` status+        body response `shouldBe` [i|An error has occurred. (#{statusCode status})|]++  describe "HEAD" $ do+    it "is answered as GET, without the body" $ withApp "CONTENT" $ \port -> do+      time <- now+      let path = filePath (fileIdFor "CONTENT" "png") time fetchable+      response <- request "HEAD" port path+      responseStatus response `shouldBe` status200+      map (`header` response) [ "Content-Type", "Content-Length" ]+        `shouldBe` [ Just "image/png", Just "7" ]+      body response `shouldBe` ""++    it "goes through the same checks" $ withApp "x" $ \port -> do+      time <- now+      refused <- request "HEAD" port (filePath (fileIdFor "x" "jpg") (time - 900) fetchable)+      responseStatus refused `shouldBe` status403+      unknown <- request "HEAD" port "/anything"+      responseStatus unknown `shouldBe` status404+      header "Content-Length" unknown `shouldBe` Just "28"+      body unknown `shouldBe` ""++    it "of a file it lacks fetches it, as the official client" $ do+      seen <- newIORef []+      withMock+        defaultMockRPC { mockFileContent = "x", mockSeen = atomicModifyIORef'_ seen . (:) . fst }+        $ \port -> do+          time <- now+          let path = filePath (fileIdFor "x" "jpg") time fetchable+          responseStatus <$> request "HEAD" port path `shouldReturn` status200+          -- the fetch finishes on its own; give it a moment to reach the cache+          threadDelay 1000000+          body <$> fetch port path `shouldReturn` "x"+      readIORef seen `shouldReturn` [ "srfetch" ]++  describe "The TLS server" $ do+    it "starts, and serves a file" $ do+      let content = "INIT_TEST_CONTENT"+      withServer (const defaultMockRPC { mockFileContent = content }) $ \port _ -> do+        time <- now+        -- plain HTTP is allowed on the TLS port+        response <- fetch port (filePath (fileIdFor content "jpg") time fetchable)+        responseStatus response `shouldBe` status200+        body response `shouldBe` content++    it "swaps certificates in place: open connections survive, new handshakes get the new one"+      $ withSystemTempDirectory "hath-cert"+      $ \dir -> do+        let openssl = void . flip (readProcess "openssl") ""+        -- the certificate the mock RPC server hands out on refresh; with one day+        -- left it would be refused+        openssl+          [ "req"+          , "-x509"+          , "-newkey"+          , "rsa:2048"+          , "-nodes"+          , "-days"+          , "2"+          , "-subj"+          , "/CN=rotated"+          , "-keyout"+          , dir </> "key.pem"+          , "-out"+          , dir </> "cert.pem"+          ]+        openssl+          [ "pkcs12"+          , "-export"+          , "-inkey"+          , dir </> "key.pem"+          , "-in"+          , dir </> "cert.pem"+          , "-out"+          , dir </> "cert.p12"+          , "-passout"+          , "pass:" <> toString (key testConfig)+          ]+        p12 <- readFileBS (dir </> "cert.p12")+        rotated <- CertificateChain <$> readSignedObject (dir </> "cert.pem")+        withServer (const defaultMockRPC { mockCertificate = p12 }) $ \port actions -> do+          servedCertificate port >>= (`shouldNotBe` Just rotated)+          bracket (connectLocal port) Sock.close $ \sock -> do+            -- a request still being sent while the certificate changes+            sendAll sock "GET /robots.txt HTTP/1.1\r\nHost: localhost\r\n"+            atomically $ writeTChan actions Cert+            let waitForRotated = do+                  served <- servedCertificate port+                  unless (served == Just rotated) $ threadDelay 100000 >> waitForRotated+            timeout 5000000 waitForRotated >>= (`shouldBe` Just ())+            sendAll sock "\r\n"+            reply <- recv sock 4096+            reply `shouldSatisfy` ("HTTP/1.1 200" `BS.isPrefixOf`)++    it "listens before it sends client_start" $ do+      listeningAtStart <- newEmptyMVar+      let mock port = defaultMockRPC { mockSeen = \( act, _ ) -> when (act == "client_start")+                                         $ void . tryPutMVar listeningAtStart =<< accepts port }+      withServer mock $ \_ _ -> takeMVar listeningAtStart >>= (`shouldBe` True)++    it "on shutdown, sends client_stop once, serves 5 s more, then stops listening" $ do+      seen <- newIORef []+      let mock _ = defaultMockRPC { mockSeen = atomicModifyIORef'_ seen . (:) . fst }+      withServer mock $ \port actions -> do+        atomically $ writeTChan actions GracefulShutdown+        threadDelay 1000000+        readIORef seen >>= (`shouldBe` [ "client_stop" ]) . filter (== "client_stop")+        response <- fetch port "/robots.txt"+        responseStatus response `shouldBe` status200+        threadDelay 6000000+        accepts port >>= (`shouldBe` False)++-- | Run 'startServer' on a free port with an SQLite cache in a temporary+-- directory, and shut it down afterwards. The mock RPC server is told the port.+withServer :: (Int -> MockRPC) -> (Int -> TChan ServerAction -> IO a) -> IO a+withServer mock action = withSystemTempDirectory "hath-server" $ \dir -> do+  port <- bracket Warp.openFreePort (Sock.close . snd) (pure . fst)+  withMockRPC (mock port) $ do+    -- warp-tls' own test pair: its age does not matter here+    certs <- CertificateChain <$> readSignedObject "vendor/warp-tls-3.4.18/certificate.pem"+    privateKey <- readKeyFile "vendor/warp-tls-3.4.18/key.pem"+      >>= maybe (fail "no private key in the test key file") pure . listToMaybe+    actions <- newTChanIO+    let config   = testConfig { cachePath = toText (dir </> "cache.db") }+        settings = defaultHathSettings { clientPort = port }+        options  = ServerOptions { disableRateLimit = True, trustProxyHeaders = False }+    bracket (forkIO $ startServer config options settings ( certs, privateKey ) actions) killThread+      $ \_ -> do+        waitUntilListening port+        result <- action port actions+        atomically $ writeTChan actions GracefulShutdown+        -- time to tell the RPC server before the mock goes away+        threadDelay 500000+        pure result++accepts :: Int -> IO Bool+accepts port = isRight <$> try @IOException (bracket (connectLocal port) Sock.close (const pass))++waitUntilListening :: Int -> IO ()+waitUntilListening port = go (100 :: Int)+  where+    go n = try @IOException (bracket (connectLocal port) Sock.close (const pass)) >>= \case+      Right () -> pass+      Left e+        | n > 0 -> threadDelay 100000 >> go (n - 1)+        | otherwise -> throwIO e++connectLocal :: Int -> IO Sock.Socket+connectLocal port = do+  sock <- Sock.socket Sock.AF_INET Sock.Stream Sock.defaultProtocol+  Sock.connect sock (Sock.SockAddrInet (fromIntegral port) (tupleToHostAddress ( 127, 0, 0, 1 )))+    `catch` \(e :: IOException) -> Sock.close sock >> throwIO e+  pure sock++-- | The chain a TLS handshake presents, without SNI (as when connecting by IP).+servedCertificate :: Int -> IO (Maybe CertificateChain)+servedCertificate port = bracket (connectLocal port) Sock.close $ \sock -> do+  seen <- newIORef Nothing+  let params = TLS.defaultParamsClient "localhost" ""+  ctx <- TLS.contextNew+    sock+    params { TLS.clientUseServerNameIndication = False+           , TLS.clientHooks = (TLS.clientHooks params)+               { TLS.onServerCertificate = \_ _ _ chain -> [] <$ writeIORef seen (Just chain) }+           }+  TLS.handshake ctx+  void $ try @SomeException (TLS.bye ctx)+  readIORef seen
− test/InterfaceAPIOptimizationSpec.hs
@@ -1,54 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}--module InterfaceAPIOptimizationSpec-  ( interfaceAPIOptimizationSpecs-  , main-  ) where--import qualified Data.IORef            as IORef--import           Relude                hiding ( readFile )--import           Test.Hspec--import           Utils                 ( firstJustM )--main :: IO ()-main = hspec interfaceAPIOptimizationSpecs--interfaceAPIOptimizationSpecs :: Spec-interfaceAPIOptimizationSpecs = describe "Interface.API optimization" $ do-  it "stops after the first Just" $ do-    visitedRef <- IORef.newIORef ([] :: [ Text ])-    let visit label = do-          IORef.modifyIORef' visitedRef (<> [ label ])-          pure $ if label == "hit-2" then Just label else Nothing--    result <- firstJustM visit [ "miss-1", "hit-2", "miss-3" ]-    visited <- IORef.readIORef visitedRef--    result `shouldBe` Just "hit-2"-    visited `shouldBe` [ "miss-1", "hit-2" ]--  it "returns Nothing after visiting every candidate when all fail" $ do-    visitedRef <- IORef.newIORef ([] :: [ Text ])-    let visit label = do-          IORef.modifyIORef' visitedRef (<> [ label ])-          pure Nothing--    result <- firstJustM visit [ "miss-1", "miss-2", "miss-3" ]-    visited <- IORef.readIORef visitedRef--    result `shouldBe` (Nothing :: Maybe Text)-    visited `shouldBe` [ "miss-1", "miss-2", "miss-3" ]--  it "preserves left-to-right evaluation order" $ do-    visitedRef <- IORef.newIORef ([] :: [ Text ])-    let visit label = do-          IORef.modifyIORef' visitedRef (<> [ label ])-          pure $ if label == "hit-3" then Just label else Nothing--    _ <- firstJustM visit [ "miss-1", "miss-2", "hit-3", "miss-4" ]-    visited <- IORef.readIORef visitedRef--    visited `shouldBe` [ "miss-1", "miss-2", "hit-3" ]
+ test/LocateSpec.hs view
@@ -0,0 +1,65 @@+module LocateSpec ( locateSpecs ) where++import           Control.Exception  ( throwIO )++import           Hath.Download      ( receive )+import           Hath.FileURI       ( FileURI(..) )+import           Hath.Locate        ( usableReply )++import           Network.HTTP.Types ( hContentLength )++import           Relude++import           System.IO.Error    ( userError )++import           Test.Hspec++thousandBytes :: FileURI+thousandBytes+  = FileURI { fileHash = "0123456789abcdef0123456789abcdef01234567"+            , fileSize = 1000+            , fileXRes = 800+            , fileYRes = 600+            , fileExt  = "jpg"+            }++locateSpecs :: Spec+locateSpecs = describe "Locate" $ do+  describe "usableReply" $ do+    let announcing status len+          = usableReply 1073741824 thousandBytes status [ ( hContentLength, len ) ]++    it "takes a 200 that announces the file's length" $ announcing 200 "1000" `shouldBe` True++    it "refuses another status, another length, or none" $ do+      announcing 404 "1000" `shouldBe` False+      announcing 200 "999" `shouldBe` False+      usableReply 1073741824 thousandBytes 200 [] `shouldBe` False++    it "refuses a file larger than max_allowed_filesize"+      $ usableReply 999 thousandBytes 200 [ ( hContentLength, "1000" ) ] `shouldBe` False++  describe "receive" $ do+    it "hands over each chunk with its offset"+      $ receiving 6 [ "abc", "def" ] `shouldReturn` ( True, [ ( 0, "abc" ), ( 3, "def" ) ] )++    it "fails a body that ends short of its length"+      $ receiving 6 [ "abc" ] `shouldReturn` ( False, [ ( 0, "abc" ) ] )++    it "fails a body that brings more than its length, keeping nothing past it"+      $ receiving 4 [ "abc", "def" ] `shouldReturn` ( False, [ ( 0, "abc" ) ] )++    it "fails when reading fails" $ do+      outcome <- receive 1000000 6 (throwIO (userError "connection reset")) (\_ _ -> pass)+      outcome `shouldSatisfy` isLeft+  where+    receiving size chunks = do+      rest <- newIORef chunks+      kept <- newIORef []+      let body = atomicModifyIORef' rest $ \case+            []          -> ( [], "" )+            next : more -> ( more, next )+      outcome+        <- receive 1000000 size body (\offset chunk -> modifyIORef' kept (( offset, chunk ) :))+      chunksKept <- reverse <$> readIORef kept+      pure ( isRight outcome, chunksKept )
test/MockRPC.hs view
@@ -1,165 +1,155 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeOperators #-}---- | Mock RPC server for testing hs-hath client without connecting to real H@H network.------ Usage:---   1. Start the mock server with 'withMockRPC'---   2. Set HATH_RPC_HOST environment variable to "localhost"---   3. Run client code against the mock server-module MockRPC-  ( withMockRPC-  , MockRPCConfig(..)-  , defaultMockConfig-  , startMockRPC-  , stopMockRPC-  ) where+-- | A stand-in for the H\@H RPC server, which also plays the image server that+-- files are fetched from.+module MockRPC ( MockRPC(..), defaultMockRPC, withMockRPC, loginSettings ) where -import           Control.Concurrent       ( ThreadId, forkIO, killThread )-import           Control.Concurrent.MVar  ( MVar, newEmptyMVar, putMVar, takeMVar )-import           Control.Exception        ( SomeException, catch, throwIO )+import           Control.Concurrent       ( threadDelay ) +import           Data.ByteString.Builder  ( byteString ) import qualified Data.ByteString.Char8    as BS-import qualified Data.ByteString.Lazy     as LBS-import           Data.Text                ( Text )-import qualified Data.Text                as T+import qualified Data.List                as List+import           Data.Time.Clock.System   ( SystemTime(systemSeconds), getSystemTime ) +import           Hath.Protocol            ( sha1Hex )+ import qualified Network.HTTP.Types       as HTTP import qualified Network.Wai              as Wai import qualified Network.Wai.Handler.Warp as Warp -import           Relude                   hiding ( newEmptyMVar, putMVar, takeMVar )+import           Relude -import           Utils                    ( hash )+import           System.Environment       ( setEnv )+import           System.IO.Error          ( ioError, mkIOError, resourceVanishedErrorType ) --- | Configuration for mock RPC server behavior-data MockRPCConfig-  = MockRPCConfig-  { mockServerStatResponse :: ByteString  -- ^ Response for server_stat-  , mockClientLoginResponse :: ByteString  -- ^ Response for client_login-  , mockStillAliveResponse :: ByteString  -- ^ Response for still_alive-  , mockClientStartResponse :: ByteString  -- ^ Response for client_start-  , mockClientStopResponse :: ByteString  -- ^ Response for client_stop-  , mockGetCertResponse :: ByteString  -- ^ Response for get_cert (or error)-  , mockClientSettingsResponse :: ByteString -- ^ Response for client_settings-  , mockFileContent :: ByteString -- ^ Content for file downloads-  , mockPort :: Int         -- ^ Port to run mock server on-  , mockClientKey :: ByteString -- ^ Key OK replies are signed with (1.6.6 RPC)+data MockRPC+  = MockRPC+  { mockFileContent   :: ByteString+    -- ^ what the image server sends, whichever file is asked for+  , mockCertificate   :: ByteString+    -- ^ the reply to get_cert+  , mockClientKey     :: ByteString+    -- ^ the key OK replies are signed with+  , mockClockAhead    :: Int64+    -- ^ how far its clock is ahead of ours. A request signed more than a+    -- minute off it is answered KEY_EXPIRED, except server_stat, which+    -- tells the time.+  , mockOutages       :: Int+    -- ^ how many RPC requests it answers with a 500 before it works+  , mockBlacklist     :: [ ByteString ]+  , mockSeen          :: ( ByteString, ByteString ) -> IO ()+    -- ^ told the act and add of each RPC or gallery downloader request as it arrives+  , mockRPCServerIP   :: ByteString+    -- ^ its address as its settings give it (@rpc_server_ip@); by default the+    -- one the tests send server commands from+  , mockPauseHalfway  :: Int+    -- ^ microseconds the image server waits halfway through a file+  , mockHangUpHalfway :: Bool+    -- ^ whether it hangs up there instead, having announced the whole file+  , mockQueue         :: ByteString -> ByteString+    -- ^ the reply to a fetchqueue, given its add (the gallery it marks done)   } --- | Default mock configuration with valid responses-defaultMockConfig :: MockRPCConfig-defaultMockConfig-  = MockRPCConfig-  { mockServerStatResponse = "OK"-  , mockClientLoginResponse = BS.unlines-      [ "OK"-      , "host=127.0.0.1"-      , "port=8080"-      , "throttle_bytes=0"-      , "disklimit_bytes=107374182400"-      , "diskremaining_bytes=107374182400"-      ]-  , mockStillAliveResponse = "OK"-  , mockClientStartResponse = "OK"-  , mockClientStopResponse = "OK"-  , mockGetCertResponse = "FAIL\nNo test certificate available"-  , mockClientSettingsResponse = BS.unlines [ "OK", "host=127.0.0.1", "port=8080" ]-  , mockFileContent = "TEST_FILE_CONTENT"-  , mockPort = 19800-  , mockClientKey = "test-secret-key-12345"+defaultMockRPC :: MockRPC+defaultMockRPC+  = MockRPC+  { mockFileContent   = "TEST_FILE_CONTENT"+  , mockCertificate   = "FAIL\nNo test certificate available"+  , mockClientKey     = "test-secret-key-12345"+  , mockClockAhead    = 0+  , mockOutages       = 0+  , mockBlacklist     = []+  , mockSeen          = const pass+  , mockRPCServerIP   = "127.0.0.1"+  , mockPauseHalfway  = 0+  , mockHangUpHalfway = False+  , mockQueue         = const "NO_PENDING_DOWNLOADS"   } --- | Mock RPC application-mockRPCApp :: MockRPCConfig -> Wai.Application-mockRPCApp config req respond = do-  let path  = Wai.rawPathInfo req-      query = Wai.queryString req-      act   = join $ snd <$> find ((== "act") . fst) query  -- Extract "act" parameter+loginSettings :: MockRPC -> [ ByteString ]+loginSettings mock+  = [ "host=127.0.0.1"+    , "port=8080"+    , "throttle_bytes=0"+    , "disklimit_bytes=107374182400"+    , "diskremaining_bytes=107374182400"+    , "rpc_server_ip=" <> mockRPCServerIP mock+    ] -  case ( path, act ) of-    -- RPC endpoint-    ( "/16/rpc", Just action ) -> handleRPCAction config action respond-    ( "/file", _ ) -> handleDownload config respond-    _ -> respond $ notFoundResponse+withMockRPC :: MockRPC -> IO a -> IO a+withMockRPC mock action = do+  outagesLeft <- newIORef (mockOutages mock)+  Warp.testWithApplication (pure (application mock outagesLeft)) $ \port -> do+    setEnv "HATH_RPC_HOST" ("localhost:" <> show port)+    action --- | Handle RPC actions based on the 'act' parameter-handleRPCAction :: MockRPCConfig-                -> ByteString-                -> (Wai.Response -> IO Wai.ResponseReceived)-                -> IO Wai.ResponseReceived-handleRPCAction config action respond = respond $ okResponse $ case action of-  "get_cert" -> mockGetCertResponse config  -- the raw PKCS#12 file, never signed-  "server_stat" -> sign $ mockServerStatResponse config-  "client_login" -> sign $ mockClientLoginResponse config-  "still_alive" -> sign $ mockStillAliveResponse config-  "client_start" -> sign $ mockClientStartResponse config-  "client_stop" -> sign $ mockClientStopResponse config-  "client_settings" -> sign $ mockClientSettingsResponse config-  "get_blacklist" -> sign "OK"  -- Empty blacklist-  "dlfetch" -> sign "OK"  -- No download URLs-  "dlfails" -> sign "OK"  -- Acknowledge failures-  "srfetch" -> sign-    $ "OK\nhttp://localhost:" <> encodeUtf8 (show (mockPort config) :: Text) <> "/file"-  _ -> "FAIL\nUnknown action: " <> action+application :: MockRPC -> IORef Int -> Wai.Application+application mock outagesLeft req respond = case ( Wai.rawPathInfo req, parameter "act" ) of+  ( "/16/rpc", Just act ) -> do+    mockSeen mock ( act, add )+    down <- atomicModifyIORef' outagesLeft $ \n -> ( n - 1, n > 0 )+    clock <- (+ mockClockAhead mock) . systemSeconds <$> getSystemTime+    respond+      $ if down+        then Wai.responseLBS HTTP.status500 [] ""+        else Wai.responseLBS HTTP.status200 [] $ fromStrict $ rpcReply clock act+  ( "/16/dl", Just "fetchqueue" ) -> do+    mockSeen mock ( "fetchqueue", add )+    respond $ Wai.responseLBS HTTP.status200 [] $ fromStrict $ mockQueue mock add+  ( "/file", _ ) -> respond+    -- image servers always send the length+    $ Wai.responseStream HTTP.status200 [ ( "Content-Length", show (BS.length content) ) ]+    $ \write flush -> do+      write (byteString firstHalf) >> flush+      threadDelay (mockPauseHalfway mock)+      -- Warp drops a connection whose body throws, and does not report this kind of error+      if mockHangUpHalfway mock+        then ioError (mkIOError resourceVanishedErrorType "hanging up halfway" Nothing Nothing)+        else write (byteString secondHalf)+  _ -> respond $ Wai.responseLBS HTTP.status404 [ ( "Content-Type", "text/plain" ) ] "Not Found"   where-    -- the 1.6.6 server signs every OK reply with the client key-    sign body = case BS.lines body of-      "OK" : rest -> BS.unlines-        (("OK-1-" <> hash ("1-" <> mockClientKey config <> foldMap ("\n" <>) rest)) : rest)-      _ -> body+    content = mockFileContent mock --- | Handle download endpoint-handleDownload-  :: MockRPCConfig -> (Wai.Response -> IO Wai.ResponseReceived) -> IO Wai.ResponseReceived-handleDownload config respond-  = respond-  $ Wai.responseLBS-    HTTP.status200-    [ ( "Content-Length", show (BS.length (mockFileContent config)) ) ] -- image servers always send it-    (LBS.fromStrict (mockFileContent config))+    ( firstHalf, secondHalf ) = BS.splitAt (BS.length content `div` 2) content --- | Create a 200 OK response with given body-okResponse :: ByteString -> Wai.Response-okResponse body-  = Wai.responseLBS-    HTTP.status200-    [ ( "Content-Type", "text/plain; charset=utf-8" ) ]-    (LBS.fromStrict body)+    parameter name = join (List.lookup name (Wai.queryString req)) --- | Create a 404 Not Found response-notFoundResponse :: Wai.Response-notFoundResponse = Wai.responseLBS HTTP.status404 [ ( "Content-Type", "text/plain" ) ] "Not Found"+    -- as sent: Wai would end it at its first ';'+    add+      = maybe "" (HTTP.urlDecode True . BS.drop 4)+      $ find ("add=" `BS.isPrefixOf`)+      $ BS.split '&'+      $ BS.drop 1 (Wai.rawQueryString req) --- | Start mock RPC server in background thread-startMockRPC :: MockRPCConfig -> IO ( ThreadId, MVar () )-startMockRPC config = do-  ready <- newEmptyMVar-  tid <- forkIO $ do-    let settings-          = Warp.setPort (mockPort config)-          $ Warp.setBeforeMainLoop (putMVar ready ())-          $ Warp.defaultSettings-    Warp.runSettings settings (mockRPCApp config)-  takeMVar ready  -- Wait for server to be ready-  return ( tid, ready )+    -- we are the image server too, at whatever address the client reached us+    ownFile = "http://" <> fromMaybe "localhost" (Wai.requestHeaderHost req) <> "/file" --- | Stop mock RPC server-stopMockRPC :: ThreadId -> IO ()-stopMockRPC = killThread+    signedAt clock+      = maybe+        False+        (\time -> abs (time - clock) <= 60)+        (readMaybe . BS.unpack =<< parameter "acttime") --- | Run an action with a mock RPC server--- The server is started before the action and stopped after.-withMockRPC :: MockRPCConfig -> IO a -> IO a-withMockRPC config action = do-  ( tid, _ ) <- startMockRPC config-  result <- action `finally'` stopMockRPC tid-  return result-  where-    finally' :: IO a -> IO () -> IO a-    finally' io cleanup = do-      result <- io `catch` (\(e :: SomeException) -> cleanup >> throwIO e)-      cleanup-      return result+    rpcReply clock = \case+      "server_stat" -> signed [ "server_time=" <> show clock ]+      act+        | not (signedAt clock) -> "KEY_EXPIRED"+        | otherwise -> reply act +    reply = \case+      -- the PKCS#12 file itself, never signed+      "get_cert" -> mockCertificate mock+      "client_login" -> signed (loginSettings mock)+      "client_settings"+        -> signed [ "host=127.0.0.1", "port=8080", "rpc_server_ip=" <> mockRPCServerIP mock ]+      "srfetch" -> signed [ ownFile ]+      "dlfetch" -> signed [ ownFile ]+      "get_blacklist" -> signed (mockBlacklist mock)+      act+        | act `elem` answeredOK -> signed []+        | otherwise -> "FAIL\nUnknown action: " <> act++    answeredOK = [ "still_alive", "client_start", "client_stop", "overload", "dlfails" ]++    -- the 1.6.6 server signs every OK reply with the client key+    signed payload+      = BS.unlines+        (("OK-1-" <> sha1Hex ("1-" <> mockClientKey mock <> foldMap ("\n" <>) payload)) : payload)
+ test/ProtocolSpec.hs view
@@ -0,0 +1,189 @@+module ProtocolSpec ( protocolSpecs ) where++import qualified Data.ByteString.Char8 as BS+import           Data.Char             ( toUpper )+import qualified Data.Map.Strict       as Map++import           Hath.Config           ( ClientConfig(..), defaultClientConfig )+import           Hath.Error            ( HathError(..) )+import           Hath.Protocol++import           Relude++import           Test.Hspec+import           Test.Hspec.QuickCheck ( prop )+import           Test.QuickCheck++-- | Text free of the separators these formats use.+word :: Gen ByteString+word = BS.pack <$> listOf1 (elements ([ 'a' .. 'z' ] <> [ '0' .. '9' ] <> " ._/"))++-- | A client key, the time of a reply and its payload lines.+reply :: Gen ( ByteString, ByteString, [ ByteString ] )+reply = (,,) <$> word <*> (BS.pack <$> listOf1 (elements [ '0' .. '9' ])) <*> listOf word++-- | A reply as the 1.6.6 server signs it.+signed :: ByteString -> ByteString -> [ ByteString ] -> ByteString+signed clientKey time payload+  = "OK-" <> time <> "-" <> sha1Hex (time <> "-" <> clientKey <> body) <> body+  where+    body = foldMap ("\n" <>) payload++protocolSpecs :: Spec+protocolSpecs = describe "Protocol" $ do+  describe "parseURLParams" $ do+    it "reads pairs, flags, empty values and values containing ="+      $ parseURLParams "a=1;flag;b=x=y;;c="+      `shouldBe` Map.fromList [ ( "a", "1" ), ( "flag", "" ), ( "b", "x=y" ), ( "c", "" ) ]++    prop "reads back every pair, the last of a key winning"+      $ forAll (listOf ((,) <$> word <*> word))+      $ \pairs -> parseURLParams (BS.intercalate ";" [ k <> "=" <> v | ( k, v ) <- pairs ])+      === Map.fromList pairs++  describe "rpcPayload" $ do+    it "is the lines after OK" $ rpcPayload "OK\na=1\nb=2\n" `shouldBe` Right [ "a=1", "b=2" ]++    it "is the status when that is not OK"+      $ rpcPayload "KEY_EXPIRED\nx" `shouldBe` Left (RequestFailure "KEY_EXPIRED")++    it "is an error for an empty reply" $ rpcPayload "" `shouldBe` Left EmptyResponse++  describe "retryWithServerTime" $ do+    it "is for KEY_EXPIRED on a request without add" $ do+      retryWithServerTime "" "KEY_EXPIRED" `shouldBe` True+      retryWithServerTime "" "KEY_EXPIRED\n" `shouldBe` True++    it "is not for a request with add, nor for another status" $ do+      retryWithServerTime "60" "KEY_EXPIRED" `shouldBe` False+      retryWithServerTime "" "FAIL" `shouldBe` False+      retryWithServerTime "" "KEY_EXPIRED_SOON" `shouldBe` False++  describe "verifyRPCResponse" $ do+    it "drops the empty line a reply ends with"+      $ verifyRPCResponse "k" (signed "k" "7" [ "a=1", "b=2" ] <> "\n") `shouldBe` "OK\na=1\nb=2"++    it "refuses an OK that is not signed"+      $ verifyRPCResponse "k" "OK\na=1" `shouldBe` "RESPONSE_HASH_FAILED"++    it "passes any other status through"+      $ verifyRPCResponse "k" "KEY_EXPIRED" `shouldBe` "KEY_EXPIRED"++    prop "accepts a reply signed with the client key"+      $ forAll reply+      $ \( clientKey, time, payload )+      -> verifyRPCResponse clientKey (signed clientKey time payload)+      === BS.intercalate "\n" ("OK" : payload)++    prop "refuses it once a line is added"+      $ forAll ((,) <$> reply <*> word)+      $ \( ( clientKey, time, payload ), extra )+      -> verifyRPCResponse clientKey (signed clientKey time payload <> "\n" <> extra)+      === "RESPONSE_HASH_FAILED"++    prop "refuses a reply signed with another key" $ forAll reply $ \( clientKey, time, payload )+      -> verifyRPCResponse (clientKey <> "x") (signed clientKey time payload)+      === "RESPONSE_HASH_FAILED"++  describe "signatures" $ do+    let cfg = defaultClientConfig { clientId = "12345", key = "secretkey" }++    it "signs an RPC request"+      $ rpcActionKey cfg "srfetch" "1;780;abc" 1700000000+      `shouldBe` sha1HexText "hentai@home-srfetch-1;780;abc-12345-1700000000-secretkey"++    it "knows the key of a server command"+      $ serverCommandKey cfg "speed_test" "testsize=1000" 1700000000+      `shouldBe` sha1HexText+        "hentai@home-servercmd-speed_test-testsize=1000-12345-1700000000-secretkey"++    it "knows the key of a speed test"+      $ speedTestKey cfg 1000 1700000000+      `shouldBe` sha1Hex "hentai@home-speedtest-1000-1700000000-12345-secretkey"++    it "knows the keystamp of a file"+      $ keystampHash cfg 1700000000 "abc-1-2-3-jpg"+      `shouldBe` BS.take 10 (sha1Hex "1700000000-abc-1-2-3-jpg-secretkey-hotlinkthis")++    it "writes an RPC query in the official order, with add= even when empty"+      $ rpcQuery cfg "server_stat" "" 1700000000+      `shouldBe` "clientbuild=181&act=server_stat&add=&cid=12345&acttime=1700000000&actkey="+      <> encodeUtf8 (rpcActionKey cfg "server_stat" "" 1700000000)++    it "keeps the ; between the parts of add and escapes the rest"+      $ rpcQuery cfg "srfetch" "1;org;x&act=y" 1700000000+      `shouldSatisfy` ("&add=1;org;x%26act%3Dy&" `BS.isInfixOf`)++    it "identifies itself to an image server with the file id as it was asked for"+      $ hathRequestToken cfg "abc-01-2-3-jpg" `shouldBe` "12345-"+      <> sha1Hex "secretkeyabc-01-2-3-jpg"++  describe "keystampValid" $ do+    let cfg     = defaultClientConfig { key = "secretkey" }+        fileId  = "abc-1-2-3-jpg"+        issued  = 1700000000+        stamp t = show t <> "-" <> keystampHash cfg t fileId++    it "accepts a keystamp issued less than 900 s from the server's time, either way"+      $ forM_ [ -899, 0, 899 ]+      $ \drift -> keystampValid cfg (issued + drift) fileId (stamp issued) `shouldBe` True++    it "refuses one 900 s or more away" $ forM_ [ -900, 900, 5000 ] $ \drift+      -> keystampValid cfg (issued + drift) fileId (stamp issued) `shouldBe` False++    it "compares the hash in either case"+      $ keystampValid+        cfg+        issued+        fileId+        (show issued <> "-" <> BS.map toUpper (keystampHash cfg issued fileId))+      `shouldBe` True++    it "refuses another file id, and a keystamp that is not <time>-<hash>"+      $ forM_+        [ ( "abc-1-2-3-png", stamp issued )+        , ( fileId, keystampHash cfg issued fileId )+        , ( fileId, stamp issued <> "-x" )+        , ( fileId, "x" <> stamp issued )+        , ( fileId, "" )+        ]+      $ \( requested, given ) -> keystampValid cfg issued requested given `shouldBe` False++  describe "commandTimeValid" $ do+    it "accepts a time up to 300 s from the server's, either way"+      $ map (commandTimeValid 1000) [ 700, 1000, 1300 ] `shouldBe` [ True, True, True ]++    it "refuses one further away"+      $ map (commandTimeValid 1000) [ 699, 1301 ] `shouldBe` [ False, False ]++  describe "fetchParams" $ do+    let params = fetchParams . Map.fromList++    it "takes a numeric fileindex and a numeric or org xres" $ do+      params [ ( "fileindex", "12" ), ( "xres", "org" ) ] `shouldBe` Just ( "12", "org" )+      params [ ( "fileindex", "012" ), ( "xres", "1280" ), ( "keystamp", "x" ) ]+        `shouldBe` Just ( "012", "1280" )++    it "refuses missing, empty or other values"+      $ forM_+        [ [ ( "xres", "org" ) ]+        , [ ( "fileindex", "12" ) ]+        , [ ( "fileindex", "" ), ( "xres", "org" ) ]+        , [ ( "fileindex", "12" ), ( "xres", "" ) ]+        , [ ( "fileindex", "-12" ), ( "xres", "org" ) ]+        , [ ( "fileindex", "12" ), ( "xres", "orgx" ) ]+        , [ ( "fileindex", "12" ), ( "xres", "ORG" ) ]+        ]+      $ \given -> params given `shouldBe` Nothing++  describe "responseHeaders" $ do+    it "are the official client's: no User-Agent, Cache-Control only with a body" $ do+      responseHeaders "image/jpeg" 10+        `shouldBe` [ ( "Server", "Genetic Lifeform and Distributed Open Server 1.6.6" )+                   , ( "Connection", "close" )+                   , ( "Content-Type", "image/jpeg" )+                   , ( "Content-Length", "10" )+                   , ( "Cache-Control", "public, max-age=31536000" )+                   ]+      map fst (responseHeaders "text/html" 0)+        `shouldBe` [ "Server", "Connection", "Content-Type", "Content-Length" ]
test/R2Spec.hs view
@@ -1,151 +1,113 @@ {-# LANGUAGE DataKinds #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeApplications #-}-{-# LANGUAGE TypeOperators #-}  module R2Spec ( r2Specs ) where -import           Colog                ( LogAction(..), richMessageAction, Message )-import           Colog.Polysemy       ( Log, runLogAction )+import           Colog                    ( LogAction(..), Message )+import           Colog.Polysemy           ( Log, runLogAction ) -import           Storage.Database     ( FileRecord(..) )+import           Hath.Cache               ( Cache, CacheHit(..), lookupFile, storeFile )+import           Hath.Cache.R2            ( R2Connection, mkR2Connection, runCacheR2 )+import           Hath.Config              ( R2Config(..) )+import           Hath.Error               ( HathError )+import           Hath.FileURI             ( FileURI(..) ) -import           Polysemy-import           Polysemy.Error       ( Error, runError )-import           Polysemy.KVStore     ( KVStore, lookupKV, updateKV )-import           Polysemy.Reader      ( Reader, runReader )+import           Network.HTTP.Types       ( status200 )+import           Network.Wai              ( Application+                                          , requestMethod+                                          , responseLBS+                                          , strictRequestBody+                                          )+import           Network.Wai.Handler.Warp ( testWithApplication ) -import           Storage.R2           ( R2Connection(..), mkR2Connection, runCacheR2 )+import           Polysemy+import           Polysemy.Error           ( Error, runError ) -import           Relude               hiding ( Reader, runReader )+import           Relude -import           System.Environment   ( setEnv )+import           System.Environment       ( setEnv )  import           Test.Hspec -import           Types                ( ClientConfig-                                      , FileURI-                                      , R2Config(..)-                                      , RPCError-                                      , StorageResult(..)-                                      , defaultClientConfig-                                      , parseFileURI-                                      )---- | R2 test configuration+-- | No such account: requests to it fail, which is all these tests need. testR2Config :: R2Config-testR2Config = R2Config-  { r2Endpoint = "https://111e2d89a4500e36dcda4853060fe725.r2.cloudflarestorage.com"-  , r2Bucket   = "hath"-  }---- | Sample file record for testing-sampleRecord :: FileRecord-sampleRecord = FileRecord-  { fileRecordLRUCounter = 0-  , fileRecordS4         = "test"-  , fileRecordFileId     = "testhash1234-1024-800-600-jpg"-  , fileRecordFileName   = Just "test.jpg"-  , fileRecordBytes      = "test content for R2 storage"-  }+testR2Config+  = R2Config { r2Endpoint = "https://00000000000000000000000000000000.r2.cloudflarestorage.com"+             , r2Bucket   = "hath"+             } --- | Parse a file URI from the sample record sampleURI :: FileURI-sampleURI = parseFileURI "testhash1234-1024-800-600-jpg"+sampleURI+  = FileURI+  { fileHash = "testhash1234", fileSize = 1024, fileXRes = 800, fileYRes = 600, fileExt = "jpg" } --- | Silence logs for cleaner test output-silentLogAction :: LogAction IO Message-silentLogAction = LogAction $ \_ -> pure ()+sampleContent :: ByteString+sampleContent = "test content for R2 storage" --- | Run a program with R2 cache effect (silent logging)-runWithR2Silent :: R2Connection -                -> Sem '[KVStore FileURI StorageResult, Log Message, Reader ClientConfig, Error RPCError, Embed IO] a -                -> IO (Either RPCError a)-runWithR2Silent conn program = -  runM-  . runError @RPCError-  . runReader defaultClientConfig-  . runLogAction @IO @Message silentLogAction+runR2Cache :: R2Connection+           -> Sem '[ Cache, Log Message, Error HathError, Embed IO ] a+           -> IO (Either HathError a)+runR2Cache conn+  = runM+  . runError @HathError+  . runLogAction @IO @Message (LogAction $ const pass)   . runCacheR2 conn-  $ program --- | Lookup with explicit type-lookupFile :: Member (KVStore FileURI StorageResult) r => FileURI -> Sem r (Maybe StorageResult)-lookupFile = lookupKV---- | Update with explicit type-updateFile :: Member (KVStore FileURI StorageResult) r => FileURI -> Maybe StorageResult -> Sem r ()-updateFile = updateKV+redirected :: Either HathError (Maybe CacheHit) -> Bool+redirected (Right (Just (Redirect _))) = True+redirected _ = False  r2Specs :: Spec-r2Specs = describe "R2 Cache Effect" $ do-  -  -- Setup: ensure env vars are set-  beforeAll setupEnv $ do-    -    describe "mkR2Connection" $ do-      it "should create connection with valid env vars" $ \_ -> do-        result <- mkR2Connection testR2Config-        case result of-          Left err -> expectationFailure $ "Connection failed: " <> toString err-          Right _  -> pure ()-      -      it "should fail without R2_ACCESS_KEY" $ \_ -> do-        -- Temporarily unset-        setEnv "R2_ACCESS_KEY" ""-        result <- mkR2Connection testR2Config-        -- Restore-        setEnv "R2_ACCESS_KEY" "cbb541929f87b599e8831ee1917a57fd"-        case result of-          Left _  -> pure ()  -- Expected-          Right _ -> expectationFailure "Should have failed without R2_ACCESS_KEY"+r2Specs = describe "R2 cache" $ beforeAll setupEnv $ do+  describe "mkR2Connection" $ do+    it "fails without R2_ACCESS_KEY" $ \_ -> do+      setEnv "R2_ACCESS_KEY" ""+      result <- mkR2Connection testR2Config+      setEnv "R2_ACCESS_KEY" "12345678901234567890123456789012"+      isLeft result `shouldBe` True -    describe "runCacheR2" $ do-      xit "should store a file record (PUT)" $ \conn -> do-        result <- runWithR2Silent conn $ updateFile sampleURI (Just $ Record sampleRecord)-        result `shouldBe` Right ()+  describe "runCacheR2" $ do+    it "misses a file the bucket does not answer for"+      $ \conn -> runR2Cache conn (lookupFile sampleURI { fileHash = "nonexistent999" })+      >>= (`shouldSatisfy` (not . redirected)) -      xit "should retrieve a stored file record (GET)" $ \conn -> do-        -- First store-        _ <- runWithR2Silent conn $ updateFile sampleURI (Just $ Record sampleRecord)-        -- Then retrieve-        result <- runWithR2Silent conn $ lookupFile sampleURI-        case result of-          Left err -> expectationFailure $ "R2 error: " <> show err-          Right Nothing -> expectationFailure "Expected file record but got Nothing"-          Right (Just (Record record)) -> do-            fileRecordBytes record `shouldBe` fileRecordBytes sampleRecord-          Right (Just (Redirect _)) -> expectationFailure "Expected cached record but got redirect"+  describe "link cache" $ do+    it "asks the bucket once per object, then answers from memory"+      $ \_ -> withFakeBucket $ \heads conn -> do+        a <- runR2Cache conn $ lookupFile sampleURI+        b <- runR2Cache conn $ lookupFile sampleURI+        ( redirected a, redirected b ) `shouldBe` ( True, True )+        readIORef heads `shouldReturn` 1 -      it "should return Nothing for non-existent file" $ \conn -> do-        let nonExistentURI = parseFileURI "nonexistent999-1024-800-600-jpg"-        result <- runWithR2Silent conn $ lookupFile nonExistentURI-        case result of-          Left _ -> pure ()  -- Error is acceptable (404 from R2)-          Right Nothing -> pure ()  -- Expected-          Right (Just _) -> expectationFailure "Expected Nothing but got a record"+    it "needs no HEAD for an object it just stored" $ \_ -> withFakeBucket $ \heads conn -> do+      runR2Cache conn (storeFile sampleURI sampleContent) >>= (`shouldBe` Right ())+      runR2Cache conn (lookupFile sampleURI) >>= (`shouldSatisfy` redirected)+      readIORef heads `shouldReturn` 0 -      xit "should delete a file record (DELETE)" $ \conn -> do-        -- Store first-        _ <- runWithR2Silent conn $ updateFile sampleURI (Just $ Record sampleRecord)-        -- Delete-        deleteResult <- runWithR2Silent conn $ updateFile sampleURI Nothing-        deleteResult `shouldBe` Right ()-        -- Verify deleted-        lookupResult <- runWithR2Silent conn $ lookupFile sampleURI-        case lookupResult of-          Left _ -> pure ()  -- Error acceptable-          Right Nothing -> pure ()  -- Expected-          Right (Just _) -> expectationFailure "File should have been deleted"+fakeBucket :: IORef Int -> Application+fakeBucket heads req respond = do+  when (requestMethod req == "HEAD") $ atomicModifyIORef' heads (\n -> ( n + 1, () ))+  _ <- strictRequestBody req+  respond+    $ responseLBS+      status200+      [ ( "ETag", "\"0\"" )+      , ( "Last-Modified", "Mon, 09 Nov 2026 00:00:00 GMT" )+      , ( "Content-Length", "0" )+      ]+      "" --- | Setup environment variables and create connection+withFakeBucket :: (IORef Int -> R2Connection -> IO a) -> IO a+withFakeBucket k = do+  heads <- newIORef 0+  testWithApplication (pure (fakeBucket heads)) $ \port+    -> mkR2Connection testR2Config { r2Endpoint = "http://127.0.0.1:" <> show port } >>= \case+      Left err   -> error err+      Right conn -> k heads conn++-- | Put credentials (of the right lengths) in the environment and connect. setupEnv :: IO R2Connection setupEnv = do-  -- Set env vars (in real tests, these would come from .env)-  setEnv "R2_ACCESS_KEY" "12345678901234567890123456789012" -- 32 chars-  setEnv "R2_SECRET_KEY" "1234567890123456789012345678901234567890" -- 40 chars-  -  result <- mkR2Connection testR2Config-  case result of-    Left err -> error $ "Failed to create R2 connection: " <> err-    Right conn -> pure conn+  setEnv "R2_ACCESS_KEY" "12345678901234567890123456789012"+  setEnv "R2_SECRET_KEY" "1234567890123456789012345678901234567890"+  mkR2Connection testR2Config+    >>= either (\err -> error $ "Failed to create R2 connection: " <> err) pure
+ test/RPCSpec.hs view
@@ -0,0 +1,112 @@+module RPCSpec ( rpcSpecs ) where++import qualified Data.ByteString.Char8  as BS+import qualified Data.Text              as T+import           Data.Time.Clock.System ( SystemTime(systemSeconds), getSystemTime )++import           Hath.Config            ( ClientConfig(..), defaultClientConfig )+import           Hath.Error             ( HathError(..) )+import           Hath.FileURI           ( parseFileURI )+import           Hath.RPC               ( StartFailure(..)+                                        , clientStart+                                        , getBlacklist+                                        , nextGallery+                                        , overload+                                        , runRPCIO+                                        , startFailure+                                        , toldToStop+                                        , worthRetrying+                                        )+import           Hath.ServerTime        ( serverNow, setServerTime )++import           MockRPC++import           Relude++import           Test.Hspec++config :: ClientConfig+config = defaultClientConfig { clientId = "test-client-001", key = "test-secret-key-12345" }++-- | Back to our own clock, so that the other specs sign with it.+useOurClock :: IO ()+useOurClock = setServerTime . systemSeconds =<< getSystemTime++rpcSpecs :: Spec+rpcSpecs = describe "RPC" $ after_ useOurClock $ do+  describe "worthRetrying" $ do+    it "repeats an exchange that broke off or met a server error" $ do+      worthRetrying Nothing `shouldBe` True+      worthRetrying (Just 500) `shouldBe` True+      worthRetrying (Just 403) `shouldBe` True++    it "takes 404 and 410 as final" $ do+      worthRetrying (Just 404) `shouldBe` False+      worthRetrying (Just 410) `shouldBe` False++  describe "a request" $ do+    it "gets through when two attempts fail"+      $ withMockRPC defaultMockRPC { mockOutages = 2 } (runRPCIO config clientStart)+      >>= (`shouldBe` Right ())++    it "fails after three attempts"+      $ withMockRPC defaultMockRPC { mockOutages = 3 } (runRPCIO config clientStart)+      >>= (`shouldBe` Left (TransportError "HTTP 500"))++    it "gives up on a reply of more than 10 MiB" $ do+      queued <- withMockRPC+        defaultMockRPC { mockQueue = const (BS.replicate 10485761 'x') }+        (runRPCIO config (nextGallery Nothing))+      void queued `shouldBe` Left (TransportError "a reply of more than 10 MiB")++  describe "on KEY_EXPIRED" $ do+    it "takes the server's time and signs a request without add again" $ do+      started <- withMockRPC defaultMockRPC { mockClockAhead = 3600 } (runRPCIO config clientStart)+      started `shouldBe` Right ()+      ahead <- (-) <$> serverNow <*> (systemSeconds <$> getSystemTime)+      ahead `shouldSatisfy` \seconds -> abs (seconds - 3600) <= 1++    it "gives up on a request with add, as the official client does"+      $ withMockRPC defaultMockRPC { mockClockAhead = 3600 } (runRPCIO config (getBlacklist 60))+      >>= (`shouldBe` Left (RequestFailure "KEY_EXPIRED"))++  describe "a failed client_start" $ do+    let refused = \case+          Refused _     -> True+          Unannounced _ -> False++    it "ends the client when another client holds the address or the client ID"+      $ forM_ [ "FAIL_OTHER_CLIENT_CONNECTED", "FAIL_CID_IN_USE" ]+      $ \status -> startFailure 443 (RequestFailure status) `shouldSatisfy` refused++    it "leaves it up after a failed connectivity test, naming the port"+      $ startFailure 443 (RequestFailure "FAIL_CONNECT_TEST") `shouldSatisfy` \case+        Unannounced why -> "443" `T.isInfixOf` why+        Refused _       -> False++    it "leaves it up after any other failure"+      $ forM_ [ RequestFailure "FAIL_STARTUP_FLOOD", TransportError "HTTP 500", EmptyResponse ]+      $ \failure -> startFailure 443 failure `shouldSatisfy` (not . refused)++  describe "a failed still_alive" $ do+    it "stops the client on TERM_BAD_NETWORK"+      $ toldToStop (RequestFailure "TERM_BAD_NETWORK") `shouldBe` True++    it "waits for the next one otherwise"+      $ map toldToStop [ RequestFailure "FAIL", TransportError "HTTP 500", EmptyResponse ]+      `shouldBe` [ False, False, False ]++  describe "overload" $ it "is sent as act=overload" $ do+    seen <- newIORef []+    sent <- withMockRPC+      defaultMockRPC { mockSeen = atomicModifyIORef'_ seen . (:) . fst }+      (runRPCIO config overload)+    sent `shouldBe` Right ()+    readIORef seen `shouldReturn` [ "overload" ]++  describe "get_blacklist" $ it "gives the file ids, skipping lines that are not one" $ do+    let fileId = "0123456789abcdef0123456789abcdef01234567-1000-800-600-jpg"+    blacklist <- withMockRPC+      defaultMockRPC { mockBlacklist = [ fileId, "", "not-a-file-id" ] }+      (runRPCIO config (getBlacklist 43200))+    blacklist `shouldBe` Right (maybeToList (parseFileURI fileId))
+ test/SQLiteSpec.hs view
@@ -0,0 +1,44 @@+{-# LANGUAGE DataKinds #-}++module SQLiteSpec ( sqliteSpecs ) where++import qualified Data.ByteString        as BS++import           Database.SQLite.Simple ( Connection, withConnection )++import           Hath.Cache             ( Cache, CacheHit(..), deleteFile, lookupFile, storeFile )+import           Hath.Cache.SQLite      ( initializeDB, runCacheSQLite )+import           Hath.FileURI           ( FileURI(..) )+import           Hath.Protocol          ( sha1Hex )++import           Polysemy               ( Embed, Sem, runM )++import           Relude++import           Test.Hspec++sqliteSpecs :: Spec+sqliteSpecs+  = describe "SQLite cache"+  $ it "deletes a file; deleting one it does not have is no error"+  $ withConnection ":memory:"+  $ \conn -> do+    initializeDB conn+    inCache conn $ storeFile uri content+    inCache conn (lookupFile uri) >>= (`shouldBe` Just (InMemory content))+    inCache conn $ deleteFile uri+    inCache conn (lookupFile uri) >>= (`shouldBe` Nothing)+    inCache conn $ deleteFile uri+  where+    content = "test content for SQLite storage"++    uri+      = FileURI { fileHash = sha1Hex content+                , fileSize = BS.length content+                , fileXRes = 800+                , fileYRes = 600+                , fileExt  = "jpg"+                }++inCache :: Connection -> Sem '[ Cache, Embed IO ] a -> IO a+inCache conn = runM . runCacheSQLite conn
test/Spec.hs view
@@ -1,78 +1,219 @@-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeOperators #-}+import           APISpec             ( apiSpecs ) -import qualified Data.Map.Strict  as Map-import           Data.Text        ( Text )+import           ConnectionLimitSpec ( connectionLimitSpecs ) -import           Storage.Database ( FileRecord(..), runCachePure )+import           Data.X509           ( CertificateChain(..) )+import           Data.X509.File      ( readSignedObject ) -import           FileURIRenderSpec           ( fileURIRenderSpecs )-import           FilesystemSpec              ( filesystemSpecs )-import           Integration                 ( integrationSpecs )-import           InterfaceAPIOptimizationSpec ( interfaceAPIOptimizationSpecs )+import           Dhall               ( embed, inject )+import qualified Dhall.Core          as Dhall -import           R2Spec                      ( r2Specs )+import           FileURISpec         ( fileURISpecs ) -import           Polysemy-import           Polysemy.KVStore ( KVStore, lookupKV, updateKV )+import           FilesystemSpec      ( filesystemSpecs ) -import           Relude           hiding ( Reader )+import           FloodControlSpec    ( floodControlSpecs ) -import           Test.Hspec+import           GallerySpec         ( gallerySpecs ) -import           Types            ( FileURI, parseFileURI )+import           Hath.Config         ( CacheBackend(..)+                                     , ClientConfig(..)+                                     , R2Config(..)+                                     , defaultClientConfig+                                     , readClientConfig+                                     )+import           Hath.Server         ( needsRenewal )+import           Hath.Settings       ( HathSettings(..)+                                     , defaultHathSettings+                                     , maxConnections+                                     , parseSettings+                                     , settingNumber+                                     )+import           Hath.Stats          ( addDlBytes+                                     , addDownload+                                     , addUpload+                                     , incDlFile+                                     , incDlTask+                                     , incFetched+                                     , incServed+                                     , newStatsEnv+                                     , observeDuration+                                     , renderMetrics+                                     , trackConnection+                                     ) +import           Integration         ( integrationSpecs )++import           LocateSpec          ( locateSpecs )++import           Polysemy            ( runM )+import           Polysemy.Reader     ( runReader )++import           ProtocolSpec        ( protocolSpecs )++import           R2Spec              ( r2Specs )++import           RPCSpec             ( rpcSpecs )++import           Relude              hiding ( runReader )++import           SQLiteSpec          ( sqliteSpecs )++import           Test.Hspec+ main :: IO () main = hspec $ do-  cacheSpecs-  interfaceAPIOptimizationSpecs-  fileURIRenderSpecs+  fileURISpecs+  protocolSpecs+  apiSpecs+  floodControlSpecs+  connectionLimitSpecs+  locateSpecs+  gallerySpecs+  settingsSpecs+  rpcSpecs+  configSpecs+  metricsSpecs   integrationSpecs   r2Specs   filesystemSpecs+  sqliteSpecs+  certificateSpecs --- | Helper to store a file record-storeFile :: Member (KVStore FileURI FileRecord) r => FileRecord -> Sem r ()-storeFile record = updateKV (parseFileURI $ encodeUtf8 $ fileRecordFileId record) (Just record)+settingsSpecs :: Spec+settingsSpecs = do+  describe "parseSettings" $ it "keeps what a refresh does not mention" $ do+    let login+          = parseSettings+            defaultHathSettings+            [ "port=443", "disklimit_bytes=1000", "rpc_server_ip=192.0.2.1;192.0.2.2" ]+        refreshed = parseSettings login [ "disklimit_bytes=2000" ]+    clientPort refreshed `shouldBe` 443+    diskLimitBytes refreshed `shouldBe` 2000+    rpcServerIPs refreshed `shouldBe` [ "192.0.2.1", "192.0.2.2" ] --- | Helper to lookup a file by id-lookupFile :: Member (KVStore FileURI FileRecord) r => Text -> Sem r (Maybe FileRecord)-lookupFile fid = lookupKV (parseFileURI $ encodeUtf8 fid)+  describe "max_allowed_filesize"+    $ it "is taken from the settings"+    $ maxAllowedFileSize (parseSettings defaultHathSettings [ "max_allowed_filesize=1000" ])+    `shouldBe` 1000 -cacheSpecs :: Spec-cacheSpecs = describe "Cache effect" $ do-  let sampleRecord-        = FileRecord { fileRecordLRUCounter = 1-                     , fileRecordS4         = "s4-value"-                     , fileRecordFileId     = "test-id"-                     , fileRecordFileName   = Just "test.txt"-                     , fileRecordBytes      = "test content"-                     }+  describe "maxConnections" $ do+    let withSettings = maxConnections . parseSettings defaultHathSettings -  describe "runCachePure" $ do-    it "should store and retrieve a file record" $ do-      let program = do-            storeFile sampleRecord-            lookupFile "test-id"-          result  = run $ runCachePure Map.empty program+    it "is 20, and one more for every 10,000 bytes/s of the throttle, up to 500" $ do+      withSettings [ "throttle_bytes=0" ] `shouldBe` 20+      withSettings [ "throttle_bytes=1009999" ] `shouldBe` 120+      withSettings [ "throttle_bytes=100000000" ] `shouldBe` 500 -      result `shouldBe` Just sampleRecord+    it "is what max_connections says, unless that is 0" $ do+      withSettings [ "throttle_bytes=1000000", "max_connections=900" ] `shouldBe` 900+      withSettings [ "throttle_bytes=1000000", "max_connections=0" ] `shouldBe` 120 -    it "should return Nothing for non-existent file" $ do-      let program = lookupFile "non-existent"-          result  = run $ runCachePure Map.empty program+  describe "settingNumber" $ do+    let stat = [ "min_client_build=176", "cur_client_build=181", "server_time=1763300000" ] -      result `shouldBe` Nothing+    it "reads server_time from a server_stat reply"+      $ settingNumber "server_time" stat `shouldBe` Just 1763300000 -    it "should update existing record" $ do-      let updatedRecord = sampleRecord { fileRecordBytes = "new content" }-          program       = do-            storeFile sampleRecord-            storeFile updatedRecord-            lookupFile "test-id"-          result        = run $ runCachePure Map.empty program+    it "is Nothing for a line that is missing or not a number" $ do+      settingNumber "server_time" [ "min_client_build=176" ] `shouldBe` Nothing+      settingNumber "server_time" [ "server_time=soon" ] `shouldBe` Nothing -      result `shouldBe` Just updatedRecord+certificateSpecs :: Spec+certificateSpecs = describe "needsRenewal" $ do+  -- warp-tls' test certificate expired at 2011-11-21 19:42:57 UTC+  let expiry      = 1321904577+      certificate = CertificateChain <$> readSignedObject "vendor/warp-tls-3.4.18/certificate.pem" +  it "keeps a certificate with a day or more left"+    $ certificate >>= (`shouldBe` False) . needsRenewal (expiry - 86400) +  it "replaces one with less" $ certificate >>= (`shouldBe` True) . needsRenewal (expiry - 86399)++  it "has nothing to serve without a certificate"+    $ needsRenewal 0 (CertificateChain []) `shouldBe` True++-- | hath-bench writes its config this way.+configSpecs :: Spec+configSpecs = describe "ClientConfig" $ it "reads back what ToDhall writes" $ do+  let config+        = defaultClientConfig+        { clientId     = "12345"+        , key          = "a \"quoted\" key"+        , cachePath    = "/tmp/hs-cache.db"+        , cacheBackend = CacheBackendR2+        , r2Config     = Just (R2Config "https://r2.example" "bucket")+        }+  readBack <- readClientConfig (Dhall.pretty (embed inject config))+  show @Text readBack `shouldBe` show config++metricsSpecs :: Spec+metricsSpecs+  = describe "/metrics"+  $ it "renders every metric: names, HELP and TYPE lines, order, numbers, buckets"+  $ do+    stats <- newStatsEnv+    runM $ runReader stats $ do+      addUpload 1000+      addUpload 234+      addDownload 5678+      replicateM_ 3 incServed+      replicateM_ 2 incFetched+      incDlTask+      replicateM_ 4 incDlFile+      addDlBytes 987654321+    -- on a bound (le is inclusive), between bounds, past the last one+    mapM_ (observeDuration stats) [ 0.005, 0.3, 2.5, 75, 0.0001 ]+    -- one connection has come and gone, two are open+    trackConnection stats pass+    rendered <- trackConnection stats $ trackConnection stats $ renderMetrics stats 42+    rendered `shouldBe` golden+  where+    golden+      = unlines+        [ "# HELP hath_download_size_bytes_total Total bytes downloaded from galleries"+        , "# TYPE hath_download_size_bytes_total counter"+        , "hath_download_size_bytes_total 987654321"+        , "# HELP hath_download_file_count_total Total gallery files downloaded"+        , "# TYPE hath_download_file_count_total counter"+        , "hath_download_file_count_total 4"+        , "# HELP hath_download_task_count_total Total gallery download tasks"+        , "# TYPE hath_download_task_count_total counter"+        , "hath_download_task_count_total 1"+        , "# HELP hath_cache_received_total Total number of cache misses"+        , "# TYPE hath_cache_received_total counter"+        , "hath_cache_received_total 2"+        , "# HELP hath_cache_sent_total Total number of cache hits"+        , "# TYPE hath_cache_sent_total counter"+        , "hath_cache_sent_total 3"+        , "# HELP hath_cache_received_bytes_total Total bytes received into cache"+        , "# TYPE hath_cache_received_bytes_total counter"+        , "hath_cache_received_bytes_total 5678"+        , "# HELP hath_cache_sent_bytes_total Total bytes sent from cache"+        , "# TYPE hath_cache_sent_bytes_total counter"+        , "hath_cache_sent_bytes_total 1234"+        , "# HELP hath_uptime_seconds Server uptime in seconds"+        , "# TYPE hath_uptime_seconds gauge"+        , "hath_uptime_seconds 42"+        , "# HELP hath_active_connections Number of active connections"+        , "# TYPE hath_active_connections gauge"+        , "hath_active_connections 2"+        , "# HELP hath_request_duration_seconds Time from request to the last byte sent"+        , "# TYPE hath_request_duration_seconds histogram"+        , "hath_request_duration_seconds_bucket{le=\"0.005\"} 2"+        , "hath_request_duration_seconds_bucket{le=\"0.01\"} 2"+        , "hath_request_duration_seconds_bucket{le=\"0.025\"} 2"+        , "hath_request_duration_seconds_bucket{le=\"0.05\"} 2"+        , "hath_request_duration_seconds_bucket{le=\"0.1\"} 2"+        , "hath_request_duration_seconds_bucket{le=\"0.25\"} 2"+        , "hath_request_duration_seconds_bucket{le=\"0.5\"} 3"+        , "hath_request_duration_seconds_bucket{le=\"1.0\"} 3"+        , "hath_request_duration_seconds_bucket{le=\"2.5\"} 4"+        , "hath_request_duration_seconds_bucket{le=\"5.0\"} 4"+        , "hath_request_duration_seconds_bucket{le=\"10.0\"} 4"+        , "hath_request_duration_seconds_bucket{le=\"30.0\"} 4"+        , "hath_request_duration_seconds_bucket{le=\"60.0\"} 4"+        , "hath_request_duration_seconds_bucket{le=\"+Inf\"} 5"+        , "hath_request_duration_seconds_sum 77.80510000000001"+        , "hath_request_duration_seconds_count 5"+        ]
+ vendor/cryptostore-0.3.0.1/LICENSE view
@@ -0,0 +1,30 @@+Copyright (c) 2018-2023, Olivier Chéron++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of Olivier Chéron nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ vendor/minio-hs-1.7.0/LICENSE view
@@ -0,0 +1,201 @@+                                 Apache License+                           Version 2.0, January 2004+                        http://www.apache.org/licenses/++   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++   1. Definitions.++      "License" shall mean the terms and conditions for use, reproduction,+      and distribution as defined by Sections 1 through 9 of this document.++      "Licensor" shall mean the copyright owner or entity authorized by+      the copyright owner that is granting the License.++      "Legal Entity" shall mean the union of the acting entity and all+      other entities that control, are controlled by, or are under common+      control with that entity. For the purposes of this definition,+      "control" means (i) the power, direct or indirect, to cause the+      direction or management of such entity, whether by contract or+      otherwise, or (ii) ownership of fifty percent (50%) or more of the+      outstanding shares, or (iii) beneficial ownership of such entity.++      "You" (or "Your") shall mean an individual or Legal Entity+      exercising permissions granted by this License.++      "Source" form shall mean the preferred form for making modifications,+      including but not limited to software source code, documentation+      source, and configuration files.++      "Object" form shall mean any form resulting from mechanical+      transformation or translation of a Source form, including but+      not limited to compiled object code, generated documentation,+      and conversions to other media types.++      "Work" shall mean the work of authorship, whether in Source or+      Object form, made available under the License, as indicated by a+      copyright notice that is included in or attached to the work+      (an example is provided in the Appendix below).++      "Derivative Works" shall mean any work, whether in Source or Object+      form, that is based on (or derived from) the Work and for which the+      editorial revisions, annotations, elaborations, or other modifications+      represent, as a whole, an original work of authorship. For the purposes+      of this License, Derivative Works shall not include works that remain+      separable from, or merely link (or bind by name) to the interfaces of,+      the Work and Derivative Works thereof.++      "Contribution" shall mean any work of authorship, including+      the original version of the Work and any modifications or additions+      to that Work or Derivative Works thereof, that is intentionally+      submitted to Licensor for inclusion in the Work by the copyright owner+      or by an individual or Legal Entity authorized to submit on behalf of+      the copyright owner. For the purposes of this definition, "submitted"+      means any form of electronic, verbal, or written communication sent+      to the Licensor or its representatives, including but not limited to+      communication on electronic mailing lists, source code control systems,+      and issue tracking systems that are managed by, or on behalf of, the+      Licensor for the purpose of discussing and improving the Work, but+      excluding communication that is conspicuously marked or otherwise+      designated in writing by the copyright owner as "Not a Contribution."++      "Contributor" shall mean Licensor and any individual or Legal Entity+      on behalf of whom a Contribution has been received by Licensor and+      subsequently incorporated within the Work.++   2. Grant of Copyright License. Subject to the terms and conditions of+      this License, each Contributor hereby grants to You a perpetual,+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable+      copyright license to reproduce, prepare Derivative Works of,+      publicly display, publicly perform, sublicense, and distribute the+      Work and such Derivative Works in Source or Object form.++   3. Grant of Patent License. Subject to the terms and conditions of+      this License, each Contributor hereby grants to You a perpetual,+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable+      (except as stated in this section) patent license to make, have made,+      use, offer to sell, sell, import, and otherwise transfer the Work,+      where such license applies only to those patent claims licensable+      by such Contributor that are necessarily infringed by their+      Contribution(s) alone or by combination of their Contribution(s)+      with the Work to which such Contribution(s) was submitted. If You+      institute patent litigation against any entity (including a+      cross-claim or counterclaim in a lawsuit) alleging that the Work+      or a Contribution incorporated within the Work constitutes direct+      or contributory patent infringement, then any patent licenses+      granted to You under this License for that Work shall terminate+      as of the date such litigation is filed.++   4. Redistribution. You may reproduce and distribute copies of the+      Work or Derivative Works thereof in any medium, with or without+      modifications, and in Source or Object form, provided that You+      meet the following conditions:++      (a) You must give any other recipients of the Work or+          Derivative Works a copy of this License; and++      (b) You must cause any modified files to carry prominent notices+          stating that You changed the files; and++      (c) You must retain, in the Source form of any Derivative Works+          that You distribute, all copyright, patent, trademark, and+          attribution notices from the Source form of the Work,+          excluding those notices that do not pertain to any part of+          the Derivative Works; and++      (d) If the Work includes a "NOTICE" text file as part of its+          distribution, then any Derivative Works that You distribute must+          include a readable copy of the attribution notices contained+          within such NOTICE file, excluding those notices that do not+          pertain to any part of the Derivative Works, in at least one+          of the following places: within a NOTICE text file distributed+          as part of the Derivative Works; within the Source form or+          documentation, if provided along with the Derivative Works; or,+          within a display generated by the Derivative Works, if and+          wherever such third-party notices normally appear. The contents+          of the NOTICE file are for informational purposes only and+          do not modify the License. You may add Your own attribution+          notices within Derivative Works that You distribute, alongside+          or as an addendum to the NOTICE text from the Work, provided+          that such additional attribution notices cannot be construed+          as modifying the License.++      You may add Your own copyright statement to Your modifications and+      may provide additional or different license terms and conditions+      for use, reproduction, or distribution of Your modifications, or+      for any such Derivative Works as a whole, provided Your use,+      reproduction, and distribution of the Work otherwise complies with+      the conditions stated in this License.++   5. Submission of Contributions. Unless You explicitly state otherwise,+      any Contribution intentionally submitted for inclusion in the Work+      by You to the Licensor shall be under the terms and conditions of+      this License, without any additional terms or conditions.+      Notwithstanding the above, nothing herein shall supersede or modify+      the terms of any separate license agreement you may have executed+      with Licensor regarding such Contributions.++   6. Trademarks. This License does not grant permission to use the trade+      names, trademarks, service marks, or product names of the Licensor,+      except as required for reasonable and customary use in describing the+      origin of the Work and reproducing the content of the NOTICE file.++   7. Disclaimer of Warranty. Unless required by applicable law or+      agreed to in writing, Licensor provides the Work (and each+      Contributor provides its Contributions) on an "AS IS" BASIS,+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+      implied, including, without limitation, any warranties or conditions+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+      PARTICULAR PURPOSE. You are solely responsible for determining the+      appropriateness of using or redistributing the Work and assume any+      risks associated with Your exercise of permissions under this License.++   8. Limitation of Liability. In no event and under no legal theory,+      whether in tort (including negligence), contract, or otherwise,+      unless required by applicable law (such as deliberate and grossly+      negligent acts) or agreed to in writing, shall any Contributor be+      liable to You for damages, including any direct, indirect, special,+      incidental, or consequential damages of any character arising as a+      result of this License or out of the use or inability to use the+      Work (including but not limited to damages for loss of goodwill,+      work stoppage, computer failure or malfunction, or any and all+      other commercial damages or losses), even if such Contributor+      has been advised of the possibility of such damages.++   9. Accepting Warranty or Additional Liability. While redistributing+      the Work or Derivative Works thereof, You may choose to offer,+      and charge a fee for, acceptance of support, warranty, indemnity,+      or other liability obligations and/or rights consistent with this+      License. However, in accepting such obligations, You may act only+      on Your own behalf and on Your sole responsibility, not on behalf+      of any other Contributor, and only if You agree to indemnify,+      defend, and hold each Contributor harmless for any liability+      incurred by, or claims asserted against, such Contributor by reason+      of your accepting any such warranty or additional liability.++   END OF TERMS AND CONDITIONS++   APPENDIX: How to apply the Apache License to your work.++      To apply the Apache License to your work, attach the following+      boilerplate notice, with the fields enclosed by brackets "{}"+      replaced with your own identifying information. (Don't include+      the brackets!)  The text should be enclosed in the appropriate+      comment syntax for the file format. We also recommend that a+      file or class name and description of purpose be included on the+      same "printed page" as the copyright notice for easier+      identification within third-party archives.++   Copyright {yyyy} {name of copyright owner}++   Licensed under the Apache License, Version 2.0 (the "License");+   you may not use this file except in compliance with the License.+   You may obtain a copy of the License at++       http://www.apache.org/licenses/LICENSE-2.0++   Unless required by applicable law or agreed to in writing, software+   distributed under the License is distributed on an "AS IS" BASIS,+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+   See the License for the specific language governing permissions and+   limitations under the License.
− vendor/placeholder-0/src/Control/Placeholder.hs
@@ -1,232 +0,0 @@-{-# LANGUAGE BlockArguments #-}-{-# LANGUAGE CPP #-}-{-# LANGUAGE DataKinds #-}-{-# LANGUAGE DeriveAnyClass #-}-{-# LANGUAGE ImplicitParams #-}-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE PatternSynonyms #-}-{-# LANGUAGE PolyKinds #-}-{-# LANGUAGE RankNTypes #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE ViewPatterns #-}---- |--- Copyright  : (c) Edward Kmett 2024--- License    : BSD-2-Clause OR Apache-2.0--- Maintainer : Edward Kmett <ekmett@gmail.com>--- Stability  : experimental--- Portability: non-portable------ Various functions to indicate unfinished or generally unimplemented code --#if __GLASGOW_HASKELL__ >= 980-#define WARNING_IN_XTODO WARNING in "x-todo"-#else-#define WARNING_IN_XTODO WARNING-#endif--module Control.Placeholder-  (-  -- * Combinators-    todo-  , unimplemented-  -- * Patterns-  , pattern TODO-  , pattern Unimplemented-  -- * IO-  , todoIO-  , unimplementedIO-  -- * Exceptions-  , TodoException(TodoException, TodoExceptionWithLocation)-  , UnimplementedException(UnimplementedException, UnimplementedExceptionWithLocation)-  ) where--import Control.Exception-import Data.List (intercalate)-import Data.Typeable-import GHC.Base (raise#, raiseIO#, TYPE, RuntimeRep)-import GHC.Exception-import GHC.Stack-import GHC.Types (IO(IO))-import System.IO.Unsafe---- | This is the 'Exception' thrown by 'todo', 'TODO' and 'todoIO'.-newtype TodoException = TodoExceptionWithLocation String-  deriving (Typeable, Exception)--instance Show TodoException where-  showsPrec _ (TodoExceptionWithLocation loc)-    = showString todoMessage . showChar '\n' . showString loc---- | This lets us discard the location information in a TodoException-pattern TodoException :: TodoException-pattern TodoException <- TodoExceptionWithLocation _ where-  TodoException = TodoExceptionWithLocation missingLocation---- | This is the 'Exception' thrown by 'unimplemented', 'Unimplemented', and 'unimplementedIO'.-newtype UnimplementedException = UnimplementedExceptionWithLocation String-  deriving (Typeable, Exception)--instance Show UnimplementedException where-  showsPrec _ (UnimplementedExceptionWithLocation loc)-    = showString unimplementedMessage . showChar '\n' . showString loc--pattern UnimplementedException :: UnimplementedException-pattern UnimplementedException <- UnimplementedExceptionWithLocation _ where-  UnimplementedException = UnimplementedExceptionWithLocation missingLocation---- | robust retrieval of the current callstack suitable for custom exception types-withCallStack :: Exception a => (String -> a) -> CallStack -> SomeException-withCallStack f stk = unsafeDupablePerformIO do-  ccsStack <- currentCallStack-  let-    implicitParamCallStack = prettyCallStackLines stk-    ccsCallStack-      | [] <- ccsStack = []-      | otherwise = "CallStack (from -prof):" : map ("  " ++) (reverse ccsStack)-    stack = intercalate "\n" $ implicitParamCallStack ++ ccsCallStack-  pure $ toException $ f stack--{- | 'todo' indicates unfinished code.--It is to be used whenever you want to indicate that you are missing a part of-the implementation and want to fill that in later.--The main difference to other alternatives like 'undefined'-or 'error' is this also emits a warning at compile time.--Similarly to 'undefined' and 'error', this will throw an error if-it is evaluated at runtime which can only be caught in 'IO'. Unlike typed holes,-at least without @-fdefer-typed-holes@ enabled, compilation is allowed to proceed-and the program can be run.--Variants such as 'TODO' and 'todoIO' allow usage in patterns and offer more control-over when the exception is thrown when this code is encountered at runtime.--This is intended to *never* stay in code but exists purely for signifying-"work in progress" code.--To make the emitted warning error instead (e.g. for the use in CI), add-the @-Werror=x-todo@ flag to your @OPTIONS_GHC@.--==== __Examples__--@-superComplexFunction :: 'Maybe' a -> 'IO' 'Int'--- we already know how to implement this in the 'Nothing' case-superComplexFunction 'Nothing' = 'pure' 42--- but the 'Just' case is super complicated, so we leave it as 'todo' for now-superComplexFunction ('Just' a) = 'todo'-@--==== __Representation Polymorphism__--'todo', in contrast to 'TODO', is fully representation polymorphic--}-todo :: forall {r :: RuntimeRep} (a :: TYPE r). HasCallStack => a-todo = raise# $ withCallStack TodoExceptionWithLocation ?callStack-{-# WARNING_IN_XTODO todo "'todo' left in code" #-}--{- | 'todoIO' indicates unfinished code that lives in the IO monad.--It should be used similarly to how 'throwIO' should be used rather than 'throw' in IO-to throw at the time the IO action is run rather than at the time it is created.---}-todoIO :: HasCallStack => IO a-todoIO = IO $ raiseIO# $ withCallStack TodoExceptionWithLocation ?callStack-{-# WARNING_IN_XTODO todoIO "'todoIO' left in code" #-}--{- | 'TODO' indicates unfinished code or an unfinished pattern match--You can use this in most positions where you could pass 'todo', but it also can be used in pattern position-to indicate that there are cases you haven't considered.--There remain some circumstances where you can only use 'todo', however, they arise when using this in a "PolyKinded" situation.--This pattern synonym is marked @COMPLETE@, implying that every match after matching on 'TODO'-will /emit a redundant pattern match warning/. Adding new options to your datatype, similarly-to how wildcard patterns (patterns starting with an underscore) work, will /not cause any warnings or errors/.--==== __Examples__--Since the pattern match is strict, even if the branch itself does not evaluate to bottom, matching on-'TODO' will.--@->>> x = []->>> case x of-...   (x : _) -> x-...   'TODO' -> 42-*** Exception: Control.Placeholder.todo: not yet implemented-@--As usual, this behaviour can be reversed by using a @~@ in front of 'TODO' in pattern position.--@->>> x = []->>> case x of-...   (x : _) -> x-...   ~'TODO' -> 42-42-@--In most situations, 'TODO' can be used just like 'todo', where the above is equivalent to the below--@->>> y :: 'Data.Int.Int' = 'todo'->>> x :: 'Data.Int.Int' = 'TODO'-@---==== __Representation Polymorphism__--Mind that pattern synonyms may not be representation polymorphic, hence, if you need something-that can be used with some kind other than 'Data.Kind.Type', you have to use 'todo'. For example,-'TODO' cannot stand instead of a pattern match on an @'GHC.Exts.Int#' :: 'TYPE' 'GHC.Exts.IntRep'@-or as a placeholder for a @'GHC.Exts.ByteArray#' :: 'GHC.Exts.UnliftedType'@--}-pattern TODO :: HasCallStack => () => a-pattern TODO <- (raise# (withCallStack TodoExceptionWithLocation ?callStack) -> _unused) where-  TODO = raise# $ withCallStack TodoExceptionWithLocation ?callStack-{-# WARNING_IN_XTODO TODO "'TODO' left in code" #-}-{-# COMPLETE TODO #-}--{- | 'unimplemented' indicates that the relevant code is unimplemented. Unlike 'todo', it is expected that this _may_ remain in code-long term, and so no warning is supplied. Use cases might include places where a typeclass would theoretically require a member to be-implemented, but where the resulting violation is actually intended.--}--unimplemented :: forall {r :: RuntimeRep} (a :: TYPE r). HasCallStack => a-unimplemented = raise# $ withCallStack UnimplementedExceptionWithLocation ?callStack--{- | 'unimplementedIO' indicates that the method is unimplemented, but it lives in IO, and so only throws when actually run, rather-than when it is constructed. Unlike 'todoIO' it does not provide a compile-time warning, as it is expected that this _may_ remain in-code long term.---}--unimplementedIO :: HasCallStack => IO a-unimplementedIO = IO $ raiseIO# $ withCallStack UnimplementedExceptionWithLocation ?callStack--{- | 'Unimplemented' can be used in most circumstances 'unimplemented' can, but it can also be used in pattern position to indicate cases-haven't been considered yet. Unlike 'TODO' it does not provide a compile-time warning, as it is expected that this _may_ remain in code long term.---}-pattern Unimplemented :: HasCallStack => () => a-pattern Unimplemented <- (raise# (withCallStack UnimplementedExceptionWithLocation ?callStack) -> _unused) where-  Unimplemented = raise# $ withCallStack UnimplementedExceptionWithLocation ?callStack-{-# COMPLETE Unimplemented #-}--missingLocation :: String-missingLocation = ""-{-# NOINLINE missingLocation #-}--todoMessage :: String-todoMessage = "Control.Placeholder.todo: not yet implemented"-{-# NOINLINE todoMessage #-}--unimplementedMessage :: String-unimplementedMessage = "Control.Placeholder.unimplemented: unimplemented"-{-# NOINLINE unimplementedMessage #-}-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Capture.hs
@@ -1,146 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes, TemplateHaskell, Trustworthy #-}-module Polysemy.Capture-  (-- * Effect-    Capture(..)--    -- * Actions-  , reify-  , reflect-  , delimit-  , delimit'-  , capture--    -- * Interpretations-  , runCapture-  , runCaptureWithC--  -- * Prompt types-  , Ref(..)-  ) where--import Control.Monad-import Control.Monad.Cont (ContT(..))--import Polysemy-import Polysemy.Internal-import Polysemy.Internal.Union--import Polysemy.Cont.Internal (Ref(..))---------------------------------------------------------------------------------- | A less powerful variant of 'Polysemy.Shift.Shift' that may always be--- interpreted safely. Unlike 'Polysemy.Shift.Shift',--- continuations can't leave the scope in which they are provided.------ __Note__: Any computation used in a higher-order effect will--- be delimited.------ Activating polysemy-plugin is highly recommended when using this effect--- in order to avoid ambiguous types.-data Capture ref m a where-  Reify    :: (forall s. ref s a -> m s) -> Capture ref m a-  Reflect  :: ref s a -> a -> Capture ref m s-  Delimit  :: m a -> Capture ref m a-  Delimit' :: m a -> Capture ref m (Maybe a)--makeSem_ ''Capture---------------------------------------------------------------------------------- | Reifies the current continuation in the form of a prompt, and passes it to--- the first argument.-reify :: forall ref a r-      .  Member (Capture ref) r-      => (forall s. ref s a -> Sem r s)-      -> Sem r a---------------------------------------------------------------------------------- | Provide an answer to a prompt, jumping to its reified continuation.--- This will not abort the current continuation, and the--- reified computation will return its final result when finished.------ The provided continuation may fail locally in its subcontinuations.--- It may sometimes become necessary to handle such cases. To do so,--- use 'delimit'' together with 'reflect' (the reified continuation--- is already delimited).-reflect :: forall ref s a r-        .  Member (Capture ref) r-        => ref s a-        -> a-        -> Sem r s---------------------------------------------------------------------------------- | Delimits any continuations-delimit :: forall ref a r-        .  Member (Capture ref) r-        => Sem r a-        -> Sem r a---------------------------------------------------------------------------------- | Delimits any continuations, and detects if any subcontinuation--- has failed locally.-delimit' :: forall ref a r-         .  Member (Capture ref) r-         => Sem r a-         -> Sem r (Maybe a)---------------------------------------------------------------------------------- | A restricted version of 'Polysemy.Shift.shift'.--- Executing the provided continuation will not abort execution.------ The provided continuation may fail locally in its subcontinuations.--- It may sometimes become necessary to handle such cases, in--- which case such failure may be detected by using 'delimit'' together--- with the provided continuation (the provided continuation--- is already delimited).-capture :: forall ref r a-         . Member (Capture ref) r-        => (forall s. (a -> Sem r s) -> Sem r s)-        -> Sem r a-capture cc = reify @ref (\ref -> cc (reflect ref))-{-# INLINE capture #-}---------------------------------------------------------------------------------- | Runs a 'Capture' effect by providing @'pure' '.' 'Just'@ as the final--- continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.-runCapture :: Sem (Capture (Ref (Sem r))': r) a -> Sem r (Maybe a)-runCapture = runCaptureWithC (pure . Just)-{-# INLINE runCapture #-}---------------------------------------------------------------------------------- | Runs a 'Capture' effect by explicitly providing a final--- continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.-runCaptureWithC :: (a -> Sem r (Maybe s))-                -> Sem (Capture (Ref (Sem r)) ': r) a-                -> Sem r (Maybe s)-runCaptureWithC c (Sem m) = (`runContT` c) $ m $ \u ->-    case decomp u of-      Right (Weaving e s wv ex ins) ->-        ContT $ \c' ->-          case e of-            Reflect ref a ->-                  runRef ref a-              >>= c' . ex . (<$ s)-            Reify main ->-              runCaptureWithC-                (pure . join . ins)-                (wv (main (Ref (c' . ex . (<$ s))) <$ s))-            Delimit main ->-                  runCaptureWithC-                    (pure . Just)-                    (wv (main <$ s))-              >>= maybe (pure Nothing) (c' . ex)-            Delimit' main ->-                  runCaptureWithC-                    (pure . Just)-                    (wv (main <$ s))-              >>= maybe (c' (ex (Nothing <$ s))) (c' . ex . fmap Just)-      Left g -> ContT $ \c' ->-            liftSem (weave (Just ()) (maybe (pure Nothing) runCapture) id g)-        >>= maybe (pure Nothing) c'-{-# INLINE runCaptureWithC #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/ConstraintAbsorber.hs
@@ -1,44 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes         #-}-{-# LANGUAGE ConstraintKinds             #-}--module Polysemy.ConstraintAbsorber-  ( -- * Absorb builder-    absorbWithSem--    -- * Re-exports-  , Reifies-  , (:-) (Sub)-  , Dict (Dict)-  , reflect-  , Proxy (Proxy)-  ) where--import           Data.Constraint (Dict(Dict), (:-)(Sub), (\\))-import qualified Data.Constraint as C-import qualified Data.Constraint.Unsafe as C-import           Data.Kind (Type, Constraint)-import           Data.Proxy (Proxy (..))-import           Data.Reflection (Reifies, reflect)-import qualified Data.Reflection as R-import           Polysemy----------------------------------------------------------------------------------- | This function can be used to locally introduce typeclass instances for--- 'Sem'. See 'Polysemy.ConstraintAbsorber.MonadState' for an example of how to--- use it.------ @since 0.3.0.0-absorbWithSem-    :: forall -- Constraint to be absorbed-              (p :: (Type -> Type) -> Constraint)-              -- Wrapper to avoid orphan instances-              (x :: (Type -> Type) -> Type -> Type -> Type)-              d r a-     . d  -- ^ Reified dictionary-    -> (forall s. R.Reifies s d :- p (x (Sem r) s))  -- ^ This parameter should always be @'Sub' 'Dict'@-    -> (p (Sem r) => Sem r a)-    -> Sem r a-absorbWithSem d i m =  R.reify d $ \(_ :: Proxy (s :: Type)) -> m \\ C.trans-  (C.unsafeCoerceConstraint :: ((p (x m s) :- p m))) i-{-# INLINEABLE absorbWithSem #-}-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/ConstraintAbsorber/MonadCatch.hs
@@ -1,125 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes         #-}-{-# LANGUAGE FlexibleContexts            #-}-{-# LANGUAGE GADTs                       #-}-{-# LANGUAGE GeneralizedNewtypeDeriving  #-}-{-# LANGUAGE RankNTypes                  #-}-{-# LANGUAGE ScopedTypeVariables         #-}-{-# LANGUAGE TypeApplications            #-}-{-# LANGUAGE UndecidableInstances        #-}--module Polysemy.ConstraintAbsorber.MonadCatch-  (-    -- * Constraint Absorbers-    absorbMonadThrow-  , absorbMonadCatch-  -- * run helper-  , runMonadCatch-  , runMonadCatchAsText-    -- * Re-exports-  , Exception(..)-  , SomeException-  )-where--import qualified Control.Monad.Catch           as C-import           Control.Monad.Catch            ( Exception(..)-                                                , SomeException-                                                , toException-                                                )--import qualified Data.Text                     as T-import           Polysemy-import           Polysemy.ConstraintAbsorber-import qualified Polysemy.Error                as E------------------------------------------------------------------------------------ | Like 'E.runError' but applies a given function from 'SomeException'--- to some other type, typically something less opaque.--- e.g.:---  @runMonadCatch C.displayException@--- --- @since 0.7.0.0-runMonadCatch-  :: Exception e-  => (Maybe e -> e')-  -> Sem (E.Error C.SomeException : E.Error e' : r) a-  -> Sem r (Either e' a)-runMonadCatch f = E.runError . E.mapError (f . C.fromException)--runMonadCatchAsText-  :: Sem (E.Error C.SomeException : E.Error T.Text : r) a-  -> Sem r (Either T.Text a)-runMonadCatchAsText = E.runError . E.mapError (T.pack . C.displayException)------ | Introduce a local 'S.MonadCatch' constraint on 'Sem' --- allowing it to--- interop nicely with exceptions------ @since 0.7.0.0-absorbMonadCatch-  :: Member (E.Error C.SomeException) r-  => (C.MonadCatch (Sem r) => Sem r a)-       -- ^ A computation that requires an instance of 'C.MonadCatch'-       -- or 'C.MonadThrow' for-       -- 'Sem'. This might be something with type @'C.MonadCatch' e m => m a@.-  -> Sem r a-absorbMonadCatch =-  absorbWithSem @C.MonadCatch @Action (CatchDict E.throw E.catch) (Sub Dict)-{-# INLINABLE absorbMonadCatch #-}---- | Introduce a local 'S.MonadThrow' constraint on 'Sem' --- allowing it to--- interop nicely with exceptions------ @since 0.7.0.0-absorbMonadThrow-  :: Member (E.Error C.SomeException) r-  => (C.MonadThrow (Sem r) => Sem r a)-       -- ^ A computation that requires an instance of 'C.MonadCatch'-       -- or 'C.MonadThrow' for-       -- 'Sem'. This might be something with type @'C.MonadCatch' e m => m a@.-  -> Sem r a-absorbMonadThrow main = absorbMonadCatch main-{-# INLINABLE absorbMonadThrow #-}----------------------------------------------------------------------------------- | A dictionary of the functions we need to supply--- to make an instance of Error-data CatchDict m = CatchDict-  { throwM_ :: forall a. C.SomeException -> m a-  , catch_ :: forall a. m a -> (C.SomeException -> m a) -> m a-  }------------------------------------------------------------------------------------ | Wrapper for a monadic action with phantom--- type parameter for reflection.--- Locally defined so that the instance we are going--- to build with reflection must be coherent, that is--- there cannot be orphans.-newtype Action m s' a = Action { action :: m a }-  deriving (Functor, Applicative, Monad)------------------------------------------------------------------------------------ | Given a reifiable mtl Error dictionary,--- we can make an instance of @MonadError@ for the action--- wrapped in @Action@.-instance ( Monad m-         , Reifies s' (CatchDict m)-         ) => C.MonadThrow (Action m s') where-  throwM e = Action $ throwM_ (reflect $ Proxy @s') (C.toException e)-  {-# INLINEABLE throwM #-}--instance ( Monad m-         , Reifies s' (CatchDict m)-         )  => C.MonadCatch (Action m s') where-  catch x f =-    let catchF = catch_ (reflect $ Proxy @s')-    in Action $ (action x) `catchF` \e -> case C.fromException e of-      Just e' -> action $ f e'-      _ -> throwM_ (reflect $ Proxy @s') (C.toException e)-  {-# INLINEABLE catch #-}--
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/ConstraintAbsorber/MonadCont.hs
@@ -1,63 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes         #-}-{-# LANGUAGE FlexibleInstances           #-}-{-# LANGUAGE GeneralizedNewtypeDeriving  #-}-{-# LANGUAGE MultiParamTypeClasses       #-}-{-# LANGUAGE UndecidableInstances        #-}--module Polysemy.ConstraintAbsorber.MonadCont-  ( absorbCont-  ) where--import           Data.Coerce-import qualified Control.Monad.Cont.Class as C-import           Polysemy-import           Polysemy.ConstraintAbsorber-import           Polysemy.Cont------------------------------------------------------------------------------------ | Introduce a local 'C.MonadCont' constraint on 'Sem' --- allowing it to--- interop nicely with MTL.------ @since 0.3.0.0-absorbCont-    :: forall ref r a-     . Member (Cont ref) r-    => (C.MonadCont (Sem r) => Sem r a)-       -- ^ A computation that requires an instance of 'C.MonadCont' for-       -- 'Sem'. This might be something with type @'C.MonadCont' m => m a@.-    -> Sem r a-absorbCont = absorbWithSem @C.MonadCont @Action-  (ContDict (\main -> callCC @ref $ \exit -> main exit))-  (Sub Dict)-{-# INLINEABLE absorbCont #-}------------------------------------------------------------------------------------ | A dictionary of the functions we need to supply--- to make an instance of Cont-newtype ContDict m = ContDict-  { callCC_ :: forall a b. ((a -> m b) -> m a) -> m a-  }------------------------------------------------------------------------------------ | Wrapper for a monadic action with phantom--- type parameter for reflection.--- Locally defined so that the instance we are going--- to build with reflection must be coherent, that is--- there cannot be orphans.-newtype Action m s' a = Action (m a)-  deriving (Functor, Applicative, Monad)------------------------------------------------------------------------------------ | Given a reifiable mtl Cont dictionary,--- we can make an instance of @MonadCont@ for the action--- wrapped in @Action@.-instance ( Monad m-         , Reifies s' (ContDict m)-         ) => C.MonadCont (Action m s') where-  callCC (cc :: (a -> Action m s' b) -> Action m s' a)-    = Action $ callCC_ (reflect $ Proxy @s') @a @b (coerce cc)-  {-# INLINEABLE callCC #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/ConstraintAbsorber/MonadError.hs
@@ -1,62 +0,0 @@-{-# LANGUAGE FlexibleInstances           #-}-{-# LANGUAGE GeneralizedNewtypeDeriving  #-}-{-# LANGUAGE MultiParamTypeClasses       #-}-{-# LANGUAGE UndecidableInstances        #-}--module Polysemy.ConstraintAbsorber.MonadError-  ( absorbError-  ) where--import qualified Control.Monad.Error.Class as S-import           Polysemy-import           Polysemy.ConstraintAbsorber-import           Polysemy.Error------------------------------------------------------------------------------------ | Introduce a local 'S.MonadError' constraint on 'Sem' --- allowing it to--- interop nicely with MTL.------ @since 0.3.0.0-absorbError-    :: Member (Error e) r-    => (S.MonadError e (Sem r) => Sem r a)-       -- ^ A computation that requires an instance of 'S.MonadError' for-       -- 'Sem'. This might be something with type @'S.MonadError' e m => m a@.-    -> Sem r a-absorbError = absorbWithSem @(S.MonadError _) @Action-  (ErrorDict throw catch)-  (Sub Dict)-{-# INLINEABLE absorbError #-}------------------------------------------------------------------------------------ | A dictionary of the functions we need to supply--- to make an instance of Error-data ErrorDict e m = ErrorDict-  { throwError_ :: forall a. e -> m a-  , catchError_ :: forall a. m a -> (e -> m a) -> m a-  }------------------------------------------------------------------------------------ | Wrapper for a monadic action with phantom--- type parameter for reflection.--- Locally defined so that the instance we are going--- to build with reflection must be coherent, that is--- there cannot be orphans.-newtype Action m s' a = Action { action :: m a }-  deriving (Functor, Applicative, Monad)------------------------------------------------------------------------------------ | Given a reifiable mtl Error dictionary,--- we can make an instance of @MonadError@ for the action--- wrapped in @Action@.-instance ( Monad m-         , Reifies s' (ErrorDict e m)-         ) => S.MonadError e (Action m s') where-  throwError e = Action $ throwError_ (reflect $ Proxy @s') e-  {-# INLINEABLE throwError #-}-  catchError x f = Action $ catchError_ (reflect $ Proxy @s') (action x) (action . f)-  {-# INLINEABLE catchError #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/ConstraintAbsorber/MonadReader.hs
@@ -1,63 +0,0 @@-{-# LANGUAGE FlexibleInstances           #-}-{-# LANGUAGE GeneralizedNewtypeDeriving  #-}-{-# LANGUAGE MultiParamTypeClasses       #-}-{-# LANGUAGE UndecidableInstances        #-}--module Polysemy.ConstraintAbsorber.MonadReader-  ( absorbReader-  ) where--import qualified Control.Monad.Reader.Class as S-import           Polysemy-import           Polysemy.ConstraintAbsorber-import           Polysemy.Reader------------------------------------------------------------------------------------ | Introduce a local 'S.MonadReader' constraint on 'Sem' --- allowing it to--- interop nicely with MTL.------ @since 0.3.0.0-absorbReader-    :: Member (Reader i) r-    => (S.MonadReader i (Sem r) => Sem r a)-       -- ^ A computation that requires an instance of 'S.MonadReader' for-       -- 'Sem'. This might be something with type @'S.MonadReader' r m => m a@.-    -> Sem r a-absorbReader = absorbWithSem @(S.MonadReader _) @Action-  (ReaderDict ask local)-  (Sub Dict)-{-# INLINEABLE absorbReader #-}------------------------------------------------------------------------------------ | A dictionary of the functions we need to supply--- to make an instance of Reader-data ReaderDict i m = ReaderDict-  { ask_ :: m i-  , local_ :: forall a. (i -> i) -> m a -> m a-  }------------------------------------------------------------------------------------ | Wrapper for a monadic action with phantom--- type parameter for reflection.--- Locally defined so that the instance we are going--- to build with reflection must be coherent, that is--- there cannot be orphans.-newtype Action m s a = Action { action :: m a }-  deriving (Functor, Applicative, Monad)------------------------------------------------------------------------------------ | Given a reifiable mtl Reader dictionary,--- we can make an instance of @MonadReader@ for the action--- wrapped in @Action@.-instance ( Monad m-         , Reifies s' (ReaderDict i m)-         ) => S.MonadReader i (Action m s') where-  ask = Action $ ask_ $ reflect $ Proxy @s'-  {-# INLINEABLE ask #-}-  local f m = Action $ local_ (reflect $ Proxy @s') f $ action m-  {-# INLINEABLE local #-}-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/ConstraintAbsorber/MonadState.hs
@@ -1,62 +0,0 @@-{-# LANGUAGE FlexibleInstances           #-}-{-# LANGUAGE GeneralizedNewtypeDeriving  #-}-{-# LANGUAGE MultiParamTypeClasses       #-}-{-# LANGUAGE UndecidableInstances        #-}--module Polysemy.ConstraintAbsorber.MonadState-  ( absorbState-  ) where--import           Polysemy-import           Polysemy.ConstraintAbsorber-import           Polysemy.State-import qualified Control.Monad.State.Class as S------------------------------------------------------------------------------------ | Introduce a local 'S.MonadState' constraint on 'Sem' --- allowing it to--- interop nicely with MTL.------ @since 0.3.0.0-absorbState-    :: Member (State s) r-    => (S.MonadState s (Sem r) => Sem r a)-       -- ^ A computation that requires an instance of 'S.MonadState' for-       -- 'Sem'. This might be something with type @'S.MonadState' s m => m a@.-    -> Sem r a-absorbState = absorbWithSem @(S.MonadState _) @Action-  (StateDict get put)-  (Sub Dict)-{-# INLINEABLE absorbState #-}------------------------------------------------------------------------------------ | A Dictionary of the functions we need to supply--- to make an instance of State-data StateDict s m = StateDict-  { get_ :: m s-  , put_ :: s -> m ()-  }------------------------------------------------------------------------------------ | Wrapper for a monadic action with phantom type parameter for reflection.---- Locally defined so that the instance we are going to build with reflection--- must be coherent, that is there cannot be orphans.-newtype Action m s' a = Action (m a)-  deriving (Functor, Applicative, Monad)------------------------------------------------------------------------------------ | Given a reifiable mtl State dictionary,--- we can make an instance of @MonadState@ for the action--- wrapped in @Action@.-instance ( Monad m-         , Reifies s' (StateDict s m)-         ) => S.MonadState s (Action m s') where-  get = Action $ get_ $ reflect $ Proxy @s'-  {-# INLINEABLE get #-}-  put s = Action $ put_ (reflect $ Proxy @s') s-  {-# INLINEABLE put #-}-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/ConstraintAbsorber/MonadWriter.hs
@@ -1,77 +0,0 @@-{-# LANGUAGE FlexibleInstances           #-}-{-# LANGUAGE GeneralizedNewtypeDeriving  #-}-{-# LANGUAGE MultiParamTypeClasses       #-}-{-# LANGUAGE UndecidableInstances        #-}--module Polysemy.ConstraintAbsorber.MonadWriter-  ( absorbWriter-  ) where--import qualified Control.Monad.Writer.Class as S-import           Polysemy-import           Polysemy.ConstraintAbsorber-import           Polysemy.Writer------------------------------------------------------------------------------------ | Introduce a local 'S.MonadWriter' constraint on 'Sem' --- allowing it to--- interop nicely with MTL.------ @since 0.3.0.0-absorbWriter-    :: forall w r a-     . ( Monoid w-       , Member (Writer w) r-       )-    => (S.MonadWriter w (Sem r) => Sem r a)-       -- ^ A computation that requires an instance of 'S.MonadWriter' for-       -- 'Sem'. This might be something with type @'S.MonadWriter' w m => m a@.-    -> Sem r a-absorbWriter =-  let swapTuple (x,y) = (y,x)-      semTell = tell-      semListen :: Member (Writer w) r => Sem r b -> Sem r (b, w)-      semListen = fmap swapTuple . listen @w-      semPass :: Member (Writer w) r => Sem r (b, w -> w) -> Sem r b-      semPass = pass @w . fmap swapTuple-  in absorbWithSem @(S.MonadWriter _) @Action-     (WriterDict semTell semListen semPass)-     (Sub Dict)-{-# INLINEABLE absorbWriter #-}------------------------------------------------------------------------------------ | A dictionary of the functions we need to supply--- to make an instance of Writer-data WriterDict w m = WriterDict-  { tell_ :: w -> m ()-  , listen_ :: forall a. m a -> m (a, w)-  , pass_ :: forall a. m (a, w -> w) -> m a-  }------------------------------------------------------------------------------------ | Wrapper for a monadic action with phantom--- type parameter for reflection.--- Locally defined so that the instance we are going--- to build with reflection must be coherent, that is--- there cannot be orphans.-newtype Action m s' a = Action { action :: m a }-  deriving (Functor, Applicative, Monad)------------------------------------------------------------------------------------ | Given a reifiable mtl Writer dictionary,--- we can make an instance of @MonadWriter@ for the action--- wrapped in @Action@.-instance ( Monad m-         , Monoid w-         , Reifies s' (WriterDict w m)-         ) => S.MonadWriter w (Action m s') where-  tell w = Action $ tell_ (reflect $ Proxy @s') w-  {-# INLINEABLE tell #-}-  listen x = Action $ listen_ (reflect $ Proxy @s') (action x)-  {-# INLINEABLE listen #-}-  pass x = Action $ pass_ (reflect $ Proxy @s') (action x)-  {-# INLINEABLE pass #-}-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Cont.hs
@@ -1,147 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes, Trustworthy #-}-module Polysemy.Cont-  (-- * Effect-    Cont(..)--    -- * Actions-  , jump-  , subst-  , callCC--    -- * Interpretations-  , runContPure-  , runContM-  , contToFinal--    -- * Experimental Interpretations-  , runContViaFresh--    -- * Unsafe Interpretations-  , runContUnsafe--  -- * Prompt types-  , Ref(..)-  , ExitRef(..)-  , ViaFreshRef-  ) where--import Data.Void--import Polysemy-import Polysemy.Final--import Polysemy.Cont.Internal--import Polysemy.Error-import Polysemy.Fresh--import Control.Monad.Cont (MonadCont(), ContT(..), runContT)-import qualified Control.Monad.Cont as C (callCC)---------------------------------------------------------------------------------- | Call with current continuation.--- Executing the provided continuation will abort execution.------ Using the provided continuation--- will rollback all local effectful state back to the point where--- 'callCC' was invoked.------ Higher-order effects do not interact with the continuation in any meaningful--- way; i.e. 'Polysemy.Reader.local' or 'Polysemy.Writer.censor' does not affect--- it, and 'Polysemy.Error.catch' will fail to catch any of its exceptions.--- The only exception to this is if you interpret such effects /and/ 'Cont'--- in terms of the final monad, and the final monad can perform such interactions--- in a meaningful manner.-callCC :: forall ref r a-       .  Member (Cont ref) r-       => ((forall b. a -> Sem r b) -> Sem r a)-       -> Sem r a-callCC cc = subst @ref (\ref -> cc (jump ref)) pure-{-# INLINE callCC #-}---------------------------------------------------------------------------------- | Runs a 'Cont' effect by providing 'pure' as the final continuation.------ This is a safe variant of 'runContUnsafe', as this may only be used--- as the final interpreter before 'run'.-runContPure :: Sem '[Cont (Ref (Sem '[]) a)] a -> Sem '[] a-runContPure = runContUnsafe-{-# INLINE runContPure #-}---------------------------------------------------------------------------------- | Runs a 'Cont' effect by providing 'pure' as the final continuation.------ This is a safe variant of 'runContUnsafe', as this may only be used--- as the final interpreter before 'runM'.-runContM :: Sem '[Cont (Ref (Sem '[Embed m]) a), Embed m] a -> Sem '[Embed m] a-runContM = runContUnsafe-{-# INLINE runContM #-}---------------------------------------------------------------------------------- | Runs a 'Cont' effect in terms of a final 'MonadCont'------ /Beware/: Effects that aren't interpreted in terms of the final monad--- will have local state semantics in regards to 'Cont' effects--- interpreted this way. See 'Final'.-contToFinal :: (Member (Final m) r, MonadCont m)-            => Sem (Cont (ExitRef m) ': r) a-            -> Sem r a-contToFinal = interpretFinal $ \case-  Jump ref a    -> pure $ enterExit ref a-  Subst main cb -> do-    main' <- bindS main-    cb'   <- bindS cb-    s     <- getInitialStateS-    pure $ C.callCC $ \exit ->-      main' (ExitRef (\a -> cb' (a <$ s) >>= vacuous . exit) <$ s)-{-# INLINE contToFinal #-}---------------------------------------------------------------------------------- | A highly experimental 'Cont' interpreter that functions--- through a combination of 'Error' and 'Fresh'. This may be used safely--- anywhere in the effect stack.------ 'runContViaFresh' is still under development.--- You're encouraged to experiment with it, but don't rely on it.--- For best results, use 'runContViaFresh' as the first interpreter you run,--- such that all other effects are global in respect to it.------ This interpreter may return 'Nothing' if the control flow becomes--- split into separate, inconsistent parts,--- such that backtracking fails when trying to invoke continuations.--- For example, if you reify a continuation inside an--- 'async':ed thread, and then have that thread return the reified--- continuation back to the main thread through an 'await', then--- 'runContViaFresh' will return 'Nothing' upon executing the continuation--- in the main thread.-runContViaFresh :: forall uniq r a-                 . (Member (Fresh uniq) r, Eq uniq)-                => Sem (Cont (ViaFreshRef uniq) ': r) a-                -> Sem r (Maybe a)-runContViaFresh =-  let-    hush (Right a) = Just a-    hush _         = Nothing-  in-      fmap hush-    . runError-    . (`runContT` pure)-    . runContViaFreshInC-{-# INLINE runContViaFresh #-}---------------------------------------------------------------------------------- | Runs a 'Cont' effect by providing 'pure' as the final continuation.------ __Beware__: This interpreter will invalidate all higher-order effects of any--- interpreter run after it; i.e. 'Polysemy.Reader.local' and--- 'Polysemy.Writer.censor' will be no-ops, 'Polysemy.Error.catch' will fail--- to catch exceptions, and 'Polysemy.Writer.listen' will always return 'mempty'.------ __You should therefore use 'runContUnsafe' only /after/ running all__--- __interpreters for your higher-order effects.__------ Note that 'Final' is a higher-order effect, and thus 'runContUnsafe' can't--- safely be used together with 'runFinal'.-runContUnsafe :: Sem (Cont (Ref (Sem r) a) ': r) a -> Sem r a-runContUnsafe = runContWithCUnsafe pure-{-# INLINE runContUnsafe #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Cont/Internal.hs
@@ -1,256 +0,0 @@-{-# LANGUAGE TemplateHaskell, Unsafe #-}-module Polysemy.Cont.Internal where--import Data.Functor.Contravariant--import Polysemy-import Polysemy.Internal-import Polysemy.Internal.Union-import Polysemy.Fresh-import Polysemy.Error--import Control.Monad-import Control.Monad.Trans.Cont hiding (Cont)--import Unsafe.Coerce-import GHC.Exts (Any)---------------------------------------------------------------------------------- | An effect for abortive continuations.------ Formulated à la Tom Schrijvers et al.--- "Monad Transformers and Modular Algebraic Effects: What Binds Them Together"--- (2016). <http://www.cs.kuleuven.be/publicaties/rapporten/cw/CW699.pdf>------ Activating polysemy-plugin is highly recommended when using this effect--- in order to avoid ambiguous types.-data Cont ref m a where-  Jump    :: ref a -> a -> Cont ref m b-  Subst   :: (ref a -> m b) -> (a -> m b) -> Cont ref m b--makeSem_ ''Cont---------------------------------------------------------------------------------- | Provide an answer to a prompt, jumping to its reified continuation,--- and aborting the current continuation.------ Using 'jump' will rollback all effectful state back to the point where the--- prompt was created, unless such state is interpreted in terms of the final--- monad, /or/ the associated interpreter of the effectful state--- is run after 'runContUnsafe', which may be done if the effect isn't--- higher-order.------ Higher-order effects do not interact with the continuation in any meaningful--- way; i.e. 'Polysemy.Reader.local' or 'Polysemy.Writer.censor' does not affect--- it, and 'Polysemy.Error.catch' will fail to catch any of its exceptions.--- The only exception to this is if you interpret such effects /and/ 'Cont'--- in terms of the final monad, and the final monad can perform such interactions--- in a meaningful manner.-jump :: forall ref a b r.-        Member (Cont ref) r-     => ref a-     -> a-     -> Sem r b---------------------------------------------------------------------------------- | Reifies the current continuation in the form of a prompt, and passes it to--- the first argument. If the prompt becomes invoked via 'jump', then the--- second argument will be run before the reified continuation, and otherwise--- will not be called at all.-subst :: forall ref a b r-      .  Member (Cont ref) r-      => (ref a -> Sem r b)-      -> (a -> Sem r b)-      -> Sem r b---------------------------------------------------------------------------------- | Runs a 'Cont' effect by providing a final continuation.------ __Beware__: This interpreter will invalidate all higher-order effects of any--- interpreter run after it; i.e. 'Polysemy.Reader.local' and--- 'Polysemy.Writer.censor' will be no-ops, 'Polysemy.Error.catch' will fail--- to catch exceptions, and 'Polysemy.Writer.listen' will always return 'mempty'.------ __You should therefore use 'runContWithCUnsafe' /after/ running all interpreters--- for your higher-order effects.__-runContWithCUnsafe :: (a -> Sem r s) -> Sem (Cont (Ref (Sem r) s) ': r) a -> Sem r s-runContWithCUnsafe c (Sem m) = (`runContT` c) $ m $ \u -> case decomp u of-  Right weaving -> runContWeaving runContWithCUnsafe weaving-  Left g -> ContT $ \c' -> embedSem g >>= runContWithCUnsafe c'-{-# INLINE runContWithCUnsafe #-}--runContWeaving :: Monad m-               => (forall x. (x -> m s) -> Sem r x -> m s)-               -> Weaving (Cont (Ref m s)) (Sem r) a-               -> ContT s m a-runContWeaving runW (Weaving e s wv ex _) =-    ContT $ \c ->-      case e of-        Jump ref a    -> runRef ref a-        Subst main cb ->-          let-            callback a = runW (c . ex) (wv (cb a <$ s))-          in-            runW (c . ex) (wv (main (Ref callback) <$ s))-{-# INLINE runContWeaving #-}--inspectSem :: Sem r a -> Maybe a-inspectSem (Sem m) = m (\_ -> Nothing)-{-# INLINE inspectSem #-}--embedSem :: Union r (Sem r') a -> Sem r (Sem r' a)-embedSem = liftSem . weave (pure ()) (pure . join) inspectSem-{-# INLINE embedSem #-}--newtype Ref m s a = Ref { runRef :: a -> m s }--instance Contravariant (Ref m s) where-  contramap f ref = Ref (runRef ref . f)--newtype ExitRef m a = ExitRef { enterExit :: forall b. a -> m b }--instance Contravariant (ExitRef m) where-  contramap f ref = ExitRef $ \a -> enterExit ref (f a)--data ViaFreshRef uniq a = ViaFreshRef { getBacktrackException :: a -> (uniq, Any) }--instance Contravariant (ViaFreshRef uniq) where-  contramap f ref = ViaFreshRef $ \a -> getBacktrackException ref (f a)--{--  KingoftheHomeless: OK, so let's discuss how this works.-  The idea here is to instead of providing a monadic computation-  to the call of 'callCC' that simply short-circuits everything like-  'ContT' does, we fake that behaviour by instead providing an-  exception to 'callCC', and then try to 'catch' that exception-  on the continuation. If the exception is caught, then we run the-  continuation again. This way, we can get abortive continuations-  without having to scope over a result type variable, avoiding-  the problem that 'runContUnsafe' has, and making it possible-  to weave other effects through without breaking everything.--  Even with that solution, weaving effects through have more problems-  of their own; namely, if we simply lower a-  'forall s. ContT s (Sem r) a' to 'Sem r a', then we effectively-  delimit all higher-order computations. This is bad, because-  if a reified continuation produced within-  the higher-order computation escapes from it,-  then nothing can catch the underlying backtrack exception-  once it is thrown.--  The solution to this is anothor kludge: when weaving other effects through,-  we instead use 'runContViaFreshInCWeave'; this makes use 'ContFreshState'-  as its functorial state, which stores /handlers/ for backtrack exceptions.-  'runContViaFreshInCWeave', in addition to 'catch'ing exceptions on the continuation-  it is given, /also/ returns the handler it uses for the 'catch'.-  This handler is then used by 'runContViaFresh' to catch exceptions on the-  continuation /it/ gets, but can't provide to the higher-order computation.--  I'm astonished that this even remotely works, but it does have some rather-  weird behaviour I haven't completely figured out yet.--  I'm reasonably happy with how 'runContViaFreshInC' looks;-  I'm a lot less happy with 'runContViaFreshInCWeave', I just kinda threw-  it haphazardly. I figure most weirdness stem from issues in-  'runContViaFreshInCWeave', so I need to think it through some more.--}--- | Intermediary monadic interpretation used for running 'runContViaFresh'.--- See source for a discussion on how this works.-runContViaFreshInC :: forall uniq s r a-                    . (Member (Fresh uniq) r, Eq uniq)-                   => Sem (Cont (ViaFreshRef uniq) ': r) a-                   -> ContT s (Sem (Error (uniq, Any) ': r)) a-runContViaFreshInC = usingSem $ \u -> ContT $ \c ->-  case decomp u of-    Right (Weaving e s wv ex _) ->-      case e of-        Subst main cn -> do-          ref <- fresh-          let-            main' = runContViaFreshInC . wv . fmap main . (<$ s)-            cn'   = runContViaFreshInC . wv . fmap cn . (<$ s)-            loop act =-              runContT (ex <$> act) c `catch` \ x@(ref', a') -> do-                if ref == ref' then-                  loop (cn' $ unsafeCoerce a')-                else-                  throw @(uniq, Any) x-          loop $ main' $ ViaFreshRef (\a -> (ref, unsafeCoerce a))-        Jump ref a -> throw (getBacktrackException ref a)-    Left g -> do-      ResAndHandler a rc <- liftSem $-        weave-          (ResAndHandler @uniq @r () throw)-          -- TODO(KingoftheHomeless): is this the distributive law we want?-          (\(ResAndHandler a rc) ->-            runContT-              (runContViaFreshInCWeave a)-              (\x -> pure $-                ResAndHandler-                  x-                  (rc >=> (`runContT` pure) . runContViaFreshInC)-              )-          )-          (Just . getResult)-          (weaken g)-      let loop x = c x `catch` (rc >=> loop)-      loop a---- | A variant of 'runContViaFreshInC' which it uses when weaving other effects through.-runContViaFreshInCWeave :: forall uniq s r a-                         . (Member (Fresh uniq) r, Eq uniq)-                        => Sem (Cont (ViaFreshRef uniq) ': r) a-                        -> ContT (ContFreshState uniq r s)-                            (Sem (Error (uniq, Any) ': r))-                            a-runContViaFreshInCWeave = usingSem $ \u -> ContT $ \c ->-  case decomp u of-    Right (Weaving e s wv ex _) ->-      case e of-        Subst main cn -> do-          ref <- fresh-          let-            -- TODO(KingoftheHomeless): runContViaFreshInC?-            main' = runContViaFreshInCWeave . wv . fmap main . (<$ s)-            cn'   = runContViaFreshInCWeave . wv . fmap cn . (<$ s)-            loop act =-              runContT (ex <$> act) c `catch` \ x@(ref', a') -> do-                if ref == ref' then-                  loop (cn' $ unsafeCoerce a')-                else-                  throw @(uniq, Any) x-          ResAndHandler res h <--            loop $ main' $ ViaFreshRef (\a -> (ref, unsafeCoerce a))-          return $ ResAndHandler res-              -- TODO(KingoftheHomeless): This handler is dubious.-            $ \x -> fmap getResult $ loop $ ContT $ \_ -> fmap (`ResAndHandler` h) (h x)-        Jump ref a -> throw (getBacktrackException ref a)-    Left g -> do-      ResAndHandler a h <- liftSem $-        weave-          (ResAndHandler @uniq @r () throw)-          (\(ResAndHandler a rc) ->-            runContT-              (runContViaFreshInCWeave a)-              (\x -> pure $-                ResAndHandler-                  x-                  (rc >=> (`runContT` pure) . runContViaFreshInC)-              )-          )-          (Just . getResult)-          (weaken g)-      let loop x = c x `catch` (h >=> loop)-      ResAndHandler res h' <- loop a-      -- TODO(KingoftheHomeless): This handler is dubious.-      return (ResAndHandler res $ \x -> (h' x `catch` (h >=> fmap getResult . loop)))---- | This is the effectful state used by 'runContViaFreshInC' when weaving through--- other effectful actions. The point of it is to avoid delimiting computations--- in higher-order effects, by having them return a handler which may be used--- to intercept backtrack exceptions of the current continuation.-data ContFreshState uniq r a = ResAndHandler {-    getResult :: a-  , getHandler :: (uniq, Any) -> Sem (Error (uniq, Any) ': r) a-  }-  deriving Functor
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/EndState.hs
@@ -1,51 +0,0 @@-{-# LANGUAGE TemplateHaskell #-}-module Polysemy.EndState-  (-- * Effect-    EndState(..)--    -- * Actions-  , getEndState--    -- * Interpretations-  , runEndState-  , runEndAtomicState-  ) where--import Polysemy-import Polysemy.Fixpoint-import Polysemy.Reader-import Polysemy.Reader.More--import Polysemy.State-import Polysemy.AtomicState----------------------------------------------------------------------------------- | An effect for getting the end state of a computation in advance.-data EndState s m a where-  GetEndState :: EndState s m s--makeSem ''EndState---------------------------------------------------------------------------------- | Runs an 'EndState' effect by getting the state after the computation--- has finished, and providing it recursively back to calls of 'getEndState'.-runEndState :: forall s r a-             . (Member (State s) r, Member Fixpoint r)-            => Sem (EndState s ': r) a-            -> Sem r a-runEndState =-    runReaderFixSem (get @s)-  . reinterpret (\GetEndState -> ask)----------------------------------------------------------------------------------- | Like 'runEndState', but for 'AtomicState' rather than 'State'.-runEndAtomicState-  :: forall s r a-   . (Member (AtomicState s) r, Member Fixpoint r)-  => Sem (EndState s ': r) a-  -> Sem r a-runEndAtomicState =-    runReaderFixSem (atomicGet @s)-  . reinterpret (\GetEndState -> ask)
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Final/IO.hs
@@ -1,58 +0,0 @@-module Polysemy.Final.IO-  (-    -- * Combinators for Interpreting to the Final Monad-    interpretFinalGlobal--    -- * Interpretations for other effects-  , asyncToIOFinalGlobal-  , resourceToIOFinalGlobal-  ) where--import qualified Control.Concurrent.Async as A-import qualified Control.Exception as X--import Polysemy-import Polysemy.Final-import Polysemy.Final.IO.Internal-import Polysemy.Async-import Polysemy.Resource----------------------------------------------------------------------------------- | 'asyncToIOFinal' implemented using 'interpretFinalGlobal'.------ This behaves semantically very much like 'asyncToIO',--- but doesn't need to spin up an interpreter thread, making it more--- efficient (but not any more safe).-asyncToIOFinalGlobal :: Member (Final IO) r-                     => Sem (Async ': r) a-                     -> Sem r a-asyncToIOFinalGlobal = interpretFinalGlobal $ \case-  Async m -> do-    ins <- getInspectorS-    m'  <- runS m-    liftS $ A.async (inspect ins <$> m')-  Await a -> liftS (A.wait a)-  Cancel a -> liftS (A.cancel a)-{-# INLINE asyncToIOFinalGlobal #-}---------------------------------------------------------------------------------- | 'resourceToIOFinal' implemented using 'interpretFinalGlobal'.------ This behaves semantically very much like 'resourceToIO',--- but doesn't need to spin up an interpreter thread,--- making it more efficient (but not any more safe).-resourceToIOFinalGlobal :: Member (Final IO) r-                        => Sem (Resource ': r) a-                        -> Sem r a-resourceToIOFinalGlobal = interpretFinalGlobal $ \case-  Bracket alloc dealloc use -> do-    a <- runS  alloc-    d <- bindS dealloc-    u <- bindS use-    pure $ X.bracket a d u--  BracketOnError alloc dealloc use -> do-    a <- runS  alloc-    d <- bindS dealloc-    u <- bindS use-    pure $ X.bracketOnError a d u-{-# INLINE resourceToIOFinalGlobal #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Final/IO/Internal.hs
@@ -1,107 +0,0 @@-{-# LANGUAGE TemplateHaskell #-}-module Polysemy.Final.IO.Internal where--import Data.Functor.Compose-import Data.Maybe--import Control.Concurrent.MVar-import Control.Exception-import Control.Monad-import Control.Monad.Trans.Maybe--import Polysemy-import Polysemy.Final-import Polysemy.Internal-import Polysemy.Internal.Union-import Polysemy.Internal.Strategy----------------------------------------------------------------------------------- | Like 'interpretFinal' specialized to 'IO', but also tries very hard--- to preserve state semantics dependant on the order interpreters are run,--- adressing the primary issue with 'Final'.------ Semantically, interpreters written using this behave very much as--- though they were written using 'withLowerToIO'.--- However, this does not need to spawn an interpreter thread, making--- it more efficient (but not any more safe.)------ 'interpretFinalGlobal' operates under the assumption that any effectful--- state which can't be inspected using 'Polysemy.Inspector' can't contain any--- values. For example, the effectful state for 'Polysemy.runError' is--- @'Either' e a@. The inspector for this effectful state only fails if the--- effectful state is a @'Left'@ value, which therefore doesn't contain any--- values of @a@.------ The assumption holds true for all interpreters featured in polysemy,--- and is presumably always true for any properly implemented interpreter.--- 'interpretFinalGlobal' may throw an exception if it is used together with an--- interpreter that uses 'Polysemy.Internal.Union.weave' improperly.-interpretFinalGlobal-    :: forall e a r-     . Member (Final IO) r-    => (forall x n. e n x -> Strategic IO n x)-    -> Sem (e ': r) a-    -> Sem r a-interpretFinalGlobal f sem = withWeavingToFinal $ \s wv ins -> do-  st  <- newMVar s-  res <- runMaybeT $ runViaFinalGlobal st wv ins f sem-  s'  <- readMVar st-  return (fromMaybe bomb res <$ s')-{-# INLINE interpretFinalGlobal #-}--runViaFinalGlobal :: (Member (Final IO) r, Functor f)-                  => MVar (f ())-                  -> (forall x. f (Sem r x) -> IO (f x))-                  -> (forall x. f x -> Maybe x)-                  -> ( forall x n-                     . e n x-                    -> Strategic IO n x-                     )-                  -> Sem (e ': r) a-                  -> MaybeT IO a-runViaFinalGlobal st wv ins f = usingSem $ \u -> case decomp u of-  Right (Weaving e s' wv' ex ins') ->-    fmap ex $ MaybeT $ fmap getCompose $ runStrategy (f e)-          (Compose (Just s'))-          (  maybe-              (pure (Compose Nothing))-              (  fmap Compose-               . runMaybeT-               . runViaFinalGlobal st wv ins f-               . wv'-              )-           . getCompose-          )-          (getCompose >=> ins')-  Left g -> case prj g of-      Just (Weaving (WithWeavingToFinal wav) s' wv' ex' ins') ->-        MaybeT $ fmap (fmap ex' . getCompose) $-          wav-            (Compose (Just s'))-            (  maybe-                (pure (Compose Nothing))-                ( fmap Compose-                . runMaybeT-                . runViaFinalGlobal st wv ins f-                . wv'-                )-             . getCompose-            )-            (getCompose >=> ins')-      _ -> MaybeT $ mask $ \restore -> do-        -- TODO(KingoftheHomeless): Figure out a solution to polysemy issue #205.-        -- Although we're using a different mechanism, the exact same problem manifests-        -- here.-        s   <- takeMVar st-        res <- restore (wv (liftSem (hoist (interpretFinalGlobal f) g) <$ s))-          `onException` putMVar st s-        putMVar st (() <$ res)-        return $ ins res-{-# INLINE runViaFinalGlobal #-}--bomb :: a-bomb = error-  "interpretFinalGlobal: Uninspectable functorial state \-                        \still carried a result. You're likely using an interpreter \-                        \that uses 'weave' improperly. \-                        \See documentation for more information."
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Final/MTL.hs
@@ -1,102 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes #-}-module Polysemy.Final.MTL-  (-    module Polysemy.Final-  , errorToFinal-  , readerToFinal-  , stateToEmbed-  , writerToFinal-  ) where--import Control.Monad.Error.Class hiding (Error)-import Control.Monad.Reader.Class-import Control.Monad.State.Class-import Control.Monad.Writer.Class--import Polysemy-import Polysemy.Final-import Polysemy.Error hiding (throw, catch)-import Polysemy.Reader hiding (ask, local)-import Polysemy.State hiding (get, put)-import Polysemy.Writer hiding (tell, listen, pass)---------------------------------------------------------------------------------- | Run an 'Error' effect through a final 'MonadError'------ /Beware/: Effects that aren't interpreted in terms of the final--- monad will have local state semantics in regards to 'Error' effects--- interpreted this way. See 'Final'.-errorToFinal :: forall m e r a-              . (Member (Final m) r, MonadError e m)-             => Sem (Error e ': r) a-             -> Sem r a-errorToFinal = interpretFinal @m $ \case-  Throw e   -> pure $ throwError e-  Catch m h -> do-    m' <- runS m-    h' <- bindS h-    s  <- getInitialStateS-    pure $ m' `catchError` (h' . (<$ s))-{-# INLINE errorToFinal #-}---------------------------------------------------------------------------------- | Run a 'Reader' effect through a final 'MonadReader'------ /Beware/: Effects that aren't interpreted in terms of the final--- monad will have local state semantics in regards to 'Reader' effects--- interpreted this way. See 'Final'.-readerToFinal :: forall m i r a-               . (Member (Final m) r, MonadReader i m)-              => Sem (Reader i ': r) a-              -> Sem r a-readerToFinal = interpretFinal @m $ \case-  Ask       -> liftS ask-  Local f m -> do-    m' <- runS m-    pure $ local f m'-{-# INLINE readerToFinal #-}---------------------------------------------------------------------------------- | Run a 'State' effect in terms of an underlying 'MonadState' instance------ Although this is not a 'Final' interpreter, as the target monad need not--- actually be the final monad, 'stateToEmbed' still possesses the--- unusual semantics of interpreters that runs--- effects by embedding them into another monad.------ /Beware/: Effects that aren't interpreted in terms of the embedded--- monad will have local state semantics in regards to 'State' effects--- interpreted this way. See 'Final'.-stateToEmbed :: forall m s r a-              . (Member (Embed m) r, MonadState s m)-             => Sem (State s ': r) a-             -> Sem r a-stateToEmbed = interpret $ \case-  Get   -> embed @m get-  Put s -> embed @m (put s)-{-# INLINE stateToEmbed #-}---------------------------------------------------------------------------------- | Run a 'Writer' effect through a final 'MonadWriter'------ /Beware/: Effects that aren't interpreted in terms of the final--- monad will have local state semantics in regards to 'Writer' effects--- interpreted this way. See 'Final'.-writerToFinal :: forall m o r a-               . (Member (Final m) r, MonadWriter o m)-              => Sem (Writer o ': r) a-              -> Sem r a-writerToFinal = interpretFinal @m $ \case-  Tell s    -> liftS (tell s)-  Listen m -> do-    m' <- runS m-    pure $-      (\ ~(s, o) -> (,) o <$> s) <$> listen m'-  Pass m -> do-    m'  <- runS m-    ins <- getInspectorS-    pure $ pass $ do-      t <- m'-      let f = maybe id fst (inspect ins t)-      pure (fmap snd t, f)-{-# INLINE writerToFinal #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Final/More.hs
@@ -1,32 +0,0 @@-module Polysemy.Final.More where--import Polysemy-import Polysemy.Internal-import Polysemy.Internal.Union-import Polysemy.Final----------------------------------------------------------------------------------- | Run a @'Final' ('Sem' r)@ effect if the remaining effect stack is @r@.------ This is sometimes useful for interpreters that make use of--- 'reinterpret', 'raiseUnder', or any of their friends.-runFinalSem :: Sem (Final (Sem r) ': r) a -> Sem r a-runFinalSem = usingSem $ \u -> case decomp u of-  Right (Weaving (WithWeavingToFinal wav) s wv ex ins) ->-    ex <$> wav s (runFinalSem . wv) ins-  Left g -> liftSem (hoist runFinalSem g)-{-# INLINE runFinalSem #-}----------------------------------------------------------------------------------- | Run a @'Final' m@ effect by providing an explicit lowering function.------ /Beware/: The lowering function may be invoked multiple times, so--- __don't do any initialization work inside the lowering function__:--- it will be duplicated.-lowerFinal :: Member (Embed m) r-           => (forall x. Sem r x -> m x)-           -> Sem (Final m ': r) a-           -> Sem r a--- TODO(KingoftheHomeless): Write everything out for efficiency?-lowerFinal f = runFinalSem . finalToFinal embed f . raiseUnder-{-# INLINE lowerFinal #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Final/NonDet.hs
@@ -1,30 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes #-}-module Polysemy.Final.NonDet-  (-    module Polysemy.NonDet-  , nonDetToFinal-  ) where--import Control.Applicative--import Polysemy-import Polysemy.NonDet-import Polysemy.Final---------------------------------------------------------------------------------- | Run an 'NonDet' effect through a final 'Alternative'------ /Beware/: Effects that aren't interpreted in terms of the final--- monad will have local state semantics in regards to 'NonDet' effects--- interpreted this way. See 'Final'.-nonDetToFinal :: forall m r a-               . (Member (Final m) r, Alternative m)-              => Sem (NonDet ': r) a-              -> Sem r a-nonDetToFinal = interpretFinal @m $ \case-  Empty -> pure empty-  Choose left right -> do-    left'  <- runS left-    right' <- runS right-    pure $ left' <|> right'-{-# INLINE nonDetToFinal #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Floodgate.hs
@@ -1,64 +0,0 @@-{-# LANGUAGE TemplateHaskell #-}--module Polysemy.Floodgate-  ( -- * Effect-    Floodgate (..)-    -- * Actions-  , hold-  , release--    -- * Interpretations-  , runFloodgate-  , runFloodgateDry-  ) where--import Control.Monad-import GHC.Types-import Polysemy-import Polysemy.State-import Unsafe.Coerce----------------------------------------------------------------------------------- |------ @since 0.3.1.0-data Floodgate m a where-  Hold    :: m () -> Floodgate m ()-  Release :: Floodgate m ()--makeSem ''Floodgate------------------------------------------------------------------------------------ |------ @since 0.3.1.0-runFloodgate-    :: Sem (Floodgate ': r) a-    -> Sem r a-runFloodgate = fmap snd . runState @[Any] [] . reinterpretH-  ( \case-      Hold m -> do-        m' <- fmap void $ runT m-        -- These 'Any's are here because the monadic action references 'r', and-        -- if we exposed that, 'r' would be an infinite type-        modify (unsafeCoerce @_ @Any (raise $ runFloodgate m') :)-        getInitialStateT--      Release -> do-        ms' <- gets @[Any] (fmap unsafeCoerce . reverse)-        sequence_ ms'-        getInitialStateT-  )------------------------------------------------------------------------------------ | Like 'runFloodgate', but will do a final flush to 'release' anything that--- might still be behind the floodgate.------ @since 0.3.1.0-runFloodgateDry-    :: Sem (Floodgate ': r) a-    -> Sem r a-runFloodgateDry m = runFloodgate $ m <* release-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Fresh.hs
@@ -1,118 +0,0 @@-{-# LANGUAGE BangPatterns, GeneralizedNewtypeDeriving, TemplateHaskell, Trustworthy #-}-module Polysemy.Fresh-  (-- * Effect-    Fresh(..)--    -- * Actions-  , fresh--    -- * Interpretations-  , freshToIO--    -- * Unsafe Interpretations-  , runFreshEnumUnsafe-  , runFreshUnsafePerformIO-  ) where--import Data.Unique--import Polysemy.Internal-import Polysemy.Internal.Union-import System.IO.Unsafe (unsafePerformIO)-import Polysemy-import Polysemy.State---------------------------------------------------------------------------------- | An effect for creating unique objects which may be used as references,--- a la 'Unique'. Polymorphic code making use of 'Fresh' is expected--- to place constraints upon @uniq@ as necessary.------ Any interpreter for 'Fresh' has the responsibilty of ensuring--- that any call to 'fresh' produces an object that /never/--- compares equal to an object produced by a previous call to 'fresh'.-data Fresh uniq m a where-  Fresh :: Fresh uniq m uniq--makeSem ''Fresh---------------------------------------------------------------------------------- | Runs a 'Fresh' effect through generating 'Unique's using 'IO'.-freshToIO :: Member (Embed IO) r-          => Sem (Fresh Unique ': r) a-          -> Sem r a-freshToIO = interpret $ \Fresh -> embed newUnique-{-# INLINE freshToIO #-}---------------------------------------------------------------------------------- | Run a 'Fresh' effect purely by specifying an 'Enum' to be used as the--- type of unique objects.------ __Beware:__ This is safe only if:------   1. This is run after all interpreters which may revert local state---      or produce multiple, inconsistent instances of local state.---      This includes interpreters that may backtrack or produce multiple results---      (such as 'Polysemy.Error.runError' or 'Polysemy.NonDet.runNonDet').------   2. You don't use any interpreter which may cause the final monad---      to revert local state or produce multiple, inconsistent instances of local---      state. This includes certain 'Polysemy.Final.Final'/@lower-@ interpeters---      such as 'Polysemy.Error.lowerError' or 'Polysemy.Final.MTL.errorToFinal',---      as well as interpreters for 'Polysemy.Async.Async'.------ Prefer 'freshToIO' whenever possible.--- If you can't use 'runFreshEnumUnsafe' safely, nor use 'freshToIO', consider--- 'runFreshUnsafePerformIO'.-runFreshEnumUnsafe :: forall n a r-                    . Enum n-                   => Sem (Fresh n ': r) a-                   -> Sem r a-runFreshEnumUnsafe =-    (fmap snd .)-  $ (runState @n (toEnum 0) .)-  $ reinterpret-  $ \Fresh -> do-    s <- get-    put $! succ s-    return s-{-# INLINE runFreshEnumUnsafe #-}---------------------------------------------------------------------------------- | Runs a 'Fresh' effect through generating 'Unique's using--- 'unsafePerformIO'.------ Ironically, despite the fact that this uses 'unsafePerformIO', and--- 'runFreshUnsafe' uses no unsafe operations whatsoever, this is still--- typically safer to use than 'runFreshUnsafe', although 'runFreshUnsafe'--- is perhaps more efficient.------ The worst thing that this particular use of 'unsafePerformIO' could result--- in is the loss of referential transparency, as rerunning an interpreter stack--- using 'runFreshUnsafePerformIO' will create different 'Unique's. This should--- never matter.------ This could be potentially be less efficient than 'runFreshUnsafe'.------ If you ever observe that multiple invocations of 'fresh' produce the same--- 'Unique' under 'runFreshUnsafePerformIO', then open an issue over at the--- GitHub repository.-runFreshUnsafePerformIO :: Sem (Fresh Unique ': r) a-                        -> Sem r a-runFreshUnsafePerformIO = usingSem $ \u ->-  case decomp u of-    Right (Weaving Fresh s _ ex _) -> do-      let !uniq = unsafePerformIO (newUnique' u)-          {-# NOINLINE uniq #-}-      pure $ ex (uniq <$ s)-    Left g -> liftSem (hoist runFreshUnsafePerformIO g)--- KingoftheHomeless: I've tried very hard to prevent optimizations from--- sharing the call to 'unsafePerformIO'.--- The inlining of 'interpret' is so that I can give--- 'u' to 'newUnique'', and thus prevent 'uniq' from floating outside the--- lambda. This interpreter might even be safe to INLINE, but I'm not taking--- any chances.-{-# NOINLINE runFreshUnsafePerformIO #-}--newUnique' :: Union (Fresh Unique ': r) (Sem (Fresh Unique ': r)) a -> IO Unique-newUnique' (Union _ _) = newUnique-{-# NOINLINE newUnique' #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Input/Streaming.hs
@@ -1,86 +0,0 @@-module Polysemy.Input.Streaming-  ( -- * Underlying Effect-    module Polysemy.Input--    -- * Actions-  , yieldInput-  , yieldRace-  , exhaust--    -- * Intepretations-  , runInputViaStream-  , runInputViaInfiniteStream-  ) where--import qualified Control.Concurrent.Async as A-import           Data.Functor.Of-import           Data.Void-import           Polysemy-import           Polysemy.Final-import           Polysemy.Input-import           Polysemy.State-import qualified Streaming as S-import qualified Streaming.Prelude as S----runInputViaStream-    :: S.Stream (Of i) (Sem r) ()-    -> InterpreterFor (Input (Maybe i)) r-runInputViaStream stream-  = evalState (Just stream)-  . reinterpret ( \Input ->-      get >>= \case-        Nothing -> pure Nothing-        Just s ->-          raise (S.inspect s) >>= \case-            Left () -> pure Nothing-            Right (i :> s') -> do-              put $ Just s'-              pure $ Just i-  )---runInputViaInfiniteStream-    :: forall i r-     . S.Stream (Of i) (Sem r) Void-    -> InterpreterFor (Input i) r-runInputViaInfiniteStream stream-  = evalState stream-  . reinterpret ( \Input -> do-      s <- get-      raise (S.inspect s) >>= \case-        Left g -> absurd g-        Right (i :> s') -> do-          put s'-          pure i-  )---yieldRace-    :: Members-        '[ Final IO-         , Input i1-         , Input i2-         ] r-    => S.Stream (S.Of (Either i1 i2)) (Sem r) ()-yieldRace = do-  z <- S.lift $ withStrategicToFinal $ do-         input1 <- runS input-         input2 <- runS input-         pure $ fmap sequenceEither $ A.race input1 input2-  S.yield z---sequenceEither :: Functor f => Either (f a) (f b) -> f (Either a b)-sequenceEither (Left fa) = Left <$> fa-sequenceEither (Right fb) = Right <$> fb---yieldInput :: Member (Input i) r => S.Stream (Of i) (Sem r) ()-yieldInput = S.lift input >>= S.yield---exhaust :: Member (Input i) r => S.Stream (Of i) (Sem r) a-exhaust = S.repeatM input-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/KVStore.hs
@@ -1,101 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes, TemplateHaskell #-}--module Polysemy.KVStore-  ( -- * Effect-    KVStore (..)--    -- * Actions-  , lookupKV-  , lookupOrThrowKV-  , existsKV-  , writeKV-  , deleteKV-  , updateKV-  , modifyKV--    -- * Interpretations-  , runKVStoreAsState-  , runKVStorePurely-  ) where--import qualified Data.Map as M-import           Data.Maybe (isJust)-import           Polysemy-import           Polysemy.Error-import           Polysemy.State------------------------------------------------------------------------------------ | Models things like Redis, HTTP GET/POST, etc. Things that are keyed, have--- a value, and may or may not be there.-data KVStore k v m a where-  LookupKV :: k -> KVStore k v m (Maybe v)-  UpdateKV :: k -> Maybe v -> KVStore k v m ()--makeSem ''KVStore---writeKV :: Member (KVStore k v) r => k -> v -> Sem r ()-writeKV k = updateKV k . Just-{-# INLINE writeKV #-}---deleteKV :: forall k v r. Member (KVStore k v) r => k -> Sem r ()-deleteKV k = updateKV k (Nothing @v)-{-# INLINE deleteKV #-}------------------------------------------------------------------------------------ |------ @since 0.3.1.0-lookupOrThrowKV-    :: Members '[ KVStore k v-                , Error e-                ] r-    => (k -> e)-    -> k-    -> Sem r v-lookupOrThrowKV f k =-  fromEither . maybe (Left $ f k) Right =<< lookupKV k------------------------------------------------------------------------------------ |------ @since 0.3.1.0-existsKV :: forall k v r. Member (KVStore k v) r => k -> Sem r Bool-existsKV = fmap isJust . lookupKV @k @v------------------------------------------------------------------------------------ |------ @since 0.3.1.0-modifyKV-    :: Member (KVStore k v) r-    => v  -- ^ Default value if the key isn't present-    -> (v -> v)-    -> k-    -> Sem r ()-modifyKV d f k =-  lookupKV k >>= \case-    Just v  -> writeKV k $ f v-    Nothing -> writeKV k $ f d---runKVStoreAsState :: Ord k => Sem (KVStore k v ': r) a -> Sem (State (M.Map k v) ': r) a-runKVStoreAsState = reinterpret $ \case-  LookupKV k   -> gets $ M.lookup k-  UpdateKV k v -> modify $ M.alter (const v) k-{-# INLINE runKVStoreAsState #-}---runKVStorePurely-    :: Ord k-    => M.Map k v-    -> Sem (KVStore k v ': r) a-    -> Sem r (M.Map k v, a)-runKVStorePurely m = runState m . runKVStoreAsState-{-# INLINE runKVStorePurely #-}-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Operators.hs
@@ -1,270 +0,0 @@--- | Operators meant as replacements for traditional 'Sem' type and 'Member' /--- 'Members' constraints, that allow you to specify types of your actions and--- interpreters in more concise way, without mentioning unnecessary details:------ @--- foo :: 'Member' ('Embed' 'IO') r => 'String' -> 'Int' -> 'Sem' r ()--- @------ can be written simply as:------ @--- foo :: 'String' -> 'Int' -> 'IO' '~@>' ()--- @------ Working example with operators:------ @--- import Data.Function--- import Polysemy--- import Polysemy.Operators--- import Polysemy.Random------ data ConsoleIO m a where---   WriteStrLn ::           'String' -> ConsoleIO m ()---   ReadStrLn  ::                     ConsoleIO m 'String'---   ShowStrLn  :: 'Show' a => a      -> ConsoleIO m ()------ 'makeSem' ''ConsoleIO------ -- runConsoleIO :: Member (Embed IO) r => Sem (ConsoleIO : r) a -> Sem r a--- runConsoleIO :: ConsoleIO : r '@>' a -> 'IO' '~@' r '@>' a--- runConsoleIO = 'interpret' \\case---   WriteStrLn s -> 'sendM' '$' 'putStrLn' s---   ReadStrLn    -> 'sendM'   'getLine'---   ShowStrLn  v -> 'sendM' '$' 'print' v------ main :: 'IO' ()--- main = program---      'Data.Function.&' runConsoleIO---      'Data.Function.&' 'Polysemy.Random.runRandomIO'---      'Data.Function.&' 'runM'------ -- program :: Members \'[Random, ConsoleIO] r => Sem r ()--- program :: \'['Polysemy.Random', ConsoleIO] '>@>' ()--- program = do---   writeStrLn "It works! Write something:"---   val <- readStrLn---   writeStrLn '$' "Here it is!: " '++' val---   num <- 'Polysemy.Random.random' \@'Int'---   writeStrLn '$' "Some random number:"---   showStrLn num--- @------ Please keep in mind that constraints created through these operators are--- limited to the action they are being used on, for example:------ @--- foo :: (forall x. r '@>' x -> 'IO' x)---     -> 'IO' (forall a. Foo : r '@>' a -> 'IO' '~@' r '@>' a)--- @------ The first argument in the signature above won't have access to the--- @('IO' ~\@)@ constraint in the result - in such cases, use a normal--- constraint instead:------ @--- foo :: 'Member' ('Embed' 'IO') r---     => (forall x. r '@>' x -> 'IO' x)---     -> 'IO' (forall a. Foo : r '@>' a -> r '@>' a)--- @------ See the documentation of specific operators for more details.--module Polysemy.Operators-  ( -- * 'Sem' operators-    type (@>)-  , type (@-)-  , type (@~)-    -- $SemOperators--  , -- * 'Member' operators-    type (>@)-  , type (-@)-  , type (~@)-    -- $MemberOperators--  , -- * Combined operators-    type (>@>)-  , type (-@>)-  , type (~@>)-    -- $CombinedOperators-  ) where--import Polysemy---- Miscellaneous ---------------------------------------------------------------- | Gets list of effects from 'Sem'.-type family SemList s where-  SemList (Sem r _) = r---- Operators --------------------------------------------------------------------- $SemOperators--- Infix equivalents of 'Sem' with versions for specifiying list of effects--- ('@>'), single effect ('@-') and single monad ('@~') as effects of union.--- Use ('>@>'), ('-@>') or ('~@>') instead if you are not making any--- transformations on union and just want to use some members instead.------ __Examples:__------ 'Sem' with list of multiple effects:------ @--- foo :: 'Sem' ('Polysemy.State.State' 'Int' : r) ()--- @------ can be written as:------ @--- foo :: 'Polysemy.State.State' 'Int' : r '@>' ()--- @------ 'Sem' with list of one effect:------ @--- foo :: 'Sem' \'['Polysemy.State.State' 'Int'] ()--- @------ can be written as both (with the latter preferred):------ @--- foo :: \'['Polysemy.State.State' 'Int'] '@>' ()--- @------ and:------ @--- foo :: 'Polysemy.State.State' 'Int' '@-' ()--- @------ where effect without list gets put into one automatically.------ 'Sem' with __exactly__ one, lifted monad:------ @--- foo :: 'Sem' \'['Embed' 'IO'] ()--- @------ can be written simply as:------ @--- foo :: 'IO' '@~' ()--- @------ and will be automatically lifted and put into list.-infix 2 @>, @-, @~--type (@>)   = Sem-type (@-) e = Sem '[e]-type (@~) m = Sem '[Embed m]---- $MemberOperators--- Infix equivalents of 'Member'(s) constraint used directly in /return/ type,--- specifiying list of members ('>@'), single member ('-@') or single monad--- ('~@'), meant to be paired with some of the 'Sem' operators (('@>'), ('@-')--- and ('@~')). Use ('>@>'), ('-@>') or ('~@>') instead if you are not making--- any transformations on union and just want to use some members instead.------ __Examples:__------ List of multiple members:------ @--- foo :: 'Members' \'['Polysemy.State.State' 'Int', 'Polysemy.Input.Input' 'String'] r => 'Sem' ('Polysemy.Output.Output' ['String'] : r) () -> 'Sem' r ()--- @------ can be written as:------ @--- foo :: 'Polysemy.Output.Output' ['String'] : r '@>' () -> \'['Polysemy.State.State' 'Int', 'Polysemy.Input.Input' 'String'] '>@' r '@>' ()--- @------ One member:------ @--- foo :: 'Member' ('Polysemy.State.State' 'Int') r => 'Sem' ('Polysemy.Output.Output' ['String'] : r) () -> 'Sem' r ()--- @------ can be written as both (with the latter preferred):------ @--- foo :: 'Polysemy.Output.Output' ['String'] : r '@>' () -> \'['Polysemy.State.State' 'Int'] '>@' r '@>' ()--- @------ and:------ @--- foo :: 'Polysemy.Output.Output' ['String'] : r '@>' () -> 'Polysemy.State.State' 'Int' '-@' r '@>' ()--- @------ __Exactly__ one, lifted monad as a member:------ @--- foo :: 'Member' ('Embed' 'IO') r => 'Sem' ('Polysemy.Output.Output' ['String'] : r) () -> 'Sem' r ()--- @------ can be written simply as:------ @--- foo :: 'Polysemy.Output.Output' ['String'] : r '@>' () -> 'IO' '~@' r '@>' ()--- @-infix 1 >@, -@, ~@--type (>@) es s = Members es       (SemList s) => s-type (-@) e  s = Member  e        (SemList s) => s-type (~@) m  s = Member  (Embed m) (SemList s) => s---- $CombinedOperators--- Joined versions of one of ('>@'), ('-@'), ('~@') and ('@>') with implicit,--- hidden list of effects in union --- suited for actions that only use one--- 'Sem' in their type.------ __Examples:__------ List of members over some 'Sem':------ @--- foo :: 'Members' \'['Polysemy.State.State' 'String', 'Polysemy.Input.Input' 'Int'] r => 'String' -> 'Int' -> 'Sem' r ()--- @------ can be written as:------ @--- foo :: 'String' -> 'Int' -> \'['Polysemy.State.State' 'String', 'Polysemy.Input.Input' 'Int'] '>@>' ()--- @------ Single member:------ @--- foo :: 'Member' ('Polysemy.Input.Input' 'Int') r => 'String' -> 'Int' -> 'Sem' r ()--- @------ can be written as both (with the latter preferred):------ @--- foo :: 'String' -> 'Int' -> \'['Polysemy.Input.Input' 'Int'] '>@>' ()--- @------ and:------ @--- foo :: 'String' -> 'Int' -> 'Polysemy.Input.Input' 'Int' '-@>' ()--- @------ __Exactly__ one, lifted monad as a member:------ @--- foo :: 'Member' ('Embed' 'IO') r => 'Sem' r ()--- @------ can be written simply as:------ @--- foo :: 'IO' '~@>' ()--- @-infix 1 >@>, -@>, ~@>--type (>@>) es a = forall r. Members es       r => Sem r a-type (-@>) e  a = forall r. Member  e        r => Sem r a-type (~@>) m  a = forall r. Member  (Embed m) r => Sem r a
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Random.hs
@@ -1,57 +0,0 @@-{-# LANGUAGE TemplateHaskell #-}--module Polysemy.Random-  ( -- * Effect-    Random (..)--    -- * Actions-  , random-  , randomR--    -- * Interpretations-  , runRandom-  , runRandomIO-  ) where--import           Polysemy-import           Polysemy.State-import qualified System.Random as R----------------------------------------------------------------------------------- | An effect capable of providing 'R.Random' values.-data Random m a where-  Random :: R.Random x => Random m x-  RandomR :: R.Random x => (x, x) -> Random m x--makeSem ''Random------------------------------------------------------------------------------------ | Run a 'Random' effect with an explicit 'R.RandomGen'.-runRandom-    :: forall q r a-     . R.RandomGen q-    => q-    -> Sem (Random ': r) a-    -> Sem r (q, a)-runRandom q = runState q . reinterpret (\case-  Random -> do-    ~(a, q') <- gets @q R.random-    put q'-    pure a-  RandomR r -> do-    ~(a, q') <- gets @q $ R.randomR r-    put q'-    pure a-                                       )-{-# INLINE runRandom #-}------------------------------------------------------------------------------------ | Run a 'Random' effect by using the 'IO' random generator.-runRandomIO :: Member (Embed IO) r => Sem (Random ': r) a -> Sem r a-runRandomIO m = do-  q <- embed @IO R.newStdGen-  snd <$> runRandom q m-{-# INLINE runRandomIO #-}-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Reader/Compact.hs
@@ -1,37 +0,0 @@-module Polysemy.Reader.Compact where--import           Polysemy-import           Polysemy.Reader--import           GHC.Compact---------------------------------------------------------------------------------- | Run a 'Reader' effect by compacting a value; otherwise behaves as normal.--- Useful for 'Reader' effects which provide a large structure.-runReaderWithCompacted-    :: forall r i a-     . Member (Embed IO) r-    => i-    -> Sem (Reader i ': r) a-    -> Sem r a-runReaderWithCompacted i sem = do-    compacted <- embed (compactWithSharing i)-    runReaderWithExistingCompacted compacted sem-{-# INLINE runReaderWithCompacted #-}---------------------------------------------------------------------------------- | Run a 'Reader' effect with a value in a compact region. Will not add --- 'local' values to the existing region, but will create a new region for it.-runReaderWithExistingCompacted-    :: forall r i a-     . Member (Embed IO) r-    => Compact i-    -> Sem (Reader i ': r) a-    -> Sem r a-runReaderWithExistingCompacted i = interpretH $ \case-    Ask       -> pureT (getCompact i)-    Local f m -> do-        mm <- runT m-        let transformed = f (getCompact i)-        raise $ runReaderWithCompacted transformed mm-{-# INLINE runReaderWithExistingCompacted #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Reader/More.hs
@@ -1,27 +0,0 @@-{-# LANGUAGE RecursiveDo #-}-module Polysemy.Reader.More-  (-    module Polysemy.Reader-    -- * Interpretations-  , runReaderFixSem-  ) where--import Polysemy-import Polysemy.Reader-import Polysemy.Fixpoint----------------------------------------------------------------------------------- | Runs a 'Reader' effect by running a monadic action /once/, after the--- 'Sem' has completed, and then providing the result to each request--- recursively.-runReaderFixSem :: forall i r a-                 . Member Fixpoint r-                => Sem r i-                -> Sem (Reader i ': r) a-                -> Sem r a-runReaderFixSem m sem = do-  rec-    a <- runReader i sem-    i <- m-  return a-{-# INLINE runReaderFixSem #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/RevState.hs
@@ -1,155 +0,0 @@-{-# LANGUAGE RecursiveDo, TemplateHaskell #-}-module Polysemy.RevState- ( -- * Effect-   RevState (..)--   -- * Actions- , revState- , revGet- , revPut- , revModify--   -- * Interpretations- , runRevState- , runLazyRevState- ) where--import Control.Monad.Fix--import Polysemy-import Polysemy.Fixpoint-import Polysemy.Internal-import Polysemy.Internal.Union----------------------------------------------------------------------------------- | A 'Polysemy.State.State' effect for threading state /backwards/ instead--- of forwards through a computation.-newtype RevState s m a where-  RevState :: (s -> (s, a)) -> RevState s m a--makeSem_ ''RevState------------------------------------------------------------------------------------ | Gets the state as sent from the next call to 'revState'--- \/ 'revPut' \/ 'revModify', use it, and send a new state into the past.-revState :: forall s a r-          . Member (RevState s) r-         => (s -> (s, a))-         -> Sem r a----------------------------------------------------------------------------------- | Gets the state as sent from the next call to 'revState' \/ 'revPut'--- \/ 'revModify'.-revGet :: forall s r-        . Member (RevState s) r-       => Sem r s-revGet = revState $ \s -> (s, s)----------------------------------------------------------------------------------- | Sends a new state into the past.-revPut :: forall s r-        . Member (RevState s) r-       => s-       -> Sem r ()-revPut s = revState $ \_ -> (s, ())----------------------------------------------------------------------------------- | Gets the state as sent from the next call to 'revState'--- \/ 'revModify' \/ 'revPut', modify it, and return it into the past.-revModify :: forall s r-           . Member (RevState s) r-          => (s -> s)-          -> Sem r ()-revModify f = revState $ \s -> (f s, ())------------------------------------------------------------------------------------ | Run a 'RevState' effect with local state that is propagated /backwards/--- through the computation, from last action to first.-runRevState :: Member Fixpoint r-            => s-            -> Sem (RevState s ': r) a-            -> Sem r (s, a)-runRevState s =-   (`runRevStateT` s)-  . runRevStateInC----------------------------------------------------------------------------------- | Run a 'RevState' effect with local state that is lazily propagated--- /backwards/ through the computation, from last action to first.-runLazyRevState :: Member Fixpoint r-                => s-                -> Sem (RevState s ': r) a-                -> Sem r (s, a)-runLazyRevState s =-   (`runLazyRevStateT` s)-  . runLazyRevStateInC--newtype RevStateT s m a = RevStateT { runRevStateT :: s -> m (s, a) }-  deriving (Functor)--instance MonadFix m => Applicative (RevStateT s m) where-  pure a = RevStateT $ \s -> pure (s, a)-  ff <*> fa = RevStateT $ \s -> do-    rec-      (s'', f) <- runRevStateT ff s'-      (s',  a) <- runRevStateT fa s-    return (s'', f a)-  fa *> fb = fa >>= \_ -> fb--instance MonadFix m => Monad (RevStateT s m) where-  m >>= f = RevStateT $ \s -> do-    rec-      (s'', a) <- runRevStateT m s'-      (s',  b) <- runRevStateT (f a) s-    return (s'', b)--newtype LazyRevStateT s m a = LazyRevStateT { runLazyRevStateT :: s -> m (s, a) }-  deriving (Functor)--instance MonadFix m => Applicative (LazyRevStateT s m) where-  pure a = LazyRevStateT $ \s -> pure (s, a)-  ff <*> fa = LazyRevStateT $ \s -> do-    rec-      ~(s'', f) <- runLazyRevStateT ff s'-      ~(s',  a) <- runLazyRevStateT fa s-    return (s'', f a)-  fa *> fb = fa >>= \_ -> fb--instance MonadFix m => Monad (LazyRevStateT s m) where-  m >>= f = LazyRevStateT $ \s -> do-    rec-      ~(s'', a) <- runLazyRevStateT m s'-      ~(s',  b) <- runLazyRevStateT (f a) s-    return (s'', b)--runRevStateInC :: Member Fixpoint r-               => Sem (RevState s ': r) a-               -> RevStateT s (Sem r) a-runRevStateInC = usingSem $ \u -> RevStateT $ \s ->-  case decomp u of-    Right (Weaving (RevState f) st _ ex _) ->-      return $ (ex . (<$ st)) <$> f s-    Left g ->-      liftSem $-        weave-          (s, ())-          (uncurry runRevState)-          (Just . snd)-          g--runLazyRevStateInC :: Member Fixpoint r-                   => Sem (RevState s ': r) a-                   -> LazyRevStateT s (Sem r) a-runLazyRevStateInC = usingSem $ \u -> LazyRevStateT $ \s ->-  case decomp u of-    Right (Weaving (RevState f) st _ ex _) ->-      return $ (ex . (<$ st)) <$> f s-    Left g ->-      liftSem $-        weave-          (s, ())-          (uncurry runLazyRevState)-          (Just . snd)-          g
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/SetStore.hs
@@ -1,36 +0,0 @@-{-# LANGUAGE TemplateHaskell #-}--module Polysemy.SetStore where--import           Data.Foldable-import qualified Data.Set as S-import           Polysemy-import           Polysemy.KVStore----data SetStore k v m a where-  AddS :: k -> v -> SetStore k v m ()-  DelS :: k -> v -> SetStore k v m ()-  MemberS :: k -> v -> SetStore k v m Bool--makeSem ''SetStore---runSetStoreAsKVStore-    :: ( Member (KVStore k (S.Set v)) r-       , Ord v-       )-    => Sem (SetStore k v ': r) x -> Sem r x-runSetStoreAsKVStore = interpret $ \case-  AddS k v ->-    lookupKV k >>= \case-      Just s  -> writeKV k $ S.insert v s-      Nothing -> writeKV k $ S.singleton v-  DelS k v -> do-    ms <- lookupKV k-    for_ ms $ writeKV k . S.delete v-  MemberS k v ->-    pure . maybe False (S.member v) =<< lookupKV k-{-# INLINE runSetStoreAsKVStore #-}-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Several.hs
@@ -1,71 +0,0 @@-{-# LANGUAGE TemplateHaskell #-}--module Polysemy.Several-  ( -- * Data-    HList(..)-  , TypeMap-  , TypeConcat-  , runSeveral-  ) where--import Polysemy-import Data.Kind----------------------------------------------------------------------------------- | A list capable of storing values of different types. Like the Sem type,--- it uses a type level list to keep track of what's stored inside. Creating an--- HList looks like:------ > 1 ::: "test" ::: True ::: HNil-infixr 5 :::-data HList a where-    HNil  :: HList '[]-    (:::) :: a -> HList (b :: [Type]) -> HList (a ': b)----------------------------------------------------------------------------------- | A map function over type level lists. For example, the following two--- lines are equivalent:------ > TypeMap Reader [Int, String, False]--- > [Reader Int, Reader String, Reader Bool]-type family TypeMap (f :: a -> b) (xs :: [a]) where-    TypeMap _ '[]       = '[]-    TypeMap f (x ': xs) = f x ': TypeMap f xs----------------------------------------------------------------------------------- | Like ++ but at the type level. The following two lines are equivalent:------ > TypeConcat [Int, String] [Bool]--- > [Int, String, Bool]-type family TypeConcat (a :: [t]) (b :: [t]) where-    TypeConcat '[] b = b-    TypeConcat (a ': as) b = a ': TypeConcat as b----------------------------------------------------------------------------------- | A helper function for building new runners which accept HLists intsead of--- individual elements. If you would normally write------ > f 5 . f "Text" . f True------ then this function can turn that into------ > runSeveral f (True ::: "Text" ::: 5 ::: HNil)------ For example, a runReaders function could be implemented as:------ > runReaders :: HList t -> Sem (TypeConcat (TypeMap Reader t) r) a -> Sem r a--- > runReaders = runSeveral runReader------ Likewise, runStates could be the following if you didn't want the returned--- state:------ > runStates :: HList t -> Sem (TypeConcat (TypeMap State t) r) a -> Sem r a--- > runStates = runSeveral (fmap (fmap snd) . runState)-runSeveral-    :: (forall r' k x. k -> Sem (e k ': r') x -> Sem r' x)-    -> HList t-    -> Sem (TypeConcat (TypeMap e t) r) a-    -> Sem r a-runSeveral f (a ::: as) = runSeveral f as . f a-runSeveral _ HNil       = id-
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Shift.hs
@@ -1,316 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes, Trustworthy #-}-module Polysemy.Shift-  (-    module Polysemy.Cont-    -- * Effect-  , Shift(..)--    -- * Actions-  , trap-  , invoke-  , abort-  , reset-  , reset'-  , shift--    -- * Interpretations-  , runShiftPure-  , runShiftM-  , shiftToFinal-  , runShiftWithCPure-  , runShiftWithCM--  , runContShiftPure-  , runContShiftM-  , runContShiftWithCPure-  , runContShiftWithCM--    -- * Unsafe Interpretations-  , runShiftUnsafe-  , runShiftWithCUnsafe-  , runContShiftUnsafe-  , runContShiftWithCUnsafe-  ) where---import Polysemy-import Polysemy.Cont-import Polysemy.Cont.Internal-import Polysemy.Shift.Internal-import Polysemy.Final-import Control.Monad.Cont (ContT(..))--import Polysemy.Internal-import Polysemy.Internal.Union----------------------------------------------------------------------------------- | A variant of 'callCC'.--- Executing the provided continuation will not abort execution.------ Any effectful state of effects which have been run before the interpreter for--- 'Shift' will be embedded in the return value of the continuation,--- and therefore the continuation won't have any apparent effects unless these--- effects are interpreted in the final monad.------ Any higher-order actions will also not interact with the continuation in any--- meaningful way; i.e. 'Polysemy.Reader.local' or 'Polysemy.Writer.censor' does--- not affect it, 'Polysemy.Error.catch' will fail to catch any of its exceptions,--- and 'Polysemy.Writer.listen' will always return 'mempty'.------ The provided continuation may fail locally in its subcontinuations.--- It may sometimes become necessary to handle such cases, in--- which case such failure may be detected by using 'reset\'' together--- with the provided continuation.-shift :: forall ref s r a-       . Member (Shift ref s) r-      => ((a -> Sem r s) -> Sem r s)-      -> Sem r a-shift cc = trap @ref $ \ref -> cc (invoke ref)-{-# INLINE shift #-}---------------------------------------------------------------------------------- | Runs a 'Shift' effect by providing @'pure' '.' 'Just'@ as the final--- continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ This is a safe variant of 'runShiftUnsafe', as this may only be used--- as the final interpreter before 'run'.-runShiftPure :: Sem '[Shift (Ref (Sem '[]) (Maybe a)) a] a-             -> Sem '[] (Maybe a)-runShiftPure = runShiftUnsafe-{-# INLINE runShiftPure #-}---------------------------------------------------------------------------------- | Runs a 'Shift' effect by providing @'pure' '.' 'Just'@ as the final--- continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ This is a safe variant of 'runShiftUnsafe', as this may only be used--- as the final interpreter before 'runM'.-runShiftM :: Sem '[Shift (Ref (Sem '[Embed m]) (Maybe a)) a, Embed m] a-          -> Sem '[Embed m] (Maybe a)-runShiftM = runShiftUnsafe-{-# INLINE runShiftM #-}---------------------------------------------------------------------------------- | Runs a 'Shift' effect in terms of a final 'ContT'------ /Beware/: Effects that aren't interpreted in terms of the final monad--- will have local state semantics in regards to 'Shift' effects--- interpreted this way. See 'Final'.-shiftToFinal :: forall s m a r-              .  (Member (Final (ContT (Maybe s) m)) r, Monad m)-              => Sem (Shift (Ref m (Maybe s)) s ': r) a-              -> Sem r a-shiftToFinal = interpretFinal $ \case-  Trap main -> do-    main'         <- bindS main-    s             <- getInitialStateS-    Inspector ins <- getInspectorS-    pure $ ContT $ \c ->-      runContT (main' (Ref (c . (<$ s)) <$ s)) (pure . ins)-  Invoke ref a -> liftS $ ContT $ \c -> runRef ref a >>= maybe (pure Nothing) c-  Abort s -> pure $ ContT $ \_ -> pure (Just s)-  Reset main -> do-    main'         <- runS main-    Inspector ins <- getInspectorS-    liftS $ ContT $ \c ->-      runContT main' (pure . ins) >>= maybe (pure Nothing) c-  Reset' main -> do-    main'         <- runS main-    Inspector ins <- getInspectorS-    liftS $ ContT $ \c ->-      runContT main' (pure . ins) >>= c-{-# INLINE shiftToFinal #-}---------------------------------------------------------------------------------- | Runs a 'Shift' effect by explicitly providing a final--- continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ This is a safe variant of 'runShiftWithCUnsafe', as this may only be used--- as the final interpreter before 'run'.-runShiftWithCPure :: (a -> Sem '[] (Maybe b))-                  -> Sem '[Shift (Ref (Sem '[]) (Maybe b)) b] a-                  -> Sem '[] (Maybe b)-runShiftWithCPure = runShiftWithCUnsafe-{-# INLINE runShiftWithCPure #-}---------------------------------------------------------------------------------- | Runs a 'Shift' effect by explicitly providing a final--- continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ This is a safe variant of 'runShiftWithCUnsafe', as this may only be used--- as the final interpreter before 'runM'.-runShiftWithCM :: (a -> Sem '[Embed m] (Maybe b))-               -> Sem '[Shift (Ref (Sem '[Embed m]) (Maybe b)) b, Embed m] a-               -> Sem '[Embed m] (Maybe b)-runShiftWithCM = runShiftWithCUnsafe-{-# INLINE runShiftWithCM #-}---------------------------------------------------------------------------------- | Runs a 'Cont' and a 'Shift' effect simultaneously by providing--- @'pure' '.' 'Just'@ as the final continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ This is a safe variant of 'runContShiftUnsafe', as this may only be used--- as the final interpreter before 'run'.-runContShiftPure :: Sem [ Cont (Ref (Sem '[]) (Maybe a))-                        , Shift (Ref (Sem '[]) (Maybe a)) a-                        ] a-                 -> Sem '[] (Maybe a)-runContShiftPure = runContShiftUnsafe-{-# INLINE runContShiftPure #-}---------------------------------------------------------------------------------- | Runs a 'Cont' and a 'Shift' effect simultaneously by providing--- @'pure' '.' 'Just'@ as the final continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ This is a safe variant of 'runContShiftUnsafe', as this may only be used--- as the final interpreter before 'runM'.-runContShiftM :: Sem [ Cont (Ref (Sem '[Embed m]) (Maybe a))-                     , Shift (Ref (Sem '[Embed m]) (Maybe a)) a-                     , Embed m-                     ] a-              -> Sem '[Embed m] (Maybe a)-runContShiftM = runContShiftUnsafe-{-# INLINE runContShiftM #-}---------------------------------------------------------------------------------- | Runs a 'Cont' and a 'Shift' effect simultaneously by explicitly providing--- a final continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ This is a safe variant of 'runContShiftWithCUnsafe', as this may only be--- used as the final interpreter before 'run'.-runContShiftWithCPure :: (a -> Sem '[] (Maybe s))-                      -> Sem [ Cont (Ref (Sem '[]) (Maybe s))-                             , Shift (Ref (Sem '[]) (Maybe s)) s-                             ] a-                      -> Sem '[] (Maybe s)-runContShiftWithCPure = runContShiftWithCUnsafe-{-# INLINE runContShiftWithCPure #-}---------------------------------------------------------------------------------- | Runs a 'Cont' and a 'Shift' effect simultaneously by explicitly providing--- a final continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ This is a safe variant of 'runContShiftWithCUnsafe', as this may only be used--- as the final interpreter before 'runM'.-runContShiftWithCM :: (a -> Sem '[Embed m] (Maybe s))-                   -> Sem [ Cont (Ref (Sem '[Embed m]) (Maybe s))-                          , Shift (Ref (Sem '[Embed m]) (Maybe s)) s-                          , Embed m-                          ] a-                   -> Sem '[Embed m] (Maybe s)-runContShiftWithCM = runContShiftWithCUnsafe-{-# INLINE runContShiftWithCM #-}----------------------------------------------------------------------------------- | Runs a 'Shift' effect by providing @'pure' '.' 'Just'@--- as the final continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ __Beware__: This interpreter will invalidate all higher-order effects of any--- interpreter run after it; i.e. 'Polysemy.Reader.local' and--- 'Polysemy.Writer.censor' will be no-ops, 'Polysemy.Error.catch' will fail--- to catch exceptions, and 'Polysemy.Writer.listen' will always return 'mempty'.------ __You should therefore use 'runShiftUnsafe' /after/ running all__--- __interpreters for your higher-order effects.__-runShiftUnsafe :: Sem (Shift (Ref (Sem r) (Maybe a)) a ': r) a -> Sem r (Maybe a)-runShiftUnsafe = runShiftWithCUnsafe (pure . Just)-{-# INLINE runShiftUnsafe #-}---------------------------------------------------------------------------------- | Runs a 'Shift' effect by explicitly providing a final continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that any--- continuation may fail locally.------ __Beware__: This interpreter will invalidate all higher-order effects of any--- interpreter run after it; i.e. 'Polysemy.Reader.local' and--- 'Polysemy.Writer.censor' will be no-ops, 'Polysemy.Error.catch' will fail--- to catch exceptions, and 'Polysemy.Writer.listen' will always return 'mempty'.------ __You should therefore use 'runShiftWithCUnsafe' /after/ running all__--- __interpreters for your higher-order effects.__-runShiftWithCUnsafe :: forall s a r.-                       (a -> Sem r (Maybe s))-                    -> Sem (Shift (Ref (Sem r) (Maybe s)) s ': r) a-                    -> Sem r (Maybe s)-runShiftWithCUnsafe c (Sem sem) = (`runContT` c) $ sem $ \u -> case decomp u of-  Right weaving -> runShiftWeaving runShiftWithCUnsafe weaving-  Left g -> ContT $ \c' -> embedSem g >>= runShiftWithCUnsafe c'-{-# INLINE runShiftWithCUnsafe #-}---------------------------------------------------------------------------------- | Runs a 'Cont' and a 'Shift' effect simultaneously by providing--- @'pure' '.' 'Just'@ as the final continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ __Beware__: This interpreter will invalidate all higher-order effects of any--- interpreter run after it; i.e. 'Polysemy.Reader.local' and--- 'Polysemy.Writer.censor' will be no-ops, 'Polysemy.Error.catch' will fail--- to catch exceptions, and 'Polysemy.Writer.listen' will always return 'mempty'.------ __You should therefore use 'runContShiftUnsafe' /after/ running all__--- __interpreters for your higher-order effects.__-runContShiftUnsafe :: Sem (   Cont (Ref (Sem r) (Maybe a))-                           ': Shift (Ref (Sem r) (Maybe a)) a-                           ': r) a-                   -> Sem r (Maybe a)-runContShiftUnsafe = runContShiftWithCUnsafe (pure . Just)-{-# INLINE runContShiftUnsafe #-}---------------------------------------------------------------------------------- | Runs a 'Cont' and a 'Shift' effect simultaneously by explicitly providing--- a final continuation.------ The final return type is wrapped in a 'Maybe' due to the fact that--- any continuation may fail locally.------ __Beware__: This interpreter will invalidate all higher-order effects of any--- interpreter run after it; i.e. 'Polysemy.Reader.local' and--- 'Polysemy.Writer.censor' will be no-ops, 'Polysemy.Error.catch' will fail--- to catch exceptions, and 'Polysemy.Writer.listen' will always return 'mempty'.------ __You should therefore use 'runContShiftWithCUnsafe' /after/ running all__--- __interpreters for your higher-order effects.__-runContShiftWithCUnsafe :: forall s a r.-                           (a -> Sem r (Maybe s))-                        -> Sem (   Cont (Ref (Sem r) (Maybe s))-                                ': Shift (Ref (Sem r) (Maybe s)) s-                                ': r) a-                        -> Sem r (Maybe s)-runContShiftWithCUnsafe c (Sem m) = (`runContT` c) $ m $ \u -> case decomp u of-  Right weaving -> runContWeaving runContShiftWithCUnsafe weaving-  Left g -> case decomp g of-    Right weaving -> runShiftWeaving runContShiftWithCUnsafe weaving-    Left g'       -> ContT $ \c' -> embedSem g' >>= runContShiftWithCUnsafe c'-{-# INLINE runContShiftWithCUnsafe #-}
− vendor/polysemy-zoo-0.8.2.0/src/Polysemy/Shift/Internal.hs
@@ -1,97 +0,0 @@-{-# LANGUAGE AllowAmbiguousTypes, TemplateHaskell #-}-module Polysemy.Shift.Internal where--import Polysemy-import Polysemy.Internal.Union-import Polysemy.Cont.Internal (Ref(..))-import Control.Monad.Cont (ContT(..))---------------------------------------------------------------------------------- | An effect for delimited continuations, formulated algebraically--- through a variant of the 'Polysemy.Cont.Jump/'Polysemy.Cont.Subst'--- formulation of abortive continuations.------ Activating polysemy-plugin is highly recommended when using this effect--- in order to avoid ambiguous types.-data Shift ref s m a where-  Trap    :: (ref a -> m s) -> Shift ref s m a-  Invoke  :: ref a -> a -> Shift ref s m s-  Abort   :: s   -> Shift ref s m a-  Reset   :: m s -> Shift ref s m s-  Reset'  :: m s -> Shift ref s m (Maybe s)--makeSem_ ''Shift---------------------------------------------------------------------------------- | Reifies the current continuation in the form of a prompt, and passes it to--- the first argument. Unlike 'subst', control will never return to the current--- continuation unless the prompt is invoked via 'release'.-trap :: forall ref s a r-     .  Member (Shift ref s) r-     => (ref a -> Sem r s)-     -> Sem r a---------------------------------------------------------------------------------- | Provide an answer to a prompt, jumping to its reified continuation.--- Unlike 'jump', this will not abort the current continuation, and the--- reified computation will instead return its final result when finished.------ Any effectful state of effects which have been run before the interpreter for--- 'Shift' will be embedded in the return value, and therefore the invocation--- won't have any apparent effects unless these are interpreted in the final--- monad.------ Any higher-order actions will also not interact with the continuation in any--- meaningful way; i.e. 'Polysemy.Reader.local' or 'Polysemy.Writer.censor' does--- not affect it, 'Polysemy.Error.catch' will fail to catch any of its exceptions,--- and 'Polysemy.Writer.listen' will always return 'mempty'.------ The provided continuation may fail locally in its subcontinuations.--- It may sometimes become necessary to handle such cases. To do so,--- use 'reset\'' together with 'release'.-invoke :: forall ref s a r-       .  Member (Shift ref s) r-       => ref a-       -> a-       -> Sem r s----------------------------------------------------------------------------------- | Aborts the current continuation with a result.-abort :: forall ref s a r-      .  Member (Shift ref s) r-      => s-      -> Sem r a---------------------------------------------------------------------------------- | Delimits any continuations and calls to 'abort'.-reset :: forall ref s r-      .  Member (Shift ref s) r-      => Sem r s-      -> Sem r s---------------------------------------------------------------------------------- | Delimits any continuations and calls to 'abort', and detects if--- any subcontinuation has failed locally.-reset' :: forall ref s r-       .  Member (Shift ref s) r-       => Sem r s-       -> Sem r (Maybe s)--runShiftWeaving :: Monad m-                => (forall x. (x -> m (Maybe s)) -> Sem r x -> m (Maybe s))-                -> Weaving (Shift (Ref m (Maybe s)) s) (Sem r) a-                -> ContT (Maybe s) m a-runShiftWeaving runW (Weaving e s wv ex ins) =-  fmap (ex . (<$ s)) $ ContT $ \c ->-    case e of-      Trap main ->-        runW (pure . ins) $ wv (main (Ref c) <$ s)-      Invoke ref a ->-        runRef ref a >>= maybe (pure Nothing) c-      Abort t -> pure (Just t)-      Reset main ->-        runW (pure . ins) (wv (main <$ s)) >>= maybe (pure Nothing) c-      Reset' main ->-        runW (pure . ins) (wv (main <$ s)) >>= c-{-# INLINE runShiftWeaving #-}
− vendor/warp-tls-3.4.13/Network/Wai/Handler/WarpTLS.hs
@@ -1,562 +0,0 @@-{-# LANGUAGE CPP #-}-{-# LANGUAGE DeriveDataTypeable #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE PatternGuards #-}-{-# LANGUAGE RankNTypes #-}-{-# LANGUAGE RecordWildCards #-}---- | HTTP over TLS support for Warp via the TLS package.------   If HTTP\/2 is negotiated by ALPN, HTTP\/2 over TLS is used.---   Otherwise HTTP\/1.1 over TLS is used.------   Support for SSL is now obsoleted.-module Network.Wai.Handler.WarpTLS (-    -- * Runner-    runTLS,-    runTLSSocket,--    -- * Settings-    TLSSettings,-    defaultTlsSettings,--    -- * Smart constructors--    -- ** From files-    tlsSettings,-    tlsSettingsChain,--    -- ** From memory-    tlsSettingsMemory,-    tlsSettingsChainMemory,--    -- ** From references-    tlsSettingsRef,-    tlsSettingsChainRef,-    CertSettings,--    -- ** Dynamically retrieved-    tlsSettingsSni,--    -- * Accessors-    tlsCredentials,-    tlsLogging,-    tlsAllowedVersions,-    tlsCiphers,-    tlsWantClientCert,-    tlsServerHooks,-    tlsServerDHEParams,-    tlsSessionManagerConfig,-    tlsSessionManager,-    onInsecure,-    OnInsecure (..),--    -- * Exception-    WarpTLSException (..),--    -- * Low-level-    attachConn-) where--import Control.Applicative ((<|>))-import Control.Exception (-    Exception,-    IOException,-    SomeException (..),-    bracket,-    finally,-    fromException,-    handle,-    handleJust,-    onException,-    throwIO,-    try,- )-import Control.Monad (guard, void)-import qualified Data.ByteString as S-import qualified Data.ByteString.Lazy as L-import qualified Data.IORef as I-import Data.Streaming.Network (bindPortTCP, safeRecv)-import Data.Typeable (Typeable)-import GHC.IO.Exception (IOErrorType (..))-import Network.Socket (-    SockAddr,-    Socket,-    close,-    getSocketName,-#if MIN_VERSION_network(3,1,1)-    gracefulClose,-#endif-    withSocketsDo,- )-import qualified Control.Exception as E-import Network.Socket.BufferPool-import Network.Socket.ByteString (sendAll)-import qualified Network.TLS as TLS-import qualified Network.TLS.SessionManager as SM-import Network.Wai (Application)-import Network.Wai.Handler.Warp-import Network.Wai.Handler.Warp.Internal-import Network.Wai.Handler.WarpTLS.Internal-import System.IO.Error (ioeGetErrorType, isEOFError)-import System.Timeout (timeout)---------------------------------------------------------------------- | A smart constructor for 'TLSSettings' based on 'defaultTlsSettings'.-tlsSettings-    :: FilePath-    -- ^ Certificate file-    -> FilePath-    -- ^ Key file-    -> TLSSettings-tlsSettings cert key =-    defaultTlsSettings-        { certSettings = CertFromFile cert [] key-        }---- | A smart constructor for 'TLSSettings' that allows specifying--- chain certificates based on 'defaultTlsSettings'.------ Since 3.0.3-tlsSettingsChain-    :: FilePath-    -- ^ Certificate file-    -> [FilePath]-    -- ^ Chain certificate files-    -> FilePath-    -- ^ Key file-    -> TLSSettings-tlsSettingsChain cert chainCerts key =-    defaultTlsSettings-        { certSettings = CertFromFile cert chainCerts key-        }---- | A smart constructor for 'TLSSettings', but uses in-memory representations--- of the certificate and key based on 'defaultTlsSettings'.------ Since 3.0.1-tlsSettingsMemory-    :: S.ByteString-    -- ^ Certificate bytes-    -> S.ByteString-    -- ^ Key bytes-    -> TLSSettings-tlsSettingsMemory cert key =-    defaultTlsSettings-        { certSettings = CertFromMemory cert [] key-        }---- | A smart constructor for 'TLSSettings', but uses in-memory representations--- of the certificate and key based on 'defaultTlsSettings'.------ Since 3.0.3-tlsSettingsChainMemory-    :: S.ByteString-    -- ^ Certificate bytes-    -> [S.ByteString]-    -- ^ Chain certificate bytes-    -> S.ByteString-    -- ^ Key bytes-    -> TLSSettings-tlsSettingsChainMemory cert chainCerts key =-    defaultTlsSettings-        { certSettings = CertFromMemory cert chainCerts key-        }---- | Smart constructor for TLS settings that obtains its credentials during--- Server Name Indication. Can be used to return different credentials--- depending on the hostname but also to retrieve dynamically updated--- credentials from an IORef. Credentials can be loaded from PEM-encoded chain--- and key files using 'TLS.credentialLoadX509'.------ @since 3.4.13-tlsSettingsSni :: (Maybe TLS.HostName -> IO TLS.Credentials) -> TLSSettings-tlsSettingsSni onServerNameIndicationHook =-  defaultTlsSettings-    { tlsCredentials = Just (TLS.Credentials [])-    , tlsServerHooks = (tlsServerHooks defaultTlsSettings)-      { TLS.onServerNameIndication =  onServerNameIndicationHook-      }-    }---- | A smart constructor for 'TLSSettings', but uses references to in-memory--- representations of the certificate and key based on 'defaultTlsSettings'.------ @since 3.3.0-tlsSettingsRef-    :: I.IORef S.ByteString-    -- ^ Reference to certificate bytes-    -> I.IORef S.ByteString-    -- ^ Reference to key bytes-    -> TLSSettings-tlsSettingsRef cert key =-    defaultTlsSettings-        { certSettings = CertFromRef cert [] key-        }--{-# DEPRECATED tlsSettingsRef "This function was added to allow Warp to serve new certificates without restarting, but it has always behaved the same as 'tlsSettingsMemory'. It will be removed in the next major release. To retain existing behavior, swich to 'tlsSettingsMemory'. To dynamically update credentials, see 'tlsSettingsSni'." #-}---- | A smart constructor for 'TLSSettings', but uses references to in-memory--- representations of the certificate and key based on 'defaultTlsSettings'.------ @since 3.3.0-tlsSettingsChainRef-    :: I.IORef S.ByteString-    -- ^ Reference to certificate bytes-    -> [I.IORef S.ByteString]-    -- ^ Reference to chain certificate bytes-    -> I.IORef S.ByteString-    -- ^ Reference to key bytes-    -> TLSSettings-tlsSettingsChainRef cert chainCerts key =-    defaultTlsSettings-        { certSettings = CertFromRef cert chainCerts key-        }--{-# DEPRECATED tlsSettingsChainRef "This function was added to allow Warp to serve new certificates without restarting, but it has always behaved the same as 'tlsSettingsChainMemory'. It will be removed in the next major release. To retain existing behavior, swich to 'tlsSettingsChainMemory'. To dynamically update credentials, see 'tlsSettingsSni'." #-}---------------------------------------------------------------------- | Running 'Application' with 'TLSSettings' and 'Settings'.-runTLS :: TLSSettings -> Settings -> Application -> IO ()-runTLS tset set app =-    withSocketsDo $-        bracket-            (bindPortTCP (getPort set) (getHost set))-            close-            ( \sock -> do-                setSocketCloseOnExec sock-                runTLSSocket tset set sock app-            )--------------------------------------------------------------------loadCredentials :: TLSSettings -> IO TLS.Credentials-loadCredentials TLSSettings{tlsCredentials = Just creds} = return creds-loadCredentials TLSSettings{..} = case certSettings of-    CertFromFile cert chainFiles key -> do-        cred <- either error id <$> TLS.credentialLoadX509Chain cert chainFiles key-        return $ TLS.Credentials [cred]-    CertFromRef certRef chainCertsRef keyRef -> do-        cert <- I.readIORef certRef-        chainCerts <- mapM I.readIORef chainCertsRef-        key <- I.readIORef keyRef-        cred <--            either error return $ TLS.credentialLoadX509ChainFromMemory cert chainCerts key-        return $ TLS.Credentials [cred]-    CertFromMemory certMemory chainCertsMemory keyMemory -> do-        cred <--            either error return $-                TLS.credentialLoadX509ChainFromMemory certMemory chainCertsMemory keyMemory-        return $ TLS.Credentials [cred]--getSessionManager :: TLSSettings -> IO TLS.SessionManager-getSessionManager TLSSettings{tlsSessionManager = Just mgr} = return mgr-getSessionManager TLSSettings{..} = case tlsSessionManagerConfig of-    Nothing -> return TLS.noSessionManager-    Just config -> SM.newSessionManager config---- | Running 'Application' with 'TLSSettings' and 'Settings' using---   specified 'Socket'.-runTLSSocket :: TLSSettings -> Settings -> Socket -> Application -> IO ()-runTLSSocket tlsset set sock app = do-    settingsInstallShutdownHandler set (close sock)-    credentials <- loadCredentials tlsset-    mgr <- getSessionManager tlsset-    runTLSSocket' tlsset set credentials mgr sock app--runTLSSocket'-    :: TLSSettings-    -> Settings-    -> TLS.Credentials-    -> TLS.SessionManager-    -> Socket-    -> Application-    -> IO ()-runTLSSocket' tlsset@TLSSettings{..} set credentials mgr sock =-    runSettingsConnectionMakerSecure set get-  where-    get = getter tlsset set sock params-    params =-        TLS.defaultParamsServer-            { TLS.serverWantClientCert = tlsWantClientCert-            , TLS.serverCACertificates = []-            , TLS.serverDHEParams = tlsServerDHEParams-            , TLS.serverHooks = hooks-            , TLS.serverShared = shared-            , TLS.serverSupported = supported-#if MIN_VERSION_tls(1,5,0)-            , TLS.serverEarlyDataSize = 2018-#endif-            }-    -- Adding alpn to user's tlsServerHooks.-    hooks =-        tlsServerHooks-            { TLS.onALPNClientSuggest =-                TLS.onALPNClientSuggest tlsServerHooks-                    <|> (if settingsHTTP2Enabled set then Just alpn else Nothing)-            }-    shared =-        TLS.defaultShared-            { TLS.sharedCredentials = credentials-            , TLS.sharedSessionManager = mgr-            }-    supported =-        TLS.defaultSupported-            { TLS.supportedVersions = tlsAllowedVersions-            , TLS.supportedCiphers = tlsCiphers-            , TLS.supportedCompressions = [TLS.nullCompression]-            , TLS.supportedSecureRenegotiation = True-            , TLS.supportedClientInitiatedRenegotiation = False-            , TLS.supportedSession = True-            , TLS.supportedFallbackScsv = True-            , TLS.supportedHashSignatures = tlsSupportedHashSignatures-            , TLS.supportedExtendedMainSecret = TLS.AllowEMS-#if MIN_VERSION_tls(1,5,0)-            , TLS.supportedGroups = [TLS.X25519,TLS.P256,TLS.P384]-#endif-            }--alpn :: [S.ByteString] -> IO S.ByteString-alpn xs-    | "h2" `elem` xs = return "h2"-    | otherwise = return "http/1.1"--------------------------------------------------------------------getter-    :: TLS.TLSParams params-    => TLSSettings-    -> Settings-    -> Socket-    -> params-    -> IO (IO (Connection, Transport), SockAddr)-getter tlsset set@Settings{settingsAccept = accept'} sock params = do-    (s, sa) <- accept' sock-    setSocketCloseOnExec s-    return (mkConn tlsset set s params, sa)--mkConn-    :: TLS.TLSParams params-    => TLSSettings-    -> Settings-    -> Socket-    -> params-    -> IO (Connection, Transport)-mkConn tlsset set s params = do-    let tm = settingsTimeout set * 1000000-    mbs <- timeout tm recvFirstBS-    case mbs of-      Nothing -> throwIO IncompleteHeaders-      Just bs -> switch bs-  where-    recvFirstBS = safeRecv s 4096 `onException` close s-    switch firstBS-        | S.null firstBS = close s >> throwIO ClientClosedConnectionPrematurely-        | S.head firstBS == 0x16 = httpOverTls tlsset set s firstBS params-        | otherwise = plainHTTP tlsset set s firstBS--------------------------------------------------------------------isAsyncException :: Exception e => e -> Bool-isAsyncException e =-    case E.fromException (E.toException e) of-        Just (E.SomeAsyncException _) -> True-        Nothing -> False--throughAsync :: IO a -> SomeException -> IO a-throughAsync action (SomeException e)-  | isAsyncException e = E.throwIO e-  | otherwise          = action--httpOverTls-    :: TLS.TLSParams params-    => TLSSettings-    -> Settings-    -> Socket-    -> S.ByteString-    -> params-    -> IO (Connection, Transport)-httpOverTls TLSSettings{..} set s bs0 params =-    makeConn `onException` close s-  where-    makeConn = do-        pool <- newBufferPool 2048 16384-        rawRecvN <- makeRecvN bs0 $ receive s pool-        let recvN = wrappedRecvN rawRecvN-        ctx <- TLS.contextNew (backend recvN) params-        TLS.contextHookSetLogging ctx tlsLogging-        let tm = settingsTimeout set * 1000000-        mconn <- timeout tm $ do-            TLS.handshake ctx-            mysa <- getSocketName s-            attachConn mysa ctx-        case mconn of-          Nothing -> throwIO IncompleteHeaders-          Just conn -> return conn-    wrappedRecvN recvN n = handle (throughAsync (return "")) $ recvN n-    backend recvN =-        TLS.Backend-            { TLS.backendFlush = return ()-#if MIN_VERSION_network(3,1,1)-            , TLS.backendClose =-                gracefulClose s 5000 `E.catch` throughAsync (return ())-#else-            , TLS.backendClose = close s-#endif-            , TLS.backendSend = sendAll' s-            , TLS.backendRecv = recvN-            }-    sendAll' sock bs =-        E.handleJust-            ( \e ->-                if ioeGetErrorType e == ResourceVanished-                    then Just ConnectionClosedByPeer-                    else Nothing-            )-            throwIO-            $ sendAll sock bs---- | Get "Connection" and "Transport" for a TLS connection that is already did the handshake.--- @since 3.4.7-attachConn :: SockAddr -> TLS.Context -> IO (Connection, Transport)-attachConn mysa ctx = do-    h2 <- (== Just "h2") <$> TLS.getNegotiatedProtocol ctx-    isH2 <- I.newIORef h2-    writeBuffer <- createWriteBuffer 16384-    writeBufferRef <- I.newIORef writeBuffer-    -- Creating a cache for leftover input data.-    tls <- getTLSinfo ctx-    return (conn writeBufferRef isH2, tls)-  where-    conn writeBufferRef isH2 =-        Connection-            { connSendMany = TLS.sendData ctx . L.fromChunks-            , connSendAll = sendall-            , connSendFile = sendfile-            , connClose = close'-            , connRecv = recv-            , connRecvBuf = \_ _ -> return True -- obsoleted-            , connWriteBuffer = writeBufferRef-            , connHTTP2 = isH2-            , connMySockAddr = mysa-            }-      where-        sendall = TLS.sendData ctx . L.fromChunks . return-        recv = handle onEOF $ TLS.recvData ctx-          where-            onEOF e-#if MIN_VERSION_tls(1,8,0)-                | Just (TLS.PostHandshake TLS.Error_EOF) <- E.fromException e = return S.empty-#else-                | Just TLS.Error_EOF <- fromException e = return S.empty-#endif-                | Just ioe <- fromException e, isEOFError ioe = return S.empty-                | otherwise = throwIO e-        sendfile fid offset len hook headers = do-            writeBuffer <- I.readIORef writeBufferRef-            readSendFile-                (bufBuffer writeBuffer)-                (bufSize writeBuffer)-                sendall-                fid-                offset-                len-                hook-                headers--        close' =-            void (tryIO sendBye)-                `finally` TLS.contextClose ctx--        sendBye =-            -- It's fine if the connection was closed by the other side before-            -- receiving close_notify, see RFC 5246 section 7.2.1.-            handleJust-                (\e -> guard (e == ConnectionClosedByPeer) >> return e)-                (const (return ()))-                (TLS.bye ctx)--getTLSinfo :: TLS.Context -> IO Transport-getTLSinfo ctx = do-    proto <- TLS.getNegotiatedProtocol ctx-    minfo <- TLS.contextGetInformation ctx-    case minfo of-        Nothing -> return TCP-        Just info -> do-            let (major, minor) = case TLS.infoVersion info of-                    TLS.SSL2 -> (2, 0)-                    TLS.SSL3 -> (3, 0)-                    TLS.TLS10 -> (3, 1)-                    TLS.TLS11 -> (3, 2)-                    TLS.TLS12 -> (3, 3)-                    _ -> (3,4)-            clientCert <- TLS.getClientCertificateChain ctx-            return-                TLS-                    { tlsMajorVersion = major-                    , tlsMinorVersion = minor-                    , tlsNegotiatedProtocol = proto-                    , tlsChiperID = TLS.cipherID $ TLS.infoCipher info-                    , tlsClientCertificate = clientCert-                    }--tryIO :: IO a -> IO (Either IOException a)-tryIO = try--------------------------------------------------------------------plainHTTP-    :: TLSSettings -> Settings -> Socket -> S.ByteString -> IO (Connection, Transport)-plainHTTP TLSSettings{..} set s bs0 = case onInsecure of-    AllowInsecure -> do-        conn' <- socketConnection set s-        cachedRef <- I.newIORef bs0-        let conn'' =-                conn'-                    { connRecv = recvPlain cachedRef (connRecv conn')-                    }-        return (conn'', TCP)-    DenyInsecure lbs -> do-        -- Listening port 443 but TLS records do not arrive.-        -- We want to let the browser know that TLS is required.-        -- So, we use 426.-        --     http://tools.ietf.org/html/rfc2817#section-4.2-        --     https://tools.ietf.org/html/rfc7231#section-6.5.15-        -- FIXME: should we distinguish HTTP/1.1 and HTTP/2?-        --        In the case of HTTP/2, should we send-        --        GOAWAY + INADEQUATE_SECURITY?-        -- FIXME: Content-Length:-        -- FIXME: TLS/<version>-        sendAll-            s-            "HTTP/1.1 426 Upgrade Required\-            \\r\nUpgrade: TLS/1.0, HTTP/1.1\-            \\r\nConnection: Upgrade\-            \\r\nContent-Type: text/plain\r\n\r\n"-        mapM_ (sendAll s) $ L.toChunks lbs-        close s-        throwIO InsecureConnectionDenied---------------------------------------------------------------------- | Modify the given receive function to first check the given @IORef@ for a--- chunk of data. If present, takes the chunk of data from the @IORef@ and--- empties out the @IORef@. Otherwise, calls the supplied receive function.-recvPlain :: I.IORef S.ByteString -> IO S.ByteString -> IO S.ByteString-recvPlain ref fallback = do-    bs <- I.readIORef ref-    if S.null bs-        then fallback-        else do-            I.writeIORef ref S.empty-            return bs--------------------------------------------------------------------data WarpTLSException-    = InsecureConnectionDenied-    | ClientClosedConnectionPrematurely-    deriving (Show, Typeable)-instance Exception WarpTLSException
− vendor/warp-tls-3.4.13/Network/Wai/Handler/WarpTLS/Internal.hs
@@ -1,161 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}--module Network.Wai.Handler.WarpTLS.Internal (-    CertSettings (..),-    TLSSettings (..),-    defaultTlsSettings,-    OnInsecure (..),--    -- * Accessors-    getCertSettings,-) where--import qualified Data.ByteString as S-import qualified Data.ByteString.Lazy as L-import qualified Data.IORef as I-import qualified Network.TLS as TLS-import qualified Network.TLS.Extra as TLSExtra-import qualified Network.TLS.SessionManager as SM---------------------------------------------------------------------- | Determines where to load the certificate, chain--- certificates, and key from.-data CertSettings-    = CertFromFile !FilePath ![FilePath] !FilePath-    | CertFromMemory !S.ByteString ![S.ByteString] !S.ByteString-    | CertFromRef-        !(I.IORef S.ByteString)-        ![I.IORef S.ByteString]-        !(I.IORef S.ByteString)--instance Show CertSettings where-    show (CertFromFile a b c) = "CertFromFile " ++ show a ++ " " ++ show b ++ " " ++ show c-    show (CertFromMemory a b c) = "CertFromMemory " ++ show a ++ " " ++ show b ++ " " ++ show c-    show CertFromRef{} = "CertFromRef"---------------------------------------------------------------------- | An action when a plain HTTP comes to HTTP over TLS/SSL port.-data OnInsecure-    = DenyInsecure L.ByteString-    | AllowInsecure-    deriving (Show)---------------------------------------------------------------------- | Settings for WarpTLS.-data TLSSettings = TLSSettings-    { certSettings :: CertSettings-    -- ^ Where are the certificate, chain certificates, and key-    -- loaded from?-    ---    -- >>> certSettings defaultTlsSettings-    -- CertFromFile "certificate.pem" [] "key.pem"-    ---    -- @since 3.3.0-    , onInsecure :: OnInsecure-    -- ^ Do we allow insecure connections with this server as well?-    ---    -- >>> onInsecure defaultTlsSettings-    -- DenyInsecure "This server only accepts secure HTTPS connections."-    ---    -- Since 1.4.0-    , tlsLogging :: TLS.Logging-    -- ^ The level of logging to turn on.-    ---    -- Default: 'TLS.defaultLogging'.-    ---    -- Since 1.4.0-    , tlsAllowedVersions :: [TLS.Version]-    -- ^ The TLS versions this server accepts.-    ---    -- Since 1.4.2-    , tlsCiphers-        :: [TLS.Cipher]-    -- ^ The TLS ciphers this server accepts.-    ---    -- Since 1.4.2-    , tlsWantClientCert :: Bool-    -- ^ Whether or not to demand a certificate from the client.  If this-    -- is set to True, you must handle received certificates in a server hook-    -- or all connections will fail.-    ---    -- >>> tlsWantClientCert defaultTlsSettings-    -- False-    ---    -- Since 3.0.2-    , tlsServerHooks :: TLS.ServerHooks-    -- ^ The server-side hooks called by the tls package, including actions-    -- to take when a client certificate is received.  See the "Network.TLS"-    -- module for details.-    ---    -- Default: defaultServerHooks-    ---    -- Since 3.0.2-    , tlsServerDHEParams :: Maybe TLS.DHParams-    -- ^ Configuration for ServerDHEParams-    -- more function lives in `crypton` package-    ---    -- Default: Nothing-    ---    -- Since 3.2.2-    , tlsSessionManagerConfig :: Maybe SM.Config-    -- ^ Configuration for in-memory TLS session manager.-    -- If Nothing, 'TLS.noSessionManager' is used.-    -- Otherwise, an in-memory TLS session manager is created-    -- according to 'Config'.-    ---    -- Default: Nothing-    ---    -- Since 3.2.4-    , tlsCredentials :: Maybe TLS.Credentials-    -- ^ Specifying 'TLS.Credentials' directly.  If this value is-    --   specified, other fields such as 'certFile' are ignored.-    ---    --   Since 3.2.12-    , tlsSessionManager :: Maybe TLS.SessionManager-    -- ^ Specifying 'TLS.SessionManager' directly. If this value is-    --   specified, 'tlsSessionManagerConfig' is ignored.-    ---    --   Since 3.2.12-    , tlsSupportedHashSignatures :: [TLS.HashAndSignatureAlgorithm]-    -- ^ Specifying supported hash/signature algorithms, ordered by decreasing-    -- priority. See the "Network.TLS" module for details-    ---    --   Since 3.3.3-    }---- Since 3.3.1---- | Some programs need access to cert settings-getCertSettings :: TLSSettings -> CertSettings-getCertSettings = certSettings---- | The default 'CertSettings'.-defaultCertSettings :: CertSettings-defaultCertSettings = CertFromFile "certificate.pem" [] "key.pem"---------------------------------------------------------------------- | Default 'TLSSettings'. Use this to create 'TLSSettings' with the field record name (aka accessors).-defaultTlsSettings :: TLSSettings-defaultTlsSettings =-    TLSSettings-        { certSettings = defaultCertSettings-        , onInsecure = DenyInsecure "This server only accepts secure HTTPS connections."-        , tlsLogging = TLS.defaultLogging-        , tlsAllowedVersions = TLS.supportedVersions TLS.defaultSupported-        , tlsCiphers = ciphers-        , tlsWantClientCert = False-        , tlsServerHooks = TLS.defaultServerHooks-        , tlsServerDHEParams = Nothing-        , tlsSessionManagerConfig = Nothing-        , tlsCredentials = Nothing-        , tlsSessionManager = Nothing-        , tlsSupportedHashSignatures = TLS.supportedHashSignatures TLS.defaultSupported-        }---- taken from stunnel example in tls-extra-ciphers :: [TLS.Cipher]-ciphers = TLSExtra.ciphersuite_strong
+ vendor/warp-tls-3.4.18/LICENSE view
@@ -0,0 +1,20 @@+Copyright (c) 2012 Michael Snoyman, http://www.yesodweb.com/++Permission is hereby granted, free of charge, to any person obtaining+a copy of this software and associated documentation files (the+"Software"), to deal in the Software without restriction, including+without limitation the rights to use, copy, modify, merge, publish,+distribute, sublicense, and/or sell copies of the Software, and to+permit persons to whom the Software is furnished to do so, subject to+the following conditions:++The above copyright notice and this permission notice shall be+included in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ vendor/warp-tls-3.4.18/Network/Wai/Handler/WarpTLS.hs view
@@ -0,0 +1,676 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE PatternGuards #-}+{-# LANGUAGE RankNTypes #-}+{-# LANGUAGE RecordWildCards #-}++-- | HTTP over TLS support for Warp via the TLS package.+--+--   If HTTP\/2 is negotiated by ALPN, HTTP\/2 over TLS is used.+--   Otherwise HTTP\/1.1 over TLS is used.+--+--   Support for SSL is now obsoleted.+module Network.Wai.Handler.WarpTLS (+    -- * Runner+    runTLS,+    runTLSSocket,++    -- * Settings+    TLSSettings,+    defaultTlsSettings,++    -- * Smart constructors++    -- ** From files+    tlsSettings,+    tlsSettingsChain,++    -- ** From memory+    tlsSettingsMemory,+    tlsSettingsChainMemory,++    -- ** From references+    tlsSettingsRef,+    tlsSettingsChainRef,+    CertSettings,++    -- ** Dynamically retrieved+    tlsSettingsSni,++    -- * Accessors+    tlsCredentials,+    tlsLogging,+    tlsAllowedVersions,+    tlsCiphers,+    tlsWantClientCert,+    tlsServerHooks,+    tlsServerDHEParams,+    tlsSessionManagerConfig,+    tlsSessionManager,+    tlsEarlyDataSize,+    onInsecure,+    OnInsecure (..),++    -- * Exception+    WarpTLSException (..),++    -- * Low-level+    attachConn+) where++import Control.Applicative ((<|>))+import Control.Concurrent.STM+import Control.Exception (+    Exception,+    IOException,+    SomeException (..),+    bracket,+    finally,+    fromException,+    handle,+    handleJust,+    onException,+    throwIO,+    try,+ )+import qualified Control.Exception as E+import Control.Monad (guard, void)+import qualified Data.ByteString as S+import qualified Data.ByteString.Lazy as L+import qualified Data.IORef as I+import Data.Streaming.Network (bindPortTCP)+import GHC.IO.Exception (IOErrorType (..))+import Network.Socket (+    SockAddr,+    Socket,+    getSocketName,+#if !MIN_VERSION_network(3,1,1) || !MIN_VERSION_warp(3,4,18)+    close,+#endif+#if MIN_VERSION_network(3,1,1)+    gracefulClose,+#endif+    unsafeClose,+    withSocketsDo,+ )+#if MIN_VERSION_warp(3,4,21)+import Network.Socket (unsafeGracefulClose)+#endif+import Network.Socket.BufferPool+import Network.Socket.ByteString (recvWith, sendAll, sendAllWith)+import qualified Network.TLS as TLS+import qualified Network.TLS.SessionManager as SM+import Network.Wai (Application)+import Network.Wai.Handler.Warp+import Network.Wai.Handler.Warp.Internal+import Network.Wai.Handler.WarpTLS.Internal+import System.IO.Error (ioeGetErrorType, isEOFError)+import System.Timeout (timeout)++----------------------------------------------------------------++-- | A smart constructor for 'TLSSettings' based on 'defaultTlsSettings'.+tlsSettings+    :: FilePath+    -- ^ Certificate file+    -> FilePath+    -- ^ Key file+    -> TLSSettings+tlsSettings cert key =+    defaultTlsSettings+        { certSettings = CertFromFile cert [] key+        }++-- | A smart constructor for 'TLSSettings' that allows specifying+-- chain certificates based on 'defaultTlsSettings'.+--+-- Since 3.0.3+tlsSettingsChain+    :: FilePath+    -- ^ Certificate file+    -> [FilePath]+    -- ^ Chain certificate files+    -> FilePath+    -- ^ Key file+    -> TLSSettings+tlsSettingsChain cert chainCerts key =+    defaultTlsSettings+        { certSettings = CertFromFile cert chainCerts key+        }++-- | A smart constructor for 'TLSSettings', but uses in-memory representations+-- of the certificate and key based on 'defaultTlsSettings'.+--+-- Since 3.0.1+tlsSettingsMemory+    :: S.ByteString+    -- ^ Certificate bytes+    -> S.ByteString+    -- ^ Key bytes+    -> TLSSettings+tlsSettingsMemory cert key =+    defaultTlsSettings+        { certSettings = CertFromMemory cert [] key+        }++-- | A smart constructor for 'TLSSettings', but uses in-memory representations+-- of the certificate and key based on 'defaultTlsSettings'.+--+-- Since 3.0.3+tlsSettingsChainMemory+    :: S.ByteString+    -- ^ Certificate bytes+    -> [S.ByteString]+    -- ^ Chain certificate bytes+    -> S.ByteString+    -- ^ Key bytes+    -> TLSSettings+tlsSettingsChainMemory cert chainCerts key =+    defaultTlsSettings+        { certSettings = CertFromMemory cert chainCerts key+        }++-- | Smart constructor for TLS settings that obtains its credentials during+-- Server Name Indication. Can be used to return different credentials+-- depending on the hostname but also to retrieve dynamically updated+-- credentials from an IORef. Credentials can be loaded from PEM-encoded chain+-- and key files using 'TLS.credentialLoadX509'.+--+-- @since 3.4.13+tlsSettingsSni :: (Maybe TLS.HostName -> IO TLS.Credentials) -> TLSSettings+tlsSettingsSni onServerNameIndicationHook =+  defaultTlsSettings+    { tlsCredentials = Just (TLS.Credentials [])+    , tlsServerHooks = (tlsServerHooks defaultTlsSettings)+      { TLS.onServerNameIndication =  onServerNameIndicationHook+      }+    }++-- | A smart constructor for 'TLSSettings', but uses references to in-memory+-- representations of the certificate and key based on 'defaultTlsSettings'.+--+-- @since 3.3.0+tlsSettingsRef+    :: I.IORef S.ByteString+    -- ^ Reference to certificate bytes+    -> I.IORef S.ByteString+    -- ^ Reference to key bytes+    -> TLSSettings+tlsSettingsRef cert key =+    defaultTlsSettings+        { certSettings = CertFromRef cert [] key+        }++{-# DEPRECATED tlsSettingsRef "This function was added to allow Warp to serve new certificates without restarting, but it has always behaved the same as 'tlsSettingsMemory'. It will be removed in the next major release. To retain existing behavior, swich to 'tlsSettingsMemory'. To dynamically update credentials, see 'tlsSettingsSni'." #-}++-- | A smart constructor for 'TLSSettings', but uses references to in-memory+-- representations of the certificate and key based on 'defaultTlsSettings'.+--+-- @since 3.3.0+tlsSettingsChainRef+    :: I.IORef S.ByteString+    -- ^ Reference to certificate bytes+    -> [I.IORef S.ByteString]+    -- ^ Reference to chain certificate bytes+    -> I.IORef S.ByteString+    -- ^ Reference to key bytes+    -> TLSSettings+tlsSettingsChainRef cert chainCerts key =+    defaultTlsSettings+        { certSettings = CertFromRef cert chainCerts key+        }++{-# DEPRECATED tlsSettingsChainRef "This function was added to allow Warp to serve new certificates without restarting, but it has always behaved the same as 'tlsSettingsChainMemory'. It will be removed in the next major release. To retain existing behavior, swich to 'tlsSettingsChainMemory'. To dynamically update credentials, see 'tlsSettingsSni'." #-}++----------------------------------------------------------------++-- | Running 'Application' with 'TLSSettings' and 'Settings'.+runTLS :: TLSSettings -> Settings -> Application -> IO ()+runTLS tset set app =+    withSocketsDo $+        bracket+            (bindPortTCP (getPort set) (getHost set))+            unsafeClose+            ( \sock -> do+                setSocketCloseOnExec sock+                runTLSSocket tset set sock app+            )++----------------------------------------------------------------++loadCredentials :: TLSSettings -> IO TLS.Credentials+loadCredentials TLSSettings{tlsCredentials = Just creds} = return creds+loadCredentials TLSSettings{..} = case certSettings of+    CertFromFile cert chainFiles key -> do+        cred <- either error id <$> TLS.credentialLoadX509Chain cert chainFiles key+        return $ TLS.Credentials [cred]+    CertFromRef certRef chainCertsRef keyRef -> do+        cert <- I.readIORef certRef+        chainCerts <- mapM I.readIORef chainCertsRef+        key <- I.readIORef keyRef+        cred <-+            either error return $ TLS.credentialLoadX509ChainFromMemory cert chainCerts key+        return $ TLS.Credentials [cred]+    CertFromMemory certMemory chainCertsMemory keyMemory -> do+        cred <-+            either error return $+                TLS.credentialLoadX509ChainFromMemory certMemory chainCertsMemory keyMemory+        return $ TLS.Credentials [cred]++getSessionManager :: TLSSettings -> IO TLS.SessionManager+getSessionManager TLSSettings{tlsSessionManager = Just mgr} = return mgr+getSessionManager TLSSettings{..} = case tlsSessionManagerConfig of+    Nothing -> return TLS.noSessionManager+    Just config -> SM.newSessionManager config++-- | Running 'Application' with 'TLSSettings' and 'Settings' using+--   specified 'Socket'.+runTLSSocket :: TLSSettings -> Settings -> Socket -> Application -> IO ()+runTLSSocket tlsset set sock app = do+#if MIN_VERSION_warp(3,4,18)+    -- The accept loop is stopped by being told, not by having this socket+    -- closed under it; the socket is closed once the loop has ended.  See+    -- 'Network.Wai.Handler.Warp.Internal.Listener'.+    listener <- makeListener set sock+#else+    settingsInstallShutdownHandler set (close sock)+#endif+    credentials <- loadCredentials tlsset+    mgr <- getSessionManager tlsset+#if MIN_VERSION_warp(3,4,18)+    runTLSSocket' tlsset set listener credentials mgr sock app+#else+    runTLSSocket' tlsset set credentials mgr sock app+#endif++runTLSSocket'+    :: TLSSettings+    -> Settings+#if MIN_VERSION_warp(3,4,18)+    -> Listener+#endif+    -> TLS.Credentials+    -> TLS.SessionManager+    -> Socket+    -> Application+    -> IO ()+#if MIN_VERSION_warp(3,4,18)+runTLSSocket' tlsset@TLSSettings{..} set listener credentials mgr sock app = do+    (_, newSettings) <- makeServerState set+    let get = getter tlsset newSettings listener sock params+    runSettingsConnectionMakerSecureWith newSettings listener get app+#elif MIN_VERSION_warp(3,4,13)+runTLSSocket' tlsset@TLSSettings{..} set credentials mgr sock app = do+    (_, newSettings) <- makeServerState set+    let get = getter tlsset newSettings sock params+    runSettingsConnectionMakerSecure newSettings get app+#else+runTLSSocket' tlsset@TLSSettings{..} set credentials mgr sock app = do+    let get = getter tlsset set sock params+    runSettingsConnectionMakerSecure set get app+#endif+  where+    params =+        TLS.defaultParamsServer+            { TLS.serverWantClientCert = tlsWantClientCert+            , TLS.serverCACertificates = []+            , TLS.serverDHEParams = tlsServerDHEParams+            , TLS.serverHooks = hooks+            , TLS.serverShared = shared+            , TLS.serverSupported = supported+#if MIN_VERSION_tls(1,5,0)+            , TLS.serverEarlyDataSize = tlsEarlyDataSize+#endif+            }+    -- Adding alpn to user's tlsServerHooks.+    hooks =+        tlsServerHooks+            { TLS.onALPNClientSuggest =+                TLS.onALPNClientSuggest tlsServerHooks+                    <|> (if settingsHTTP2Enabled set then Just alpn else Nothing)+            }+    shared =+        TLS.defaultShared+            { TLS.sharedCredentials = credentials+            , TLS.sharedSessionManager = mgr+            }+    supported =+        TLS.defaultSupported+            { TLS.supportedVersions = tlsAllowedVersions+            , TLS.supportedCiphers = tlsCiphers+            , TLS.supportedCompressions = [TLS.nullCompression]+            , TLS.supportedSecureRenegotiation = True+            , TLS.supportedClientInitiatedRenegotiation = False+            , TLS.supportedSession = True+            , TLS.supportedFallbackScsv = True+            , TLS.supportedHashSignatures = tlsSupportedHashSignatures+            -- hs-hath: the one line that differs from upstream (see package.yaml)+            , TLS.supportedExtendedMainSecret = TLS.AllowEMS+#if MIN_VERSION_tls(1,5,0)+            , TLS.supportedGroups = [TLS.X25519,TLS.P256,TLS.P384]+#endif+            }++alpn :: [S.ByteString] -> IO S.ByteString+alpn xs+    | "h2" `elem` xs = return "h2"+    | otherwise = return "http/1.1"++----------------------------------------------------------------++getter+    :: TLS.TLSParams params+    => TLSSettings+    -> Settings+#if MIN_VERSION_warp(3,4,18)+    -> Listener+#endif+    -> Socket+    -> params+    -> IO (IO (Connection, Transport), SockAddr)+#if MIN_VERSION_warp(3,4,18)+getter tlsset set listener _sock params = do+    macc <- acceptOrStop listener+    (s, sa) <- maybe (throwIO AcceptStopped) return macc+    setSocketCloseOnExec s+    return (mkConn tlsset set s params, sa)+#else+getter tlsset set@Settings{settingsAccept = accept'} sock params = do+    (s, sa) <- accept' sock+    setSocketCloseOnExec s+    return (mkConn tlsset set s params, sa)+#endif++mkConn+    :: TLS.TLSParams params+    => TLSSettings+    -> Settings+    -> Socket+    -> params+    -> IO (Connection, Transport)+mkConn tlsset set s params = do+    let tm = settingsTimeout set * 1000000+    -- The first read is given up on when the deadline passes, rather than+    -- interrupted by 'timeout': an asynchronous exception does not reach a+    -- thread blocked in a socket call on Windows.+    expired <- registerDelay tm+    ebs <- recvFirstBS expired+    case ebs of+      Left () -> throwIO IncompleteHeaders+      Right bs -> switch bs+  where+    recvFirstBS expired =+        (recvWith s 4096 (check =<< readTVar expired) `E.catch` resetIsEOF)+            `onException` unsafeClose s+    -- What streaming-commons' safeRecv did for us: a connection reset is+    -- the end of the connection, not an error to report.+    resetIsEOF e+        | ioeGetErrorType e == ResourceVanished = return $ Right ""+        | otherwise = E.throwIO (e :: E.IOException)+    switch firstBS+        | S.null firstBS = unsafeClose s >> throwIO ClientClosedConnectionPrematurely+        | S.head firstBS == 0x16 = httpOverTls tlsset set s firstBS params+        | otherwise = plainHTTP tlsset set s firstBS++----------------------------------------------------------------++isAsyncException :: Exception e => e -> Bool+isAsyncException e =+    case E.fromException (E.toException e) of+        Just (E.SomeAsyncException _) -> True+        Nothing -> False++throughAsync :: IO a -> SomeException -> IO a+throughAsync action (SomeException e)+  | isAsyncException e = E.throwIO e+  | otherwise          = action++httpOverTls+    :: TLS.TLSParams params+    => TLSSettings+    -> Settings+    -> Socket+    -> S.ByteString+    -> params+    -> IO (Connection, Transport)+httpOverTls TLSSettings{..} set s bs0 params =+    makeConn `onException` unsafeClose s+  where+    makeConn = do+        pool <- newBufferPool 2048 16384+#if MIN_VERSION_warp(3,4,13)+        appsInProgress <- newTVarIO 0+        (ss, _) <- makeServerState set+        let recv = makeGracefulRecv s pool ss appsInProgress+#else+        let recv = receive s pool+#endif+        rawRecvN <- makeRecvN bs0 recv+        let recvN = wrappedRecvN rawRecvN+        ctx <- TLS.contextNew (backend recvN) params+        TLS.contextHookSetLogging ctx tlsLogging+        let tm = settingsTimeout set * 1000000+        mconn <- timeout tm $ do+            TLS.handshake ctx+            mysa <- getSocketName s+#if MIN_VERSION_warp(3,4,13)+            attachConnWith byeTimeout mysa ctx appsInProgress+#else+            attachConnWith byeTimeout mysa ctx+#endif+        case mconn of+          Nothing -> throwIO IncompleteHeaders+          Just conn -> return conn+    wrappedRecvN recvN n = handle (throughAsync (return "")) $ recvN n+    -- A peer which does not read is why a timed-out connection is+    -- closed, so close_notify is given a quarter of the timeout, at most+    -- a second, rather than waiting for that peer for ever.+    byeTimeout+        | settingsTimeout set <= 0 = 1000000+        | otherwise = max 1 $ min 1000000 $ settingsTimeout set * 250000+    gracefulCloser+#if MIN_VERSION_warp(3,4,21)+        | settingsUnsafeClose set = unsafeGracefulClose+#endif+        | otherwise = gracefulClose+    backend recvN =+        TLS.Backend+            { TLS.backendFlush = return ()+#if MIN_VERSION_network(3,1,1)+            , TLS.backendClose =+                gracefulCloser s 5000 `E.catch` throughAsync (return ())+#else+            , TLS.backendClose = close s+#endif+            , TLS.backendSend = sendAll' s+            , TLS.backendRecv = recvN+            }+    -- A plain send is a foreign call no asynchronous exception+    -- reaches on Windows; this one waits in STM, so the timeout can.+    sendAll' sock bs =+        E.handleJust+            ( \e ->+                if ioeGetErrorType e == ResourceVanished+                    then Just ConnectionClosedByPeer+                    else Nothing+            )+            throwIO+            $ void (sendAllWith sock bs (retry :: STM ()))++-- | Get "Connection" and "Transport" for a TLS connection that is already did the handshake.+-- @since 3.4.7+attachConn+    :: SockAddr+    -> TLS.Context+#if MIN_VERSION_warp(3,4,13)+    -> TVar Int -> IO (Connection, Transport)+#else+    -> IO (Connection, Transport)+#endif+attachConn = attachConnWith 1000000++-- | 'attachConn' giving close_notify the microseconds given.+attachConnWith+    :: Int+    -> SockAddr+    -> TLS.Context+#if MIN_VERSION_warp(3,4,13)+    -> TVar Int -> IO (Connection, Transport)+attachConnWith byeTimeout mysa ctx appsInProgress = do+#else+    -> IO (Connection, Transport)+attachConnWith byeTimeout mysa ctx = do+#endif+    h2 <- (== Just "h2") <$> TLS.getNegotiatedProtocol ctx+    isH2 <- I.newIORef h2+    writeBuffer <- createWriteBuffer 16384+    writeBufferRef <- I.newIORef writeBuffer+    -- Creating a cache for leftover input data.+    tls <- getTLSinfo ctx+    return (conn writeBufferRef isH2, tls)+  where+    conn writeBufferRef isH2 =+        Connection+            { connSendMany = afterCloseNotify . TLS.sendData ctx . L.fromChunks+            , connSendAll = sendall+            , connSendFile = sendfile+            , connClose = close'+            , connRecv = recv+            , connRecvBuf = \_ _ -> return True -- obsoleted+            , connWriteBuffer = writeBufferRef+            , connHTTP2 = isH2+            , connMySockAddr = mysa+#if MIN_VERSION_warp(3,4,13)+            , connAppsInProgress = appsInProgress+#endif+            }+      where+        sendall = afterCloseNotify . TLS.sendData ctx . L.fromChunks . return+        -- tls answers the peer's close_notify with its own, so a send after+        -- it has nowhere to go: dropped, as the GOAWAY which ends an HTTP/2+        -- connection the peer has closed.+        afterCloseNotify = handle $ \e -> case E.fromException e of+            Just (TLS.PostHandshake TLS.Error_EOF) -> return ()+            _ -> throwIO (e :: SomeException)+        recv = handle onEOF $ TLS.recvData ctx+          where+            onEOF e+#if MIN_VERSION_tls(1,8,0)+                | Just (TLS.PostHandshake TLS.Error_EOF) <- E.fromException e = return S.empty+#else+                | Just TLS.Error_EOF <- fromException e = return S.empty+#endif+                | Just ioe <- fromException e, isEOFError ioe = return S.empty+                | otherwise = throwIO e+        sendfile fid offset len hook headers = do+            writeBuffer <- I.readIORef writeBufferRef+            readSendFile+-- readSendFile takes the WriteBuffer itself from 3.5.0, not from 3.4.18:+-- the Listener work went out as 3.4.18 and this did not.+#if MIN_VERSION_warp(3,5,0)+                writeBuffer+#else+                (bufBuffer writeBuffer)+                (bufSize writeBuffer)+#endif+                sendall+                fid+                offset+                len+                hook+                headers++        close' =+            void (tryIO $ timeout byeTimeout sendBye)+                `finally` TLS.contextClose ctx++        sendBye =+            -- It's fine if the connection was closed by the other side before+            -- receiving close_notify, see RFC 5246 section 7.2.1.+            handleJust+                (\e -> guard (e == ConnectionClosedByPeer) >> return e)+                (const (return ()))+                (TLS.bye ctx)++getTLSinfo :: TLS.Context -> IO Transport+getTLSinfo ctx = do+    proto <- TLS.getNegotiatedProtocol ctx+    minfo <- TLS.contextGetInformation ctx+    case minfo of+        Nothing -> return TCP+        Just info -> do+            let (major, minor) = case TLS.infoVersion info of+                    TLS.SSL2 -> (2, 0)+                    TLS.SSL3 -> (3, 0)+                    TLS.TLS10 -> (3, 1)+                    TLS.TLS11 -> (3, 2)+                    TLS.TLS12 -> (3, 3)+                    _ -> (3,4)+            clientCert <- TLS.getClientCertificateChain ctx+            return+                TLS+                    { tlsMajorVersion = major+                    , tlsMinorVersion = minor+                    , tlsNegotiatedProtocol = proto+                    , tlsChiperID = TLS.cipherID $ TLS.infoCipher info+                    , tlsClientCertificate = clientCert+                    }++tryIO :: IO a -> IO (Either IOException a)+tryIO = try++----------------------------------------------------------------++plainHTTP+    :: TLSSettings -> Settings -> Socket -> S.ByteString -> IO (Connection, Transport)+plainHTTP TLSSettings{..} set s bs0 = case onInsecure of+    AllowInsecure -> do+        conn' <- socketConnection set s+        cachedRef <- I.newIORef bs0+        let conn'' =+                conn'+                    { connRecv = recvPlain cachedRef (connRecv conn')+                    }+        return (conn'', TCP)+    DenyInsecure lbs -> do+        -- Listening port 443 but TLS records do not arrive.+        -- We want to let the browser know that TLS is required.+        -- So, we use 426.+        --     http://tools.ietf.org/html/rfc2817#section-4.2+        --     https://tools.ietf.org/html/rfc7231#section-6.5.15+        -- FIXME: should we distinguish HTTP/1.1 and HTTP/2?+        --        In the case of HTTP/2, should we send+        --        GOAWAY + INADEQUATE_SECURITY?+        -- FIXME: Content-Length:+        -- FIXME: TLS/<version>+        sendAll+            s+            "HTTP/1.1 426 Upgrade Required\+            \\r\nUpgrade: TLS/1.0, HTTP/1.1\+            \\r\nConnection: Upgrade\+            \\r\nContent-Type: text/plain\r\n\r\n"+        mapM_ (sendAll s) $ L.toChunks lbs+        unsafeClose s+        throwIO InsecureConnectionDenied++----------------------------------------------------------------++-- | Modify the given receive function to first check the given @IORef@ for a+-- chunk of data. If present, takes the chunk of data from the @IORef@ and+-- empties out the @IORef@. Otherwise, calls the supplied receive function.+recvPlain :: I.IORef S.ByteString -> IO S.ByteString -> IO S.ByteString+recvPlain ref fallback = do+    bs <- I.readIORef ref+    if S.null bs+        then fallback+        else do+            I.writeIORef ref S.empty+            return bs++----------------------------------------------------------------++data WarpTLSException+    = InsecureConnectionDenied+    | ClientClosedConnectionPrematurely+    deriving (Show)+instance Exception WarpTLSException
+ vendor/warp-tls-3.4.18/Network/Wai/Handler/WarpTLS/Internal.hs view
@@ -0,0 +1,172 @@+{-# LANGUAGE OverloadedStrings #-}++module Network.Wai.Handler.WarpTLS.Internal (+    CertSettings (..),+    TLSSettings (..),+    defaultTlsSettings,+    OnInsecure (..),++    -- * Accessors+    getCertSettings,+) where++import qualified Data.ByteString as S+import qualified Data.ByteString.Lazy as L+import qualified Data.IORef as I+import qualified Network.TLS as TLS+import qualified Network.TLS.Extra as TLSExtra+import qualified Network.TLS.SessionManager as SM++----------------------------------------------------------------++-- | Determines where to load the certificate, chain+-- certificates, and key from.+data CertSettings+    = CertFromFile !FilePath ![FilePath] !FilePath+    | CertFromMemory !S.ByteString ![S.ByteString] !S.ByteString+    | CertFromRef+        !(I.IORef S.ByteString)+        ![I.IORef S.ByteString]+        !(I.IORef S.ByteString)++instance Show CertSettings where+    show (CertFromFile a b c) = "CertFromFile " ++ show a ++ " " ++ show b ++ " " ++ show c+    show (CertFromMemory a b c) = "CertFromMemory " ++ show a ++ " " ++ show b ++ " " ++ show c+    show CertFromRef{} = "CertFromRef"++----------------------------------------------------------------++-- | An action when a plain HTTP comes to HTTP over TLS/SSL port.+data OnInsecure+    = DenyInsecure L.ByteString+    | AllowInsecure+    deriving (Show)++----------------------------------------------------------------++-- | Settings for WarpTLS.+data TLSSettings = TLSSettings+    { certSettings :: CertSettings+    -- ^ Where are the certificate, chain certificates, and key+    -- loaded from?+    --+    -- >>> certSettings defaultTlsSettings+    -- CertFromFile "certificate.pem" [] "key.pem"+    --+    -- @since 3.3.0+    , onInsecure :: OnInsecure+    -- ^ Do we allow insecure connections with this server as well?+    --+    -- >>> onInsecure defaultTlsSettings+    -- DenyInsecure "This server only accepts secure HTTPS connections."+    --+    -- Since 1.4.0+    , tlsLogging :: TLS.Logging+    -- ^ The level of logging to turn on.+    --+    -- Default: 'TLS.defaultLogging'.+    --+    -- Since 1.4.0+    , tlsAllowedVersions :: [TLS.Version]+    -- ^ The TLS versions this server accepts.+    --+    -- Since 1.4.2+    , tlsCiphers+        :: [TLS.Cipher]+    -- ^ The TLS ciphers this server accepts.+    --+    -- Since 1.4.2+    , tlsWantClientCert :: Bool+    -- ^ Whether or not to demand a certificate from the client.  If this+    -- is set to True, you must handle received certificates in a server hook+    -- or all connections will fail.+    --+    -- >>> tlsWantClientCert defaultTlsSettings+    -- False+    --+    -- Since 3.0.2+    , tlsServerHooks :: TLS.ServerHooks+    -- ^ The server-side hooks called by the tls package, including actions+    -- to take when a client certificate is received.  See the "Network.TLS"+    -- module for details.+    --+    -- Default: defaultServerHooks+    --+    -- Since 3.0.2+    , tlsServerDHEParams :: Maybe TLS.DHParams+    -- ^ Configuration for ServerDHEParams+    -- more function lives in `crypton` package+    --+    -- Default: Nothing+    --+    -- Since 3.2.2+    , tlsSessionManagerConfig :: Maybe SM.Config+    -- ^ Configuration for in-memory TLS session manager.+    -- If Nothing, 'TLS.noSessionManager' is used.+    -- Otherwise, an in-memory TLS session manager is created+    -- according to 'Config'.+    --+    -- Default: Nothing+    --+    -- Since 3.2.4+    , tlsCredentials :: Maybe TLS.Credentials+    -- ^ Specifying 'TLS.Credentials' directly.  If this value is+    --   specified, other fields such as 'certFile' are ignored.+    --+    --   Since 3.2.12+    , tlsSessionManager :: Maybe TLS.SessionManager+    -- ^ Specifying 'TLS.SessionManager' directly. If this value is+    --   specified, 'tlsSessionManagerConfig' is ignored.+    --+    --   Since 3.2.12+    , tlsSupportedHashSignatures :: [TLS.HashAndSignatureAlgorithm]+    -- ^ Specifying supported hash/signature algorithms, ordered by decreasing+    -- priority. See the "Network.TLS" module for details+    --+    --   Since 3.3.3+    , tlsEarlyDataSize :: Int+    -- ^ The largest TLS 1.3 early data this server accepts, 0 for none.+    -- A request sent in early data can be replayed by anyone who+    -- recorded it, and warp-tls does nothing about that: it has no RFC+    -- 8470 handling and no 425 response. So 0-RTT is off unless this+    -- says otherwise.+    --+    -- Default: 0+    --+    --   Since 3.4.16+    }++-- Since 3.3.1++-- | Some programs need access to cert settings+getCertSettings :: TLSSettings -> CertSettings+getCertSettings = certSettings++-- | The default 'CertSettings'.+defaultCertSettings :: CertSettings+defaultCertSettings = CertFromFile "certificate.pem" [] "key.pem"++----------------------------------------------------------------++-- | Default 'TLSSettings'. Use this to create 'TLSSettings' with the field record name (aka accessors).+defaultTlsSettings :: TLSSettings+defaultTlsSettings =+    TLSSettings+        { certSettings = defaultCertSettings+        , onInsecure = DenyInsecure "This server only accepts secure HTTPS connections."+        , tlsLogging = TLS.defaultLogging+        , tlsAllowedVersions = TLS.supportedVersions TLS.defaultSupported+        , tlsCiphers = ciphers+        , tlsWantClientCert = False+        , tlsServerHooks = TLS.defaultServerHooks+        , tlsServerDHEParams = Nothing+        , tlsSessionManagerConfig = Nothing+        , tlsCredentials = Nothing+        , tlsSessionManager = Nothing+        , tlsSupportedHashSignatures = TLS.supportedHashSignatures TLS.defaultSupported+        , tlsEarlyDataSize = 0+        }++-- taken from stunnel example in tls-extra+ciphers :: [TLS.Cipher]+ciphers = TLSExtra.ciphersuite_strong
+ vendor/warp-tls-3.4.18/certificate.pem view
@@ -0,0 +1,15 @@+-----BEGIN CERTIFICATE-----+MIICWDCCAcGgAwIBAgIJAJG1ZMlcMDW6MA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX+aWRnaXRzIFB0eSBMdGQwHhcNMTExMDIyMTk0MjU3WhcNMTExMTIxMTk0MjU3WjBF+MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB+gQCfYZx7kV6ybogMyAf9MINm7Rwin5LKh+TpD1ZkbLgmqFVotQCdthgTK66SPXkx+EXGI27biNzacJhX7Ml7/4o8sp2GslYKUO46DYvgi/nnNX/bzA5cDJSSGK11eQEVs++p0GEZ/6Juhpx/oQwMDMgo0UHkiH8QtKI8ojXnFF2MsLNwIDAQABo1AwTjAdBgNV+HQ4EFgQUaA6FbOj/0VJMb4egNyIDZ/ZNV/YwHwYDVR0jBBgwFoAUaA6FbOj/0VJM+b4egNyIDZ/ZNV/YwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQCTQyOk+D86Z+yzedXjTLI6FT8QugmQne1YQ8P0w37P76z2reagSvNee2e9B1oTHoPeKZMs0+k99oS9yJ/NOQ1Ms90P+q0yBVGxAs/gF65qKgE27YGXzNtNobj/D4OoxcFG+BsORw+VvYSBV4FiVy9RwJsr7AMqkUBcOEPCuJHgTx58w==+-----END CERTIFICATE-----
+ vendor/warp-tls-3.4.18/key.pem view
@@ -0,0 +1,15 @@+-----BEGIN RSA PRIVATE KEY-----+MIICXAIBAAKBgQCfYZx7kV6ybogMyAf9MINm7Rwin5LKh+TpD1ZkbLgmqFVotQCd+thgTK66SPXkxEXGI27biNzacJhX7Ml7/4o8sp2GslYKUO46DYvgi/nnNX/bzA5cD+JSSGK11eQEVs+p0GEZ/6Juhpx/oQwMDMgo0UHkiH8QtKI8ojXnFF2MsLNwIDAQAB+AoGAR8pgAgjo7tZ60ccIUjOX/LSxB6d5J2Eu6wvNjk6qZD9OuWtOa7up/HigmZ63+CDMjQNI2/o6AOrWtEQkPYZNbibuifzg5V517nHGSqkqjoIgesAiwEsoKpeOgGTtM+MM08oHbJ9uOnDnEEnDBiE0iE3jCTDfmwjqDMpUhu9dZ1EAECQQDKVpzSSV3pzMOp+ixNxMpYxzcE+4K9jgM+MlxPBJSQhVrg/cRQWb26cKBi8LdSxF23hQTsFr+8qLwid+Ah2AgUOBAkEAyaaCjrNRCiHRpd6YzWZ6GKkxbUvxSuOKX3N7hDaE2OFzQTv2Li8B+5mrCsXnSZtOG+MBFdHU66UYie1OzDSDKtwJAKMsvkOID0ihbZmpIwDC/wUjHZkLs+eXY14hVvgShY0XPnb7r/nspWlZsr6Xyf/hhIKfr5yFrBMFMNPIJ5qjflgQJAWsyV+YTgxN4S+6BdxapvIQq58ySA3CGeo+Q4BAimibB4oTal4UpdsHZrZDB00toRs9Dlv+jN70pfGkuS+ZIkIvxQJBAKSf5qpXWp4oZcThkieAiMeAhG96xqRPXhPUxq6QF+YG+T4PF1sjlpZwqy7C+2oF3BqLP09mCW7YkH9Jgnl1zDF8=+-----END RSA PRIVATE KEY-----