diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,3 +46,8 @@ ## 0.1.3.6 -- 2023-07-10 * Complete re-work of hashFileStreamly in order to fix non-constant memory usage. + +## 0.1.4.0 -- 2023-07-14 + +* Added staticDevelStreamly, staticFilesListStreamly, staticFilesMapStreamly, staticFilesMergeMapStreamly, and publicFilesStreamly replacement functions in Yesod.Static.Streamly. +* Added supporting functionality for staticDevelStreamly, cachedETagLookupDevelStreamly, in Yesod.Static.Streamly.Internal. diff --git a/src/Yesod/Static/Streamly.hs b/src/Yesod/Static/Streamly.hs --- a/src/Yesod/Static/Streamly.hs +++ b/src/Yesod/Static/Streamly.hs @@ -18,15 +18,25 @@ -- -- If you have large files to cache within your static directory, you may very well need to increase you file descriptor limit in order to utilize 'staticStreamly' and 'staticFilesStreamly' properly. -module Yesod.Static.Streamly ( -- * Yesod.Static Replacement functions +module Yesod.Static.Streamly ( -- * Yesod.Static Replacement functions - Smart constructor staticStreamly, - staticFilesStreamly + staticDevelStreamly, + -- * Yesod.Static Replacement functions - Template Haskell helpers + staticFilesStreamly, + staticFilesListStreamly, + staticFilesMapStreamly, + staticFilesMergeMapStreamly, + publicFilesStreamly ) where import Yesod.Static.Streamly.Internal +import Data.List (foldl') +import qualified Data.Map as M import Language.Haskell.TH +import Language.Haskell.TH.Syntax as TH import Network.Wai.Application.Static (webAppSettingsWithLookup) +import qualified System.FilePath as FP import Yesod.Static -- | A more performant replacement of @@ -41,6 +51,17 @@ return $ Static $ webAppSettingsWithLookup dir hashLookup -- | A more performant replacement of +-- [staticDevel](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/Yesod-Static.html#v:staticDevel) +-- found in [Yesod.Static](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/Yesod-Static.html). +staticDevelStreamly :: FilePath -- ^ file path of static directory + -> Int -- ^ buffer size (0.25 - 0.50 x your L2 cache seems to be best.) + -> IO Static +staticDevelStreamly dir size = do + hashLookup <- cachedETagLookupDevelStreamly dir + size + return $ Static $ webAppSettingsWithLookup dir hashLookup + +-- | A more performant replacement of -- [staticFiles](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/src/Yesod.Static.html#staticFiles) -- found in [Yesod.Static](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/Yesod-Static.html). staticFilesStreamly :: FilePath -- ^ file path of static directory @@ -48,3 +69,90 @@ -> Q [Dec] staticFilesStreamly dir size = mkStaticFilesStreamly dir size + +-- | A more performant replacement of +-- [staticFilesList](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/src/Yesod.Static.html#staticFilesList) +-- found in [Yesod.Static](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/Yesod-Static.html). +staticFilesListStreamly :: FilePath -- ^ file path of static directory + -> [FilePath] + -> Int -- ^ buffer size (0.25 - 0.50 x your L2 cache seems to be best.) + -> Q [Dec] +staticFilesListStreamly dir fs size = + mkStaticFilesListStreamly dir + (map split fs) + True + size + where + split :: FilePath + -> [String] + split [] = [] + split x = let (a, b) = break (== '/') x + in a : split (drop 1 b) + +-- | A more performant replacement of +-- [staticFilesMap](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/src/Yesod.Static.html#staticFilesMap) +-- found in [Yesod.Static](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/Yesod-Static.html). +staticFilesMapStreamly :: FilePath -- ^ file path of static directory + -> M.Map FilePath FilePath + -> Int -- ^ buffer size (0.25 - 0.50 x your L2 cache seems to be best.) + -> Q [Dec] +staticFilesMapStreamly fp m size = + mkStaticFilesListStreamly' fp + (map splitBoth mapList) + True + size + where + splitBoth (k, v) = (split k, split v) + mapList = M.toList m + split :: FilePath + -> [String] + split [] = [] + split x = let (a, b) = break (== '/') x + in a : split (drop 1 b) + +-- | A more performant replacement of +-- [staticFilesMergeMap](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/src/Yesod.Static.html#staticFilesMergeMap) +-- found in [Yesod.Static](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/Yesod-Static.html). +staticFilesMergeMapStreamly :: FilePath -- ^ file path of static directory + -> M.Map FilePath FilePath + -> Int -- ^ buffer size (0.25 - 0.50 x your L2 cache seems to be best.) + -> Q [Dec] +staticFilesMergeMapStreamly fp m size = do + fs <- qRunIO $ getFileListPiecesStreamly fp + let filesList = map FP.joinPath fs + mergedMapList = M.toList $ foldl' (checkedInsert invertedMap) m filesList + mkStaticFilesListStreamly' fp + (map splitBoth mergedMapList) + True + size + where + splitBoth (k,v) = (split k, split v) + swap (x,y) = (y, x) + mapList = M.toList m + invertedMap = M.fromList $ map swap mapList + split :: FilePath + -> [String] + split [] = [] + split x = let (a, b) = break (== '/') x + in a : split (drop 1 b) + -- We want to keep mappings for all files that are pre-fingerprinted, + -- so this function checks against all of the existing fingerprinted files and + -- only inserts a new mapping if it's not a fingerprinted file. + checkedInsert + :: M.Map FilePath FilePath -- inverted dictionary + -> M.Map FilePath FilePath -- accumulating state + -> FilePath + -> M.Map FilePath FilePath + checkedInsert iDict st p = if M.member p iDict + then st + else M.insert p p st + +-- | A more performant replacement of +-- [publicFiles](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/src/Yesod.Static.html#publicFiles) +-- found in [Yesod.Static](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/Yesod-Static.html). +publicFilesStreamly :: FilePath -- ^ file path of static directory + -> Int -- ^ buffer size (0.25 - 0.50 x your L2 cache seems to be best.) + -> Q [Dec] +publicFilesStreamly dir size = mkStaticFilesStreamly' dir + False + size diff --git a/src/Yesod/Static/Streamly/Internal.hs b/src/Yesod/Static/Streamly/Internal.hs --- a/src/Yesod/Static/Streamly/Internal.hs +++ b/src/Yesod/Static/Streamly/Internal.hs @@ -36,11 +36,12 @@ -- -- This library utilizes [Streamly](https://hackage.haskell.org/package/streamly-core)'s superb performance characteristics to replace some of [Yesod](https://hackage.haskell.org/package/yesod)'s functionality with streamly-based functionality. -module Yesod.Static.Streamly.Internal ( -- * Yesod.Static Replacement functions +module Yesod.Static.Streamly.Internal ( -- * Yesod.Static Replacement functions (INTERNAL) mkStaticFilesStreamly, mkStaticFilesStreamly', mkStaticFilesListStreamly, mkStaticFilesListStreamly', + cachedETagLookupDevelStreamly, cachedETagLookupStreamly, mkHashMapStreamly, notHiddenStreamly, @@ -61,6 +62,7 @@ import qualified Data.ByteString.Base64 import qualified Data.ByteString.Char8 as S8 import Data.Char (isLower,isDigit) +import Data.IORef (readIORef,newIORef,writeIORef) import Data.List (foldl',intercalate,sort) import qualified Data.Map as M import Data.Text (pack) @@ -71,6 +73,8 @@ import Streamly.Internal.FileSystem.File as StreamlyInternalFile (chunkReaderWith) import Streamly.Internal.System.IO (arrayPayloadSize) import System.Directory (doesDirectoryExist,doesFileExist,getDirectoryContents) +import System.PosixCompat.Files (getFileStatus,modificationTime) +import System.Posix.Types (EpochTime) import WaiAppStatic.Storage.Filesystem (ETagLookup) import Yesod.Static @@ -137,6 +141,27 @@ [ Clause [] (NormalB $ (ConE 'StaticRoute) `AppE` f' `AppE` qs) [] ] ] + +-- | A replacement of +-- [cachedETagLookupDevel](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/src/Yesod.Static.html#cachedETagLookupDevel). +cachedETagLookupDevelStreamly :: FilePath + -> Int + -> IO ETagLookup +cachedETagLookupDevelStreamly dir size = do + etags <- mkHashMapStreamly dir + size + mtimeVar <- newIORef (M.empty :: M.Map FilePath EpochTime) + return $ \f -> + case M.lookup f etags of + Nothing -> return Nothing + Just checksum -> do + fs <- getFileStatus f + let newt = modificationTime fs + mtimes <- readIORef mtimeVar + oldt <- case M.lookup f mtimes of + Nothing -> writeIORef mtimeVar (M.insert f newt mtimes) >> return newt + Just oldt -> return oldt + return $ if newt /= oldt then Nothing else Just checksum -- | A replacement of -- [cachedETagLookup](https://hackage.haskell.org/package/yesod-static-1.6.1.0/docs/src/Yesod.Static.html#cachedETagLookup). diff --git a/yesod-static-streamly.cabal b/yesod-static-streamly.cabal --- a/yesod-static-streamly.cabal +++ b/yesod-static-streamly.cabal @@ -20,7 +20,7 @@ -- PVP summary: +-+------- breaking API changes -- | | +----- non-breaking API additions -- | | | +--- code changes with no API change -version: 0.1.3.6 +version: 0.1.4.0 -- A short (one-line) description of the package. synopsis: A streamly-based library providing performance-focused alternatives for functionality found in yesod-static. @@ -84,6 +84,7 @@ containers >= 0.6.7 && < 0.7, cryptonite >= 0.30 && < 0.31, directory >= 1.3.7 && < 1.4, + filepath, memory >= 0.18.0 && < 0.19, monad-control >= 1.0.3 && < 1.1, mtl >= 2.2.2 && < 2.3, @@ -92,6 +93,7 @@ streamly-core >= 0.1.0 && < 0.2, template-haskell >= 2.19.0 && < 2.20, text >= 2.0.2 && < 2.1, + unix-compat, wai-app-static >= 3.1.7 && < 3.2, yesod-core >= 1.6.24 && < 1.7, yesod-static >= 1.6.1 && < 1.7