diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.3 + +* Another API overhaul. +* Enable seamless processing of `01-index.tar.gz` as well as `01-index.tar`. + ## 0.2 * Overhaul the API. diff --git a/app/History.hs b/app/History.hs --- a/app/History.hs +++ b/app/History.hs @@ -59,7 +59,7 @@ cnfTransitive <- switch $ long "transitive" - <> help "Count transitive (both direct and indirect) dependencies" + <> help "Count transitive (both direct and indirect) dependencies. This mode is imprecise when passing multiple package names at once." pure Config {..} main :: IO () diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -49,7 +49,7 @@ cnfTransitive <- switch $ long "transitive" - <> help "Count transitive (both direct and indirect) dependencies" + <> help "Count transitive (both direct and indirect) dependencies. This mode is imprecise when passing multiple package names at once." pure Config {..} main :: IO () diff --git a/hackage-revdeps.cabal b/hackage-revdeps.cabal --- a/hackage-revdeps.cabal +++ b/hackage-revdeps.cabal @@ -1,12 +1,12 @@ cabal-version: 2.2 name: hackage-revdeps -version: 0.2 +version: 0.3 license: BSD-3-Clause license-file: LICENSE maintainer: andrew.lelechenko@gmail.com author: Bodigrim tested-with: - ghc ==9.12.2 ghc ==9.10.2 ghc ==9.8.4 ghc ==9.6.7 ghc ==9.4.8 + ghc == 9.14.1 ghc ==9.12.2 ghc ==9.10.3 ghc ==9.8.4 ghc ==9.6.7 ghc ==9.4.8 synopsis: List Hackage reverse dependencies description: @@ -38,16 +38,17 @@ bytestring <0.13, containers >=0.5.8 && <0.9, filepath <1.6, - tar <0.7, + tar <0.8, text >=2.0 && <2.2, time <1.16, + zlib <0.8, Cabal-syntax >=3.8 && <3.17 executable hackage-revdeps main-is: Main.hs hs-source-dirs: app default-language: GHC2021 - ghc-options: -Wall -Wunused-packages + ghc-options: -Wall -Wunused-packages -rtsopts build-depends: base, ansi-terminal >=0.11.3 && <1.2, @@ -64,7 +65,7 @@ main-is: History.hs hs-source-dirs: app default-language: GHC2021 - ghc-options: -Wall -Wunused-packages + ghc-options: -Wall -Wunused-packages -rtsopts build-depends: base, ansi-terminal >=0.11.3 && <1.2, diff --git a/src/Hackage/RevDeps.hs b/src/Hackage/RevDeps.hs --- a/src/Hackage/RevDeps.hs +++ b/src/Hackage/RevDeps.hs @@ -1,14 +1,16 @@ -- | Functions to list Hackage reverse dependencies. module Hackage.RevDeps ( - lastVersions, - allLastVersions, + lastVersionsOfPackagesWithNeedles, + lastVersionsOfPackages, extractDependencies, + extractAllDependencies, getReverseDependencies, getTransitiveReverseDependencies, ) where import Codec.Archive.Tar qualified as Tar import Codec.Archive.Tar.Entry qualified as Tar +import Codec.Compression.GZip qualified as Gzip import Control.Exception (throwIO) import Data.ByteString (ByteString) import Data.ByteString.Char8 qualified as B @@ -38,7 +40,7 @@ import Distribution.Version (Version, intersectVersionRanges, mkVersion, simplifyVersionRange) import System.FilePath (isPathSeparator) --- | Scan Cabal index @01-index.tar@ and return Cabal files +-- | Scan Cabal index and return Cabal files -- of last versions (not necessarily latest releases or revisions), which -- contain one of the needles as an entire word (separated by spaces -- or punctuation). @@ -46,20 +48,25 @@ -- To avoid ambiguity: we first select the last versions, -- then filter them by needles. -- --- @since 0.2 -lastVersions - :: Set ByteString +-- @since 0.3 +lastVersionsOfPackagesWithNeedles + :: (PackageName -> Bool) + -- ^ Which packages are we looking for? Could be 'const' 'True'. + -> Set ByteString -- ^ Needles to search in Cabal files. -> FilePath - -- ^ Path to @01-index.tar@. - -- One can use @Cabal.Config.cfgRepoIndex@ from @cabal-install-parsers@ - -- to obtain it. + -- ^ Path to @01-index.tar@ or @01-index.tar.gz@. -> Maybe UTCTime -- ^ Timestamp of index state at which to stop scanning. -> IO (Map PackageName ByteString) -- ^ Map from packages with largest versions to their Cabal files. -lastVersions needles idx indexState = - filterByNeedles needles <$> allLastVersions idx indexState +lastVersionsOfPackagesWithNeedles pkgPred needles idx indexState = + -- There is a memory vs. speed trade off: instead of collecting all last + -- versions into a massive map (~300 M) and then filtering them, + -- we could have filtered them as we go. The downside would be + -- filtering each and every Cabal file, even for old versions + -- which are soon to be rewritten by newer releases. + filterByNeedles needles <$> lastVersionsOfPackages pkgPred idx indexState filterByNeedles :: Set ByteString @@ -72,16 +79,21 @@ -- | Scan Cabal index @01-index.tar@ and return Cabal files -- of last versions (not necessarily latest releases or revisions). -- --- @since 0.2 -allLastVersions - :: FilePath +-- @since 0.3 +lastVersionsOfPackages + :: (PackageName -> Bool) + -- ^ Which packages are we looking for? Could be 'const' 'True'. + -> FilePath + -- ^ Path to @01-index.tar@ or @01-index.tar.gz@. -> Maybe UTCTime + -- ^ Timestamp of index state at which to stop scanning. -> IO (Map PackageName ByteString) -allLastVersions idx indexState = + -- ^ Map from packages with largest versions to their Cabal files. +lastVersionsOfPackages pkgPred idx indexState = fmap (fmap snd) $ foldCabalFilesInIndex idx indexState mempty go where - go pkg ver cnt = M.alter (Just . f) pkg + go pkg ver cnt = if pkgPred pkg then M.alter (Just . f) pkg else id where new = (ver, cnt) f = maybe new (\old@(ver', _) -> if ver' <= ver then new else old) @@ -112,7 +124,10 @@ -> (PackageName -> Version -> ByteString -> a -> a) -> IO a foldCabalFilesInIndex fp indexState ini action = do - contents <- BL.readFile fp + contents' <- BL.readFile fp + -- 1F 8B is GZip magic header, see https://en.wikipedia.org/wiki/Gzip#File_structure + let isGzip = BL.pack [0x1f, 0x8b] `BL.isPrefixOf` contents' + contents = if isGzip then Gzip.decompress contents' else contents' let entries' = Tar.read contents entries = case indexState of Nothing -> entries' @@ -129,7 +144,9 @@ Tar.NormalFile contents _ -> if isCabalFile then action pkgName version bs acc else acc where - bs = BL.toStrict contents + -- If we do not force 'bs' at this point, we'll end up + -- retaining entire 'contents' in memory, which is ~1G. + !bs = BL.toStrict contents fpath = Tar.entryPath entry isCabalFile = ".cabal" `isSuffixOf` fpath (rawPkgName, fpath') = break isPathSeparator fpath @@ -157,7 +174,7 @@ Tar.Done Tar.Fail --- | Scan Cabal file looking for package names, +-- | Scan Cabal file looking for package names matching needles, -- coalescing version bounds from all components and under all conditions. -- -- @since 0.1 @@ -168,31 +185,40 @@ -- ^ Content of a Cabal file. -> Map PackageName VersionRange -- ^ Needles found in the Cabal file and their version bounds. -extractDependencies needles = relevantDeps needles . extractDeps +extractDependencies needles = relevantDeps (`S.member` needles) . extractDeps +-- | Scan Cabal file looking for all dependencies, +-- coalescing version bounds from all components and under all conditions. +-- +-- @since 0.3 +extractAllDependencies + :: ByteString + -- ^ Content of a Cabal file. + -> Map PackageName VersionRange + -- ^ All dependencies found in the Cabal file and their version bounds. +extractAllDependencies = relevantDeps (const True) . extractDeps + extractDeps :: ByteString -> [Dependency] extractDeps cnt = case parseGenericPackageDescriptionMaybe cnt of Nothing -> mempty Just descr -> foldMap targetBuildDepends $ toListOf Lens.traverseBuildInfos descr -relevantDeps :: Set PackageName -> [Dependency] -> Map PackageName VersionRange -relevantDeps needles = +relevantDeps :: (PackageName -> Bool) -> [Dependency] -> Map PackageName VersionRange +relevantDeps predicate = fmap simplifyVersionRange . M.fromListWith intersectVersionRanges . mapMaybe go where go (Dependency pkg ver _) - | pkg `elem` needles = Just (pkg, ver) + | predicate pkg = Just (pkg, ver) | otherwise = Nothing --- | Combination of 'lastVersions' and 'extractDependencies'. +-- | Combination of 'lastVersionsOfPackagesWithNeedles' and 'extractDependencies'. -- -- @since 0.2 getReverseDependencies :: Set PackageName -- ^ Needles to search in Cabal files. -> FilePath - -- ^ Path to @01-index.tar@. - -- One can use @Cabal.Config.cfgRepoIndex@ from @cabal-install-parsers@ - -- to obtain it. + -- ^ Path to @01-index.tar@ or @01-index.tar.gz@. -> Maybe UTCTime -- ^ Timestamp of index state at which to stop scanning. -> IO (Map PackageName (Map PackageName VersionRange)) @@ -200,7 +226,12 @@ -- inner keys are needles, -- inner values are accepted version ranges. getReverseDependencies args fp indexState = do - releases <- lastVersions (S.map (B.pack . unPackageName) args) fp indexState + releases <- + lastVersionsOfPackagesWithNeedles + (const True) + (S.map (B.pack . unPackageName) args) + fp + indexState pure $ extractDependencies' args releases extractDependencies' @@ -222,15 +253,13 @@ :: Set PackageName -- ^ Needles to search in Cabal files. -> FilePath - -- ^ Path to @01-index.tar@. - -- One can use @Cabal.Config.cfgRepoIndex@ from @cabal-install-parsers@ - -- to obtain it. + -- ^ Path to @01-index.tar@ or @01-index.tar.gz@. -> Maybe UTCTime -- ^ Timestamp of index state at which to stop scanning. -> IO (Map PackageName (Map PackageName VersionRange)) -- ^ Reverse dependencies, including transitive ones. getTransitiveReverseDependencies args fp indexState = do - releases <- allLastVersions fp indexState + releases <- lastVersionsOfPackages (const True) fp indexState go mempty releases args where go