diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.4 + +* Yet another API overhaul, extending it with `ExtractDependenciesMode` argument. +* Collect dependencies from `build-tool-depends` too. +* `hackage-revdeps-history` now plots results. + ## 0.3 * Another API overhaul. diff --git a/app/History.hs b/app/History.hs --- a/app/History.hs +++ b/app/History.hs @@ -5,21 +5,28 @@ main, ) where -import Data.Foldable (for_) +import Data.Bifunctor (second) import Data.List qualified as L import Data.List.NonEmpty (NonEmpty) import Data.List.NonEmpty qualified as NE import Data.Map.Strict qualified as M import Data.Set qualified as S +import Data.Text (Text) +import Data.Text qualified as T +import Data.Text.IO qualified as T import Data.Time (Day, UTCTime (..), addDays, getCurrentTime) +import Data.Traversable (for) import Distribution.Client.Config (loadConfig, savedGlobalFlags) import Distribution.Client.GlobalFlags (globalCacheDir) +import Distribution.Parsec (simpleParsec) import Distribution.Simple.Flag (fromFlag) import Distribution.Types.PackageName (PackageName, mkPackageName, unPackageName) -import Hackage.RevDeps (getReverseDependencies, getTransitiveReverseDependencies) -import Options.Applicative (Parser, auto, execParser, fullDesc, help, helper, info, long, metavar, option, progDesc, showDefault, strArgument, switch, value) +import Granite (defPlot, lineGraph, widthChars, xFormatter, yBounds, yFormatter) +import Hackage.RevDeps (ExtractDependenciesMode (..), getReverseDependencies, getTransitiveReverseDependencies) +import Options.Applicative (Parser, argument, auto, execParser, fullDesc, help, helper, info, long, maybeReader, metavar, option, progDesc, showDefault, switch, value) import Options.Applicative.NonEmpty (some1) import System.Console.ANSI (hSupportsANSI, hyperlinkCode) +import System.Console.Terminal.Size qualified as TermSize import System.FilePath (()) import System.IO (stdout) @@ -53,13 +60,13 @@ <> showDefault cnfPackageNames <- some1 $ - strArgument $ + argument (maybeReader simpleParsec) $ metavar "PKGS" <> help "Package names to scan Hackage for their reverse dependencies" cnfTransitive <- switch $ long "transitive" - <> help "Count transitive (both direct and indirect) dependencies. This mode is imprecise when passing multiple package names at once." + <> help "Count transitive (both direct and indirect) dependencies. This mode is imprecise when passing multiple package names at once. Also in this mode we follow only dependencies of the main library" pure Config {..} main :: IO () @@ -77,13 +84,35 @@ let cacheDir = fromFlag $ globalCacheDir $ savedGlobalFlags cnf idx = cacheDir hackageHaskellOrg "01-index.tar" putStrLn $ unwords $ "Date " : map (showPackage supportsAnsi) args - for_ dates $ \date -> do + results <- for dates $ \date -> do let utcTime = Just $ UTCTime date 0 - func = if cnfTransitive then getTransitiveReverseDependencies else getReverseDependencies + func = if cnfTransitive then getTransitiveReverseDependencies OnlyMainLib else getReverseDependencies AllComponents pkgs <- func (S.fromList args) idx utcTime let pkgs' = M.delete (mkPackageName "acme-everything") pkgs - counters = M.unionsWith (+) $ fmap (fmap (const (1 :: Int))) pkgs' - putStrLn $ unwords $ show date : map (\pkg -> showPair (pkg, M.findWithDefault 0 pkg counters)) args + allCounters :: M.Map PackageName Int + allCounters = M.unionsWith (+) $ fmap (fmap (const (1 :: Int))) pkgs' + counters = map (\pkg -> M.findWithDefault 0 pkg allCounters) args + putStrLn $ unwords $ show date : zipWith (curry showPair) args counters + pure (date, counters) + mTermWidth <- fmap (fmap TermSize.width) TermSize.size + let plot = case mTermWidth of + Nothing -> defPlot + Just w -> defPlot {widthChars = max (widthChars defPlot) (w - 16)} + yMax = fromIntegral $ maximum $ 0 : concatMap snd results + plot' = plot {yBounds = (Just 0, Just yMax), xFormatter = const formatAsDate, yFormatter = const formatAsInt} + let graphLines = zip (map (T.pack . unPackageName) args) (uncurry (\ds -> map (zip (map (fromIntegral . fromEnum) ds) . map fromIntegral)) $ second L.transpose $ unzip results) + graph = lineGraph graphLines plot' + putStrLn "" + T.putStrLn graph + +formatAsDate :: Int -> Double -> Text +formatAsDate n x = T.take (n - 1) $ T.pack $ show d + where + d :: Day + d = toEnum (truncate x) + +formatAsInt :: Int -> Double -> Text +formatAsInt _ = T.pack . show . truncate @Double @Int showPair :: Show v => (PackageName, v) -> String showPair (k, v) = diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -15,14 +15,15 @@ import Distribution.Client.Config (loadConfig, savedGlobalFlags) import Distribution.Client.GlobalFlags (globalCacheDir) import Distribution.Compat.NonEmptySet (singleton) +import Distribution.Parsec (simpleParsec) import Distribution.Pretty (pretty) import Distribution.Simple.Flag (fromFlag) import Distribution.Types.Dependency (Dependency (..)) import Distribution.Types.LibraryName (LibraryName (..)) import Distribution.Types.PackageName (PackageName, mkPackageName, unPackageName) import Distribution.Types.VersionRange (VersionRange) -import Hackage.RevDeps (getReverseDependencies, getTransitiveReverseDependencies) -import Options.Applicative (Parser, ReadM, auto, execParser, fullDesc, help, helper, info, long, metavar, option, optional, progDesc, strArgument, switch) +import Hackage.RevDeps (ExtractDependenciesMode (..), getReverseDependencies, getTransitiveReverseDependencies) +import Options.Applicative (Parser, ReadM, argument, auto, execParser, fullDesc, help, helper, info, long, maybeReader, metavar, option, optional, progDesc, switch) import Options.Applicative.NonEmpty (some1) import System.Console.ANSI (hSupportsANSI, hyperlinkCode) import System.FilePath (()) @@ -43,13 +44,13 @@ <> help "Timestamp of index state at which to stop scanning, YYYY-MM-DD" cnfPackageNames <- some1 $ - strArgument $ + argument (maybeReader simpleParsec) $ metavar "PKGS" <> help "Package names to scan Hackage for their reverse dependencies" cnfTransitive <- switch $ long "transitive" - <> help "Count transitive (both direct and indirect) dependencies. This mode is imprecise when passing multiple package names at once." + <> help "Count transitive (both direct and indirect) dependencies. This mode is imprecise when passing multiple package names at once. Also in this mode we follow only dependencies of the main library" pure Config {..} main :: IO () @@ -63,7 +64,7 @@ cnf <- loadConfig minBound mempty let cacheDir = fromFlag $ globalCacheDir $ savedGlobalFlags cnf idx = cacheDir hackageHaskellOrg "01-index.tar" - func = if cnfTransitive then getTransitiveReverseDependencies else getReverseDependencies + func = if cnfTransitive then getTransitiveReverseDependencies OnlyMainLib else getReverseDependencies AllComponents pkgs <- func args idx cnfIndexState report args $ M.delete (mkPackageName "acme-everything") pkgs diff --git a/hackage-revdeps.cabal b/hackage-revdeps.cabal --- a/hackage-revdeps.cabal +++ b/hackage-revdeps.cabal @@ -1,12 +1,13 @@ cabal-version: 2.2 name: hackage-revdeps -version: 0.3 +version: 0.4 license: BSD-3-Clause license-file: LICENSE maintainer: andrew.lelechenko@gmail.com author: Bodigrim tested-with: - ghc == 9.14.1 ghc ==9.12.2 ghc ==9.10.3 ghc ==9.8.4 ghc ==9.6.7 ghc ==9.4.8 + ghc ==9.14.1 ghc ==9.12.2 ghc ==9.10.3 ghc ==9.8.4 ghc ==9.6.7 + ghc ==9.4.8 synopsis: List Hackage reverse dependencies description: @@ -36,13 +37,14 @@ alfred-margaret >=2.0 && <2.2, base >=4.17 && <5, bytestring <0.13, + Cabal >=3.8 && <3.17, + Cabal-syntax >=3.8 && <3.17, containers >=0.5.8 && <0.9, filepath <1.6, tar <0.8, text >=2.0 && <2.2, - time <1.16, - zlib <0.8, - Cabal-syntax >=3.8 && <3.17 + time <1.17, + zlib <0.8 executable hackage-revdeps main-is: Main.hs @@ -50,16 +52,16 @@ default-language: GHC2021 ghc-options: -Wall -Wunused-packages -rtsopts build-depends: - base, ansi-terminal >=0.11.3 && <1.2, + base, + Cabal >=3.8 && <3.17, cabal-install >=3.8 && <3.17, + Cabal-syntax >=3.8 && <3.17, containers, filepath, hackage-revdeps, optparse-applicative >=0.16 && <0.20, - time <1.16, - Cabal-syntax >=3.8 && <3.17, - Cabal >=3.8 && <3.17 + time <1.17 executable hackage-revdeps-history main-is: History.hs @@ -67,13 +69,16 @@ default-language: GHC2021 ghc-options: -Wall -Wunused-packages -rtsopts build-depends: - base, ansi-terminal >=0.11.3 && <1.2, + base, + Cabal >=3.8 && <3.17, cabal-install >=3.8 && <3.17, + Cabal-syntax >=3.8 && <3.17, containers, filepath, + granite >=0.7 && <0.8, hackage-revdeps, optparse-applicative >=0.16 && <0.20, - time <1.16, - Cabal-syntax >=3.8 && <3.17, - Cabal >=3.8 && <3.17 + terminal-size >=0.2 && <0.4, + text <2.2, + time <1.17 diff --git a/src/Hackage/RevDeps.hs b/src/Hackage/RevDeps.hs --- a/src/Hackage/RevDeps.hs +++ b/src/Hackage/RevDeps.hs @@ -6,6 +6,7 @@ extractAllDependencies, getReverseDependencies, getTransitiveReverseDependencies, + ExtractDependenciesMode (..), ) where import Codec.Archive.Tar qualified as Tar @@ -20,7 +21,7 @@ import Data.List (isSuffixOf) import Data.Map.Strict (Map) import Data.Map.Strict qualified as M -import Data.Maybe (mapMaybe) +import Data.Maybe (mapMaybe, maybeToList) import Data.Set (Set) import Data.Set qualified as S import Data.Text (Text) @@ -31,10 +32,16 @@ import Data.Time (UTCTime) import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) import Distribution.Compat.Lens (toListOf) +import Distribution.Compat.NonEmptySet qualified as NES (singleton) +import Distribution.PackageDescription (libBuildInfo, library) +import Distribution.PackageDescription.Configuration (flattenPackageDescription) import Distribution.PackageDescription.Parsec (parseGenericPackageDescriptionMaybe) -import Distribution.Types.BuildInfo (targetBuildDepends) +import Distribution.Simple.BuildToolDepends (getAllToolDependencies) +import Distribution.Types.BuildInfo (BuildInfo, buildable, targetBuildDepends) import Distribution.Types.BuildInfo.Lens qualified as Lens import Distribution.Types.Dependency (Dependency (..)) +import Distribution.Types.ExeDependency (ExeDependency (..)) +import Distribution.Types.LibraryName (LibraryName (..)) import Distribution.Types.PackageName (PackageName, mkPackageName, unPackageName) import Distribution.Types.VersionRange (VersionRange) import Distribution.Version (Version, intersectVersionRanges, mkVersion, simplifyVersionRange) @@ -76,7 +83,7 @@ where machine = Aho.build (map ((\x -> (x, x)) . decodeUtf8Lenient) (S.toList needles)) --- | Scan Cabal index @01-index.tar@ and return Cabal files +-- | Scan Cabal index @01-index.tar@ (or @01-index.tar.gz@) and return Cabal files -- of last versions (not necessarily latest releases or revisions). -- -- @since 0.3 @@ -179,30 +186,58 @@ -- -- @since 0.1 extractDependencies - :: Set PackageName + :: ExtractDependenciesMode + -- ^ Which package components to scan? + -> Set PackageName -- ^ Needles to search. -> ByteString -- ^ Content of a Cabal file. -> Map PackageName VersionRange -- ^ Needles found in the Cabal file and their version bounds. -extractDependencies needles = relevantDeps (`S.member` needles) . extractDeps +extractDependencies mode needles = relevantDeps (`S.member` needles) . extractDeps mode -- | Scan Cabal file looking for all dependencies, -- coalescing version bounds from all components and under all conditions. -- -- @since 0.3 extractAllDependencies - :: ByteString + :: ExtractDependenciesMode + -- ^ Which package components to scan? + -> ByteString -- ^ Content of a Cabal file. -> Map PackageName VersionRange -- ^ All dependencies found in the Cabal file and their version bounds. -extractAllDependencies = relevantDeps (const True) . extractDeps +extractAllDependencies mode = relevantDeps (const True) . extractDeps mode -extractDeps :: ByteString -> [Dependency] -extractDeps cnt = case parseGenericPackageDescriptionMaybe cnt of +-- | Which package components to look inside for dependencies? +-- +-- @since 0.4 +data ExtractDependenciesMode + = -- | All buildable components, including tests, benchmarks, sublibraries. + AllComponents + | -- | Only the main library if any. + OnlyMainLib + +extractDeps :: ExtractDependenciesMode -> ByteString -> [Dependency] +extractDeps mode cnt = case parseGenericPackageDescriptionMaybe cnt of Nothing -> mempty - Just descr -> foldMap targetBuildDepends $ toListOf Lens.traverseBuildInfos descr + Just descr -> foldMap (extractDepsFromBuildInfo getToolDeps) buildInfos + where + flatDescr = flattenPackageDescription descr + getToolDeps = getAllToolDependencies flatDescr + buildInfos = case mode of + AllComponents -> toListOf Lens.traverseBuildInfos flatDescr + OnlyMainLib -> maybeToList (fmap libBuildInfo (library flatDescr)) +extractDepsFromBuildInfo :: (BuildInfo -> [ExeDependency]) -> BuildInfo -> [Dependency] +extractDepsFromBuildInfo getToolDeps bi + | buildable bi = + targetBuildDepends bi + <> map + (\(ExeDependency pkg _ ver) -> Dependency pkg ver (NES.singleton LMainLibName)) + (getToolDeps bi) + | otherwise = [] + relevantDeps :: (PackageName -> Bool) -> [Dependency] -> Map PackageName VersionRange relevantDeps predicate = fmap simplifyVersionRange . M.fromListWith intersectVersionRanges . mapMaybe go @@ -215,7 +250,9 @@ -- -- @since 0.2 getReverseDependencies - :: Set PackageName + :: ExtractDependenciesMode + -- ^ Which package components to scan? + -> Set PackageName -- ^ Needles to search in Cabal files. -> FilePath -- ^ Path to @01-index.tar@ or @01-index.tar.gz@. @@ -225,32 +262,34 @@ -- ^ Outer keys are reverse dependencies, -- inner keys are needles, -- inner values are accepted version ranges. -getReverseDependencies args fp indexState = do +getReverseDependencies mode args fp indexState = do releases <- lastVersionsOfPackagesWithNeedles (const True) (S.map (B.pack . unPackageName) args) fp indexState - pure $ extractDependencies' args releases + pure $ extractDependencies' args $ fmap (extractDeps mode) releases extractDependencies' :: Set PackageName - -> Map PackageName ByteString + -> Map PackageName [Dependency] -> Map PackageName (Map PackageName VersionRange) extractDependencies' args releases = M.filter (not . null) $ -- Most of packages trivially depend on themselves -- (e. g., test suite depends on a library); skip it. M.mapWithKey M.delete $ - fmap (extractDependencies args) releases + fmap (relevantDeps (`S.member` args)) releases -- | Same as 'getReverseDependencies', but -- returns not only direct, but also indirect reverse dependencies. -- -- @since 0.2 getTransitiveReverseDependencies - :: Set PackageName + :: ExtractDependenciesMode + -- ^ Which package components to scan? + -> Set PackageName -- ^ Needles to search in Cabal files. -> FilePath -- ^ Path to @01-index.tar@ or @01-index.tar.gz@. @@ -258,13 +297,13 @@ -- ^ Timestamp of index state at which to stop scanning. -> IO (Map PackageName (Map PackageName VersionRange)) -- ^ Reverse dependencies, including transitive ones. -getTransitiveReverseDependencies args fp indexState = do +getTransitiveReverseDependencies mode args fp indexState = do releases <- lastVersionsOfPackages (const True) fp indexState - go mempty releases args + go mempty (fmap (extractDeps mode) releases) args where go :: Map PackageName (Map PackageName VersionRange) - -> Map PackageName ByteString + -> Map PackageName [Dependency] -> Set PackageName -> IO (Map PackageName (Map PackageName VersionRange)) go acc releases xs = do