diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +# 0.6.0 (2022-10-27) +- significant processing performance improvement: drop expensive nub +- allow multiple mode options and separate summary & rst options +- add support for package list files via http +- fix local filesystem recursion +- initial testsuite + ## 0.5.0 (2021-11-03) - use rpm-nvr library for NVRA type (API changes) and rpmVerCompare to order ver-rel's diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -5,82 +5,103 @@ [![Stackage Lts](http://stackage.org/package/pkgtreediff/badge/lts)](http://stackage.org/lts/package/pkgtreediff) [![Stackage Nightly](http://stackage.org/package/pkgtreediff/badge/nightly)](http://stackage.org/nightly/package/pkgtreediff) -`pkgtreediff` compares the NVRs (name-version-release) of RPM packages in OS package trees and/or installations: +`pkgtreediff` compares the NVRs (name-version-release) of RPM packages +in OS package trees and/or installations: - An OS tree can be referenced by an url or directory containing a tree of rpm files. -- A file containing a list(s) of rpm NVRs can also be compared. +- A text file/url containing a list of rpm NVRs can also be compared. - Commands can also be used to get installed RPMs, eg: - `"rpm -qa"` - `"ssh myhost rpm -qa"` - `"podman run --rm myimage rpm -qa"` + - `"sudo dnf -q repoquery --qf '%{name}-%{version}-%{release}.%{arch}'"` - A koji tag: koji://tag@KojiHubUrl (or koji://tag@fedora) ## Usage examples -### Containers +### Help +```shellsession +$ pkgtreediff --version +0.6.0 +$ pkgtreediff -h +Package tree comparison tool +Usage: pkgtreediff [--version] [-r|--recursive] [-d|--subdir SUBDIR] + [(-R|--ignore-release) | (-V|--ignore-version)] + [(-N|--new) | (-D|--deleted) | (-U|--updated) | + (-u|--downgraded)] + [(-s|--show-summary) | (-S|--no-summary)] [-R|--rst] + [-p|--pattern PKGPATTERN] [-t|--timeout SECONDS] + URL|DIR|FILE|KOJITAG|CMD1 URL|DIR|FILE|KOJITAG|CMD2 + pkgtreediff compares the packages in two OS trees or instances + +Available options: + -h,--help Show this help text + --version Show version + -r,--recursive Recursive down into subdirectories + -d,--subdir SUBDIR Select specific subdir (eg x86_64 or source) + -R,--ignore-release Only show version changes (ignore release) + -V,--ignore-version Only show package changes (ignore version-release) + -N,--new Show only added packages + -D,--deleted Show only removed packages + -U,--updated Show only upgraded packages + -u,--downgraded Show only downgraded packages + -s,--show-summary Show summary of changes (default when >20 changes) + -S,--no-summary Do not display summary + -R,--rst Use ReSTructured Text format + -p,--pattern PKGPATTERN Limit packages to glob matches + -t,--timeout SECONDS Maximum seconds to wait for http response before + timing out (default 30) +``` + +### Containers Compare the content of two rpm based containers versions, filtering for new added packages: ```shellsession -$ pkgtreediff --new "podman run --rm fedora:34 rpm -qa" "podman run --rm fedora:35 rpm -qa" -libfsverity-1.4-4.fc35.x86_64 -mpdecimal-2.5.1-2.fc35.x86_64 +$ pkgtreediff --new "podman run --rm fedora:36 rpm -qa" "podman run --rm fedora:37 rpm -qa" +libb2-0.98.1-7.fc37.x86_64 ``` ### Package trees -Package source changes between Fedora 33 and 34 Server at GA (ignoring release bumps): +Package source changes between Fedora 35 and 36 Server at GA +(ignoring release bumps): ```shellsession -$ pkgtreediff --ignore-release https://dl.fedoraproject.org/pub/fedora/linux/releases/{33,34}/Server/source/tree/Packages/ -ModemManager.src: 1.14.2-1.fc33 -> 1.14.10-2.fc34 -NetworkManager.src: 1.26.2-2.fc33 -> 1.30.2-1.fc34 -PackageKit.src: 1.2.1-1.fc33 -> 1.2.3-1.fc34 -abrt.src: 2.14.4-6.fc33 -> 2.14.5-2.fc34 -: -bash.src: 5.0.17-2.fc33 -> 5.1.0-2.fc34 -bash-completion.src: 2.8-9.fc33 -> 2.11-2.fc34 -- bcache-tools-1.0.8-19.fc33.src -bcm283x-firmware.src: 20201008-2.63b1922.fc33 -> 20210407-1.8c7c524.fc34 -bind.src: 9.11.23-1.fc33 -> 9.16.11-5.fc34 -binutils.src: 2.35-10.fc33 -> 2.35.1-41.fc34 -: -cockpit.src: 229-1.fc33 -> 241-1.fc34 -colord.src: 1.4.4-5.fc33 -> 1.4.5-2.fc34 -+ compsize-1.5-1.fc34.src -conmon.src: 2.0.21-3.fc33 -> 2.0.27-1.fc34 -container-selinux.src: 2.145.0-1.fc33 -> 2.158.0-1.gite78ac4f.fc34 -containernetworking-plugins.src: 0.8.7-1.fc33 -> 0.9.1-4.fc34 -+ containers-common-1-13.fc34.src -- copy-jdk-configs-3.7-7.fc33.src -- createrepo_c-0.15.11-4.fc33.src -+ criu-3.15-3.fc34.src -crun.src: 0.15-5.fc33 -> 0.18-5.fc34 +$ pkgtreediff --ignore-release https://download.fedoraproject.org/pub/fedora/linux/releases/{35,36}/Server/source/tree/Packages/ +ModemManager.src: 1.18.2-1.fc35 -> 1.18.6-1.fc36 +NetworkManager.src: 1.32.12-1.fc35 -> 1.36.4-1.fc36 +PackageKit.src: 1.2.4-3.fc35 -> 1.2.5-1.fc36 +abrt.src: 2.14.6-9.fc35 -> 2.15.1-1.fc36 : : -vim.src: 8.2.1770-1.fc33 -> 8.2.2637-1.fc34 -+ vulkan-loader-1.2.162.0-2.fc34.src -wayland.src: 1.18.0-2.fc33 -> 1.19.0-1.fc34 -webkit2gtk3.src: 2.30.1-1.fc33 -> 2.32.0-2.fc34 -wget.src: 1.20.3-8.fc33 -> 1.21.1-2.fc34 +- tree-1.8.0-7.fc35.src ++ tree-pkg-2.0.1-2.fc36.src +tzdata.src: 2021b-1.fc35 -> 2022a-1.fc36 : -zchunk.src: 1.1.5-3.fc33 -> 1.1.9-2.fc34 -zstd.src: 1.4.5-5.fc33 -> 1.4.9-1.fc34 +xorg-x11-xauth.src: 1.1-9.fc35 -> 1.1.1-2.fc36 ++ xrdb-1.2.1-3.fc36.src +xxhash.src: 0.8.0-4.fc35 -> 0.8.1-2.fc36 +yelp.src: 41.1-1.fc35 -> 42.1-1.fc36 +yelp-xsl.src: 41.0-1.fc35 -> 42.0-1.fc36 +zchunk.src: 1.1.15-2.fc35 -> 1.2.1-1.fc36 +zstd.src: 1.5.0-2.fc35 -> 1.5.2-1.fc36 Summary -Updated: 330 +Updated: 293 Downgraded: 0 -Added: 63 -Deleted: 98 +Added: 36 +Deleted: 21 Arch changed: 0 -Total packages: 932 -> 897 +Total packages: 915 -> 930 ``` -### Hosts +For very large repos like Everything it should be faster +to use a repoquery command. +### Hosts Compare the packages on local and another host: - ``` pkgtreediff "rpm -qa" "ssh otherhost rpm -qa" ``` @@ -88,15 +109,13 @@ Any types of sources can be compared, together with the use of flags. ### Koji - Compare koji tags using the `koji://tag@kojihub` syntax: - ``` pkgtreediff koji://dist-c8-updates-build@centos koji://dist-c8_1-updates-build@centos ``` -Please avoid using koji tag to compare full releases as it is more efficient to -query URL or CMD trees. +Please avoid using koji tags to compare full releases as it is more efficient +to query URL or CMD trees. ## Builds pkgtreediff is packaged in Fedora. @@ -107,10 +126,9 @@ ([more details](https://copr.fedorainfracloud.org/coprs/petersen/pkgtreediff/monitor/detailed)). ## RPM version ordering +RPM version ordering is somewhat [complicated](https://blog.jasonantman.com/2014/07/how-yum-and-rpm-compare-versions/). +For example "1.2.1~rc1" < "1.2.1" (greater for ^). + pkgtreediff use the [rpm-nvr](https://hackage.haskell.org/package/rpm-nvr) library implementation of the `rpmvercmp()` algorithm, though it has not been verified to behave identically. - -RPM version ordering is somewhat involved -. -For example "1.2.1~rc1" < "1.2.1" (greater for ^). diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -9,4 +9,6 @@ - support epochs in file package list - diff isos - "--query-format" -- testsuite!! +- more tests + +- diff via repoquery diff --git a/pkgtreediff.cabal b/pkgtreediff.cabal --- a/pkgtreediff.cabal +++ b/pkgtreediff.cabal @@ -1,21 +1,22 @@ cabal-version: 1.18 name: pkgtreediff -version: 0.5.0 +version: 0.6.0 synopsis: RPM package tree diff tool -description: Tool for comparing RPM packages and versions in OS dist trees or instances. +description: Tool for comparing RPM packages and version-releases + in OS dist trees or instances. homepage: https://github.com/juhp/pkgtreediff bug-reports: https://github.com/juhp/pkgtreediff/issues license: GPL-3 license-file: LICENSE author: Jens Petersen maintainer: juhpetersen@gmail.com -copyright: 2019-2021 Jens Petersen +copyright: 2019-2022 Jens Petersen category: Utility build-type: Simple extra-doc-files: README.md , CHANGELOG.md , TODO -tested-with: GHC == 8.8.4, GHC == 8.10.7, GHC == 9.0.1 +tested-with: GHC == 8.8.4, GHC == 8.10.7, GHC == 9.0.2, GHC == 9.2.4 GHC == 8.0.2, GHC == 8.2.2, GHC == 8.4.4, GHC == 8.6.5 source-repository head @@ -28,6 +29,7 @@ build-depends: async , base < 5 + , bytestring , directory , extra , filepath @@ -35,7 +37,8 @@ , Glob , http-client >= 0.5.0 , http-client-tls - , http-directory >= 0.1.4 && (< 0.1.6 || >= 0.1.8) + , http-directory >= 0.1.10 + , http-types , pkgtreediff , rpm-nvr , simple-cmd >= 0.1.4 @@ -50,7 +53,6 @@ default-language: Haskell2010 library - build-depends: base < 5 exposed-modules: Distribution.RPM.PackageTreeDiff hs-source-dirs: src @@ -74,3 +76,15 @@ -Wpartial-fields if impl(ghc >= 8.10) ghc-options: -Wunused-packages + +test-suite test + main-is: tests.hs + type: exitcode-stdio-1.0 + hs-source-dirs: test + + default-language: Haskell2010 + + ghc-options: -Wall + build-depends: base >= 4 && < 5, + simple-cmd + build-tool-depends: pkgtreediff:pkgtreediff diff --git a/src/Distribution/RPM/PackageTreeDiff.hs b/src/Distribution/RPM/PackageTreeDiff.hs --- a/src/Distribution/RPM/PackageTreeDiff.hs +++ b/src/Distribution/RPM/PackageTreeDiff.hs @@ -2,12 +2,12 @@ -- | A library for pkgtreediff for comparing trees of rpm packages module Distribution.RPM.PackageTreeDiff - (NVRA(..), - readNVRA, - RPMPkgDiff(..), - diffPkgs, + (RPMPkgDiff(..), Ignore(..), - Mode(..), + diffPkgs, + -- * from rpm-nvr + NVRA(..), + readNVRA ) where #if !MIN_VERSION_base(4,8,0) @@ -16,17 +16,6 @@ import Data.RPM.VerRel import Data.RPM.NVRA import Data.RPM.VerCmp (rpmVerCompare) - --- | Mode describes the kind of summary generated by compareDirs -data Mode = AutoSummary - | NoSummary - | ShowSummary - | Added - | Deleted - | Updated - | Downgraded - | RST - deriving Eq -- | Ignore describes how comparison is done data Ignore = IgnoreNone -- ^ do not ignore version or release diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -12,6 +12,7 @@ import Control.Concurrent.Async (concurrently) import Control.Monad import Control.Monad.Extra (concatMapM) +import qualified Data.ByteString.Char8 as B import Data.List import Data.Maybe import Data.RPM.NVRA @@ -19,12 +20,15 @@ import Data.Semigroup ((<>)) #endif import qualified Data.Text as T -import Network.HTTP.Client (managerResponseTimeout, newManager, - responseTimeoutMicro) +import qualified Data.Text.Lazy as TL +import Data.Text.Lazy.Encoding (decodeUtf8) +import Network.HTTP.Client (httpLbs, managerResponseTimeout, newManager, + parseRequest, responseBody, responseTimeoutMicro) import Network.HTTP.Client.TLS (tlsManagerSettings) import Network.HTTP.Directory +import Network.HTTP.Types (hContentType) import SimpleCmdArgs -import System.Directory (doesDirectoryExist, getDirectoryContents) +import System.Directory (doesDirectoryExist, listDirectory) import System.FilePath (()) import System.FilePath.Glob (compile, match) #if !MIN_VERSION_simple_cmd(0,2,0) @@ -43,11 +47,32 @@ import Distribution.RPM.PackageTreeDiff import Paths_pkgtreediff (version) +data Summary = AutoSummary + | NoSummary + | ShowSummary + deriving Eq + +data Mode = Added + | Deleted + | Updated + | Downgraded + deriving Eq + main :: IO () main = simpleCmdArgs (Just version) "Package tree comparison tool" "pkgtreediff compares the packages in two OS trees or instances" $ - compareDirs <$> recursiveOpt <*> optional subdirOpt <*> ignoreVR <*> modeOpt <*> optional patternOpt <*> timeoutOpt <*> sourceArg "1" <*> sourceArg "2" + compareDirs + <$> recursiveOpt + <*> optional subdirOpt + <*> ignoreVR + <*> many modeOpt + <*> summaryOpt + <*> switchWith 'R' "rst" "Use ReSTructured Text format" + <*> optional patternOpt + <*> timeoutOpt + <*> sourceArg "1" + <*> sourceArg "2" where sourceArg :: String -> Parser String sourceArg pos = strArg ("URL|DIR|FILE|KOJITAG|CMD" <> pos) @@ -57,9 +82,10 @@ flagWith' Added 'N' "new" "Show only added packages" <|> flagWith' Deleted 'D' "deleted" "Show only removed packages" <|> flagWith' Updated 'U' "updated" "Show only upgraded packages" <|> - flagWith' Downgraded 'u' "downgraded" "Show only downgraded packages" <|> + flagWith' Downgraded 'u' "downgraded" "Show only downgraded packages" + + summaryOpt = flagWith' ShowSummary 's' "show-summary" ("Show summary of changes (default when >" <> show summaryThreshold <> " changes)") <|> - flagWith' RST 'R' "rst" "Print summary in ReSTructured Text format" <|> flagWith AutoSummary NoSummary 'S' "no-summary" "Do not display summary" -- ignoreArch :: Parser Bool @@ -120,18 +146,19 @@ isHttp loc = "http:" `isPrefixOf` loc || "https:" `isPrefixOf` loc -- | Frontend for the pkgtreediff tool -compareDirs :: Bool -> Maybe String -> Ignore -> Mode -> Maybe String -> Int -> String -> String -> IO () -compareDirs recursive msubdir ignore mode mpattern timeout tree1 tree2 = do +compareDirs :: Bool -> Maybe String -> Ignore -> [Mode] -> Summary -> Bool + -> Maybe String -> Int -> String -> String -> IO () +compareDirs recursive msubdir ignore modes summary rst mpattern timeout tree1 tree2 = do (ps1,ps2) <- getTrees tree1 tree2 let diff = diffPkgs ignore ps1 ps2 - if mode /= RST - then mapM_ putStrLn . mapMaybe (showPkgDiff mode) $ diff - else printRST diff - when (mode /= NoSummary && isDefault mode) $ - when (mode == ShowSummary || length diff > summaryThreshold) $ do + if rst + then printRST diff + else mapM_ putStrLn . mapMaybe (showPkgDiff modes) $ diff + when (summary /= NoSummary && null modes) $ + when (summary == ShowSummary || length diff > summaryThreshold) $ do putStrLn "" - (if mode /= RST then putStrLn else printRSTHeader) "Summary" - let diffsum = summary diff + (if rst then printRSTHeader else putStrLn) "Summary" + let diffsum = changesummary diff putStrLn $ "Updated: " <> show (updateSum diffsum) putStrLn $ "Downgraded: " <> show (downgradeSum diffsum) putStrLn $ "Added: " <> show (newSum diffsum) @@ -148,13 +175,13 @@ printRSTDiffElem = printRSTElem . drop 2 printRST diff = do printRSTHeader "Updated" - mapM_ printRSTElem $ mapMaybe (showPkgDiff mode) [x | x@(PkgUpdate _ _) <- diff] + mapM_ printRSTElem $ mapMaybe (showPkgDiff modes) [x | x@(PkgUpdate _ _) <- diff] printRSTHeader "Downgraded" - mapM_ printRSTElem $ mapMaybe (showPkgDiff mode) [x | x@(PkgDowngrade _ _) <- diff] + mapM_ printRSTElem $ mapMaybe (showPkgDiff modes) [x | x@(PkgDowngrade _ _) <- diff] printRSTHeader "Added" - mapM_ printRSTDiffElem $ mapMaybe (showPkgDiff mode) [x | x@(PkgAdd _) <- diff] + mapM_ printRSTDiffElem $ mapMaybe (showPkgDiff modes) [x | x@(PkgAdd _) <- diff] printRSTHeader "Removed" - mapM_ printRSTDiffElem $ mapMaybe (showPkgDiff mode) [x | x@(PkgDel _) <- diff] + mapM_ printRSTDiffElem $ mapMaybe (showPkgDiff modes) [x | x@(PkgDel _) <- diff] getTrees :: String -> String -> IO ([NVRA],[NVRA]) getTrees t1 t2 = do @@ -183,16 +210,26 @@ Dir -> dirPackages True loc File -> filePackages loc Cmd -> cmdPackages $ words loc - let ps = map readNVRA $ filter (maybe (const True) (match . compile) mpattern) fs - return $ sort (nub ps) + let ps = map readNVRA $ maybe id (filter . (match . compile)) mpattern fs + return $ sort ps httpPackages :: Bool -> Manager -> String -> IO [String] httpPackages recurse mgr url = do exists <- httpExists mgr url - fs <- if exists - then map T.unpack . filter (\ f -> "/" `T.isSuffixOf` f || ".rpm" `T.isSuffixOf` f) <$> httpDirectory mgr url - else error' $ "Could not get " <> url - if (recurse || recursive) && all isDir fs then concatMapM (httpPackages False mgr) (map (url ) (filterSubdir fs)) else return $ filter (not . isDir) fs + fs <- + if exists + then do + mcontenttype <- lookup hContentType <$> httpFileHeaders mgr url + if mcontenttype == Just (B.pack "text/plain; charset=UTF-8") + then do + request <- parseRequest url + body <- responseBody <$> httpLbs request mgr + return $ (filter (".rpm" `isSuffixOf`) . map TL.unpack . TL.lines . decodeUtf8) body + else map T.unpack . filter (\f -> "/" `T.isSuffixOf` f || ".rpm" `T.isSuffixOf` f) <$> httpDirectory mgr url + else error' $ "Could not get " <> url + if (recurse || recursive) && all isDir fs + then concatMapM (httpPackages False mgr) (map (url ) (filterSubdir fs)) + else return $ filter (not . isDir) fs filterSubdir :: [String] -> [String] filterSubdir fs = @@ -201,11 +238,12 @@ _ -> fs dirPackages recurse dir = do - -- can replace with listDirectory after dropping ghc7 - -- should really filter out ".rpm" though not common - fs <- sort . filter (".rpm" `isSuffixOf`) <$> getDirectoryContents dir + fs <- map (dir ) . sort <$> listDirectory dir alldirs <- mapM doesDirectoryExist fs - if (recurse || recursive) && and alldirs then concatMapM (dirPackages False) (map (dir ) (filterSubdir fs)) else return $ filter (not . isDir) fs + if (recurse || recursive) && and alldirs + then concatMapM (dirPackages False) (filterSubdir fs) + -- FIXME strictly should limit to files + else return $ filter (".rpm" `isSuffixOf`) fs isDir = ("/" `isSuffixOf`) @@ -219,26 +257,22 @@ kojiPackages (tag, kojiUrl) = map Koji.kbNvr <$> Koji.kojiListTaggedBuilds kojiUrl True tag -isDefault :: Mode -> Bool -isDefault m = m `elem` [AutoSummary, NoSummary, ShowSummary, RST] - -showPkgDiff :: Mode -> RPMPkgDiff -> Maybe String -showPkgDiff mode diff = - case (mode,diff) of - (Added, PkgAdd p) -> Just $ showNVRA p - (Deleted, PkgDel p) -> Just $ showNVRA p - (Updated, PkgUpdate p1 p2) -> Just $ showPkgChange p1 p2 - (Updated, PkgArch p1 p2) -> Just $ showArchChange p1 p2 - (Downgraded, PkgDowngrade p1 p2) -> Just $ showPkgChange p1 p2 - _ -> if isDefault mode - then case diff of - PkgAdd p -> Just $ "+ " <> showNVRA p - PkgDel p -> Just $ "- " <> showNVRA p - PkgUpdate p1 p2 -> Just $ showPkgChange p1 p2 - PkgDowngrade p1 p2 -> Just $ "~ " <> showPkgChange p1 p2 - PkgArch p1 p2 -> Just $ "! " <> showArchChange p1 p2 - else Nothing +showPkgDiff :: [Mode] -> RPMPkgDiff -> Maybe String +showPkgDiff modes diff = + case diff of + PkgAdd p -> maybeShowDiff Added "+ " $ showNVRA p + PkgDel p -> maybeShowDiff Deleted "- " $ showNVRA p + PkgUpdate p1 p2 -> maybeShowDiff Updated "" $ showPkgChange p1 p2 + PkgArch p1 p2 -> maybeShowDiff Updated "" $ showArchChange p1 p2 + PkgDowngrade p1 p2 -> maybeShowDiff Downgraded "~ " $ showPkgChange p1 p2 where + maybeShowDiff :: Mode -> String -> String -> Maybe String + maybeShowDiff m prefix ds + | null modes || (m `elem` modes && length modes > 1) + = Just $ prefix ++ ds + | modes == [m] = Just ds + | otherwise = Nothing + showPkgChange :: NVRA -> NVRA -> String showPkgChange p p' = showPkgIdent p <> ": " <> showPkgVerRel p <> " -> " <> showPkgVerRel p' @@ -255,8 +289,8 @@ emptyDS :: DiffSum emptyDS = DS 0 0 0 0 0 -summary :: [RPMPkgDiff] -> DiffSum -summary = +changesummary :: [RPMPkgDiff] -> DiffSum +changesummary = foldl' countDiff emptyDS where countDiff :: DiffSum -> RPMPkgDiff -> DiffSum diff --git a/test/tests.hs b/test/tests.hs new file mode 100644 --- /dev/null +++ b/test/tests.hs @@ -0,0 +1,23 @@ +import SimpleCmd +import System.IO + +program :: [String] -> IO () +program test = do + timeIO $ cmdLog "pkgtreediff" test + putStrLn "" + +tests :: [[String]] +tests = + [ + ["-r", "test/files/35", "test/files/36"] + -- FIXME until native repoquerying use dnf-repo + , ["--ignore-release", + "dnf-repo -q -d * -u http://download.fedoraproject.org/pub/fedora/linux/releases/35/Everything/x86_64/os/ repoquery --qf %{name}-%{version}-%{release}.%{arch}", + "dnf-repo -q -d * -u http://download.fedoraproject.org/pub/fedora/linux/releases/36/Everything/x86_64/os/ repoquery --qf %{name}-%{version}-%{release}.%{arch}"] + ] + +main :: IO () +main = do + hSetBuffering stdout NoBuffering + mapM_ program tests + putStrLn $ "\n" ++ show (length tests) ++ " command tests run"