diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,13 @@ # Revision history for fedora-repoquery +## 0.6 (2024-07-16) +- defaults to no datestamp output: --time replaces --no-check +- further speed improvements by reducing http checks +- map latest pending release version to fedora-rawhide +- add --dynamic redirect to not re-use first mirror redirect +- update c8s url to Centos Vault +- drop dep on utility-ht just for spanJust + ## 0.5 (2024-07-02) - allow multiple release args - with no release arg use system yum repos diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ `$ fdrq rawhide firefox` ``` -firefox-127.0.2-1.fc41.x86_64 (fedora-rawhide) +firefox-128.0-2.fc41.x86_64 (fedora-rawhide) ``` `$ fdrq 40 --requires filesystem` @@ -27,26 +27,32 @@ `$ fdrq c10 bash` ``` -bash-5.2.26-3.el10.x86_64 (c10s-BaseOS) +bash-5.2.26-4.el10.x86_64 (c10s-BaseOS) ``` `$ fdrq eln kernel` ``` -kernel-6.10.0-0.rc6.50.eln141.x86_64 (eln-BaseOS) +kernel-6.10.0-64.eln141.x86_64 (eln-BaseOS) ``` etc. -The above output is generated with mdsh which suppresses the stderr -that includes mirror repo urls with compose timestamps, like this: +With the --time option the timestamp of repos is also shown: ``` -$ fdrq rawhide fedrq -2024-05-23 16:41:58 +08 +$ 2024-07-16 19:45:44 +08 -fedrq-1.1.0-1.fc41.noarch (fedora-rawhide) +fedrq-1.1.0-3.fc41.noarch (fedora-rawhide) ``` +The repo timestamp(s) can also be output with no args after +the release version: +`$ fdrq 40` +``` +2024-04-20 02:22:34 +08 +2024-07-16 12:29:35 +08 +``` + Also note that dnf5 currently still outputs repo update messages to stdout but it is a lot faster than dnf4. @@ -54,20 +60,20 @@ `$ fdrq --version` ``` -0.5 +0.6 ``` `$ fdrq --help` ``` fedora-repoquery tool for querying Fedora repos for packages. -Usage: fdrq [--version] [-4|--dnf4] [(-q|--quiet) | (-v|--verbose)] [--no-check] - [-K|--koji] [--devel-channel | --test-channel] +Usage: fdrq [--version] [-4|--dnf4] [(-q|--quiet) | (-v|--verbose)] [--dynamic] + [-T|--time] [-K|--koji] [--devel-channel | --test-channel] [(-m|--mirror URL) | (-D|--dl)] [(-s|--source) | (-A|--all-archs) | [-a|--arch ARCH]] [-t|--testing] [-d|--debug] ((-z|--cache-size) | (-e|--cache-clean-empty) | (-l|--list) | - RELEASE... [REPOQUERY_OPTS]... [PACKAGE]...) + [RELEASE]... [REPOQUERY_OPTS]... [PACKAGE]...) where RELEASE is {fN or N (fedora), 'rawhide', epelN, epelN-next, cN (centos stream), 'eln'}, with N the release version number. @@ -79,7 +85,8 @@ -4,--dnf4 Use dnf4 instead of dnf5 (if available) -q,--quiet Avoid output to stderr -v,--verbose Show stderr from dnf repoquery - --no-check Skip http repo url checks + --dynamic Redirect each HTTP through mirror + -T,--time Show time-stamp of repos -K,--koji Use Koji buildroot --devel-channel Use eln development compose --test-channel Use eln test compose [default: production] diff --git a/fedora-repoquery.cabal b/fedora-repoquery.cabal --- a/fedora-repoquery.cabal +++ b/fedora-repoquery.cabal @@ -1,5 +1,5 @@ name: fedora-repoquery -version: 0.5 +version: 0.6 synopsis: Fedora repoquery tool description: A CLI tool for repoquerying Fedora packages: @@ -59,7 +59,6 @@ simple-cmd, simple-cmd-args, time, - utility-ht >= 0.0.16, xdg-basedir default-language: Haskell2010 diff --git a/src/BodhiRelease.hs b/src/BodhiRelease.hs --- a/src/BodhiRelease.hs +++ b/src/BodhiRelease.hs @@ -4,7 +4,8 @@ BodhiRelease (..), activeFedoraRelease, activeFedoraReleases, - pendingFedoraRelease + pendingFedoraRelease, + rawhideFedoraRelease ) where @@ -25,8 +26,6 @@ -- Left is oldest active version activeFedoraRelease :: Natural -> IO (Either Natural BodhiRelease) --- F37 not archived yet: https://pagure.io/releng/issue/12124 -activeFedoraRelease 37 = return $ Right $ Release "37" "current" "f37" activeFedoraRelease n = do active <- activeFedoraReleases when (null active) $ error' "failed to find active releases with Bodhi API" @@ -53,3 +52,9 @@ case eactive of Left _ -> False Right rel -> releaseState rel == "pending" + +rawhideFedoraRelease :: IO Natural +rawhideFedoraRelease = do + actives <- activeFedoraReleases + let pending = map releaseVersion (filter (\r -> releaseState r == "pending") actives) + return $ read $ maximum (L.delete "eln" pending) diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -14,7 +14,8 @@ ) #endif import Control.Monad (forM_) -import Data.List.HT (spanJust) +import Data.Either (partitionEithers) +import Data.Tuple (swap) #if !MIN_VERSION_simple_cmd_args(0,1,7) import Options.Applicative (eitherReader, maybeReader, ReadM) #endif @@ -29,7 +30,7 @@ import Query (repoqueryCmd) import ShowRelease (showReleaseCmd, downloadServer) import Types (Channel(..), Mirror(..), Release (System), RepoSource(..), Verbosity(..), - readRelease) + eitherRelease) data Command = Query [String] | CacheSize | CacheEmpties | List @@ -44,7 +45,8 @@ <$> switchWith '4' "dnf4" "Use dnf4 instead of dnf5 (if available)" <*> (flagWith' Quiet 'q' "quiet" "Avoid output to stderr" <|> flagWith Normal Verbose 'v' "verbose" "Show stderr from dnf repoquery") - <*> switchLongWith "no-check" "Skip http repo url checks" + <*> switchLongWith "dynamic" "Redirect each HTTP through mirror" + <*> switchWith 'T' "time" "Show time-stamp of repos" <*> (RepoSource <$> switchWith 'K' "koji" "Use Koji buildroot" <*> (flagLongWith' ChanDevel "devel-channel" "Use eln development compose" <|> @@ -59,23 +61,24 @@ <*> (flagWith' CacheSize 'z' "cache-size" "Show total dnf repo metadata cache disksize" <|> flagWith' CacheEmpties 'e' "cache-clean-empty" "Remove empty dnf caches" <|> flagWith' List 'l' "list" "List Fedora versions" - <|> Query <$> some (strArg "RELEASE... [REPOQUERY_OPTS]... [PACKAGE]...")) + <|> Query <$> some (strArg "[RELEASE]... [REPOQUERY_OPTS]... [PACKAGE]...")) -runMain :: Arch -> Bool -> Verbosity -> Bool -> RepoSource -> [Arch] -> Bool -> Bool - -> Command -> IO () -runMain sysarch dnf4 verbose nourlchecks reposource archs testing debug command = do +runMain :: Arch -> Bool -> Verbosity -> Bool -> Bool -> RepoSource -> [Arch] + -> Bool -> Bool -> Command -> IO () +runMain sysarch dnf4 verbose dynredir time reposource archs testing debug command = do case command of CacheSize -> cacheSize CacheEmpties -> cleanEmptyCaches List -> listVersionsCmd Query relargs -> - let (releases,args) = spanJust readRelease relargs + -- spanJust from utility-ht nicer but this gets us enough + let (releases,args) = swap $ partitionEithers $ map eitherRelease relargs in forM_ (if null releases then [System] else releases) $ \release -> if null args then if null archs - then showReleaseCmd debug reposource release sysarch Nothing testing - else forM_ archs $ \arch -> showReleaseCmd debug reposource release sysarch (Just arch) testing + then showReleaseCmd debug dynredir reposource release sysarch Nothing testing + else forM_ archs $ \arch -> showReleaseCmd debug dynredir reposource release sysarch (Just arch) testing else let multiple = length releases > 1 || length archs > 1 - in repoqueryCmd dnf4 debug verbose (nourlchecks || multiple) release reposource sysarch archs testing args + in repoqueryCmd dnf4 debug verbose multiple dynredir time release reposource sysarch archs testing args diff --git a/src/Query.hs b/src/Query.hs --- a/src/Query.hs +++ b/src/Query.hs @@ -36,16 +36,15 @@ "supplements" ] --- FIXME --no-redirect? -- FIXME error if no testing repo -repoqueryCmd :: Bool -> Bool -> Verbosity -> Bool -> Release -> RepoSource - -> Arch -> [Arch] -> Bool -> [String] -> IO () -repoqueryCmd dnf4 debug verbose nourlcheck release reposource sysarch archs testing args = do +repoqueryCmd :: Bool -> Bool -> Verbosity -> Bool -> Bool -> Bool -> Release + -> RepoSource -> Arch -> [Arch] -> Bool -> [String] -> IO () +repoqueryCmd dnf4 debug verbose multiple dynredir checkdate release reposource sysarch archs testing args = do forM_ (if null archs then [sysarch] else archs) $ \arch -> do repoConfigs <- if release == System then return [] - else showRelease debug verbose True nourlcheck reposource release sysarch (Just arch) testing + else showRelease debug dynredir True checkdate reposource release sysarch (Just arch) testing let qfAllowed = not $ any (`elem` ["-i","--info","-l","--list","-s","--source","--nvr","--nevra","--envra","--qf","--queryformat", "--changelog"] ++ pkgAttrsOptions) args -- dnf5 writes repo update output to stdout -- https://github.com/rpm-software-management/dnf5/issues/1361 @@ -72,7 +71,7 @@ warning $ unwords $ ('\n' : takeBaseName dnf) : map show cmdargs res <- cmdLines dnf cmdargs unless (null res) $ do - unless (verbose == Quiet || nourlcheck || release == System) $ warning "" + unless (not checkdate || release == System || multiple) $ warning "" putStrLn $ L.intercalate "\n" res where tweakedArgs dnf5 = diff --git a/src/ShowRelease.hs b/src/ShowRelease.hs --- a/src/ShowRelease.hs +++ b/src/ShowRelease.hs @@ -14,8 +14,8 @@ import Data.List.Extra import Data.Maybe (fromMaybe) import Data.Time.LocalTime (utcToLocalZonedTime) -import Network.HTTP.Directory (httpExists, httpLastModified,httpManager, - httpRedirect, Manager, trailingSlash) +import Network.HTTP.Directory (httpLastModified, httpManager, + httpRedirect, Manager) import SimpleCmd (error', (+-+)) import Text.Regex (mkRegex, subRegex) @@ -25,17 +25,17 @@ import Types import URL -showReleaseCmd :: Bool -> RepoSource -> Release -> Arch -> Maybe Arch -> Bool - -> IO () -showReleaseCmd debug reposource release sysarch march testing = - void $ showRelease debug Normal False False reposource release sysarch march testing +showReleaseCmd :: Bool -> Bool -> RepoSource -> Release -> Arch -> Maybe Arch + -> Bool -> IO () +showReleaseCmd debug dynredir reposource release sysarch march testing = + void $ showRelease debug dynredir False True reposource release sysarch march testing -showRelease :: Bool -> Verbosity -> Bool -> Bool -> RepoSource -> Release - -> Arch -> Maybe Arch -> Bool -> IO [(String, URL)] -showRelease debug verbose warn nourlcheck reposource@(RepoSource koji _chan _mirror) release sysarch march testing = do +showRelease :: Bool -> Bool -> Bool -> Bool -> RepoSource + -> Release -> Arch -> Maybe Arch -> Bool -> IO [(String, URL)] +showRelease debug dynredir warn checkdate reposource@(RepoSource koji _chan _mirror) release sysarch march testing = do mgr <- httpManager let arch = fromMaybe sysarch march - (url,path) <- getURL debug mgr reposource release arch + (url,path) <- getURL debug dynredir mgr reposource release arch let urlpath = url +//+ path when debug $ print $ renderUrl urlpath (basicrepos,morerepos) <- @@ -61,50 +61,43 @@ []) EPELNext _n -> return ([("epelnext",urlpath)],[]) System -> error' "showRelease: system unsupported" + rawhide <- rawhideFedoraRelease let basicrepourls = - map (repoConfigArgs reposource sysarch march release) basicrepos + map (repoConfigArgs reposource sysarch march rawhide release) basicrepos morerepourls = - map (repoConfigArgs reposource sysarch march release) morerepos + map (repoConfigArgs reposource sysarch march rawhide release) morerepos forM_ basicrepourls $ \(reponame,(url',path')) -> do let baserepo = url' +//+ path' when debug $ print $ renderUrl baserepo - unless nourlcheck $ do - ok <- httpExists mgr $ trailingSlash $ renderUrl baserepo - if ok - then do - unless (verbose == Quiet || release == System) $ do - mtime <- do - let composeinfo = - if koji - then url' +//+ ["repo.json"] - else - case release of - Centos 10 -> url' +//+ ["metadata","composeinfo.json"] - Centos _ -> url' +//+ ["COMPOSE_ID"] -- ["metadata","composeinfo.json"] - ELN -> url' +//+ ["metadata","composeinfo.json"] - EPEL _ -> url' +//+ ["Everything", "state"] - EPELNext _ -> url' +//+ ["Everything", "state"] - Fedora _ -> url' +//+ - if "updates" `isSuffixOf` reponame || - "updates-testing" `isSuffixOf` reponame - then ["Everything", "state"] - else ["COMPOSE_ID"] - Rawhide -> url' +//+ ["COMPOSE_ID"] - System -> error' "system not supported" - when debug $ print $ renderUrl composeinfo - exists <- httpExists mgr (renderUrl composeinfo) - if exists - then httpLastModified mgr (renderUrl composeinfo) - else return Nothing - whenJust mtime $ \utc -> do - date <- utcToLocalZonedTime utc - (if warn then warning else putStrLn) $ show date +-+ "<" ++ renderUrl url' ++ ">" - else - error' $ renderUrl baserepo +-+ "not found" + unless (not checkdate || release == System) $ do + let composeinfo = + if koji + then url' +//+ ["repo.json"] + else + case release of + Centos 10 -> url' +//+ ["metadata","composeinfo.json"] + Centos _ -> url' +//+ ["COMPOSE_ID"] -- ["metadata","composeinfo.json"] + ELN -> url' +//+ ["metadata","composeinfo.json"] + EPEL _ -> url' +//+ ["Everything", "state"] + EPELNext _ -> url' +//+ ["Everything", "state"] + Fedora _ -> url' +//+ + if "updates" `isInfixOf` reponame || + "updates-testing" `isInfixOf` reponame + then ["Everything", "state"] + else ["COMPOSE_ID"] + Rawhide -> url' +//+ ["COMPOSE_ID"] + System -> error' "system not supported" + let composeUrl = renderUrl composeinfo + when debug $ print composeUrl + mtime <- httpLastModified mgr composeUrl + whenJust mtime $ \utc -> do + date <- utcToLocalZonedTime utc + (if warn then warning else putStrLn) $ show date +-+ "<" ++ renderUrl url' ++ ">" return $ map (fmap (uncurry (+//+))) $ basicrepourls ++ morerepourls -getURL :: Bool -> Manager -> RepoSource -> Release -> Arch -> IO (URL,[String]) -getURL debug mgr reposource@(RepoSource koji chan _mirror) release arch = +getURL :: Bool -> Bool -> Manager -> RepoSource -> Release -> Arch + -> IO (URL,[String]) +getURL debug dynredir mgr reposource@(RepoSource koji chan _mirror) release arch = case release of Centos n -> case n of @@ -120,15 +113,15 @@ then "https://odcs.stream.centos.org" else "https://mirror.stream.centos.org/9-stream/" in return (url,[]) - 8 -> return (URL "http://mirror.centos.org/centos/8-stream/", []) - _ -> error' "old Centos is not supported yet" + 8 -> return (URL "http://vault.centos.org/8-stream/", []) + _ -> error' "old Centos is not supported" ELN -> return (URL "https://odcs.fedoraproject.org/composes", [channel chan, "latest-Fedora-ELN", "compose"]) EPEL n | n < 7 -> return (URL "https://archives.fedoraproject.org/pub/archive/epel", [show n]) - EPEL n -> getFedoraServer debug mgr reposource ["epel"] [show n] - EPELNext n -> getFedoraServer debug mgr reposource ["epel","next"] [show n] + EPEL n -> getFedoraServer debug dynredir mgr reposource ["epel"] [show n] + EPELNext n -> getFedoraServer debug dynredir mgr reposource ["epel","next"] [show n] Fedora n -> do ebodhirelease <- activeFedoraRelease n case ebodhirelease of @@ -142,9 +135,9 @@ let pending = releaseState rel == "pending" rawhide = pending && releaseBranch rel == "rawhide" releasestr = if rawhide then "rawhide" else show n - in getFedoraServer debug mgr reposource fedoraTop + in getFedoraServer debug dynredir mgr reposource fedoraTop [if pending then "development" else "releases", releasestr] - Rawhide -> getFedoraServer debug mgr reposource fedoraTop ["development", "rawhide"] + Rawhide -> getFedoraServer debug dynredir mgr reposource fedoraTop ["development", "rawhide"] System -> error' "getURL: system unsupported" where fedoraTop = @@ -153,10 +146,10 @@ then ["fedora-secondary"] else ["fedora", "linux"] -repoConfigArgs :: RepoSource -> Arch -> Maybe Arch -> Release -> (String,URL) - -> (String,(URL,[String])) +repoConfigArgs :: RepoSource -> Arch -> Maybe Arch -> Natural -> Release + -> (String,URL) -> (String,(URL,[String])) -- non-koji -repoConfigArgs (RepoSource False _chan mirror) sysarch march release (repo,url) = +repoConfigArgs (RepoSource False _chan mirror) sysarch march rawhide release (repo,url) = let arch = fromMaybe sysarch march archsuffix = if arch == sysarch then "" else "-" ++ showArch arch reponame = repoVersion ++ archsuffix ++ @@ -178,7 +171,7 @@ where repoVersion :: String repoVersion = - if release == Rawhide + if release `elem` [Rawhide, Fedora rawhide] then "fedora-rawhide" else show release ++ @@ -189,7 +182,7 @@ Fedora _ | repo /= "releases" -> '-':repo _ -> "" -- koji -repoConfigArgs (RepoSource True _chan _mirror) sysarch march release (repo,url) = +repoConfigArgs (RepoSource True _chan _mirror) sysarch march _rawhide release (repo,url) = let (compose,path) = case release of Rawhide -> (["repos", show release, "latest"],"") @@ -203,9 +196,9 @@ -- reponame = repo ++ "-Centos-" ++ show release ++ "-Stream" ++ "-" ++ show chan ++ if arch == sysarch then "" else "-" ++ showArch arch -- in (reponame, (url +//+ compose, [path, showArch arch, "os/"])) -getFedoraServer :: Bool -> Manager -> RepoSource -> [String] -> [String] +getFedoraServer :: Bool -> Bool -> Manager -> RepoSource -> [String] -> [String] -> IO (URL,[String]) -getFedoraServer debug mgr (RepoSource koji _ mirror) top path = +getFedoraServer debug dynredir mgr (RepoSource koji _ mirror) top path = if koji then return (URL "https://kojipkgs.fedoraproject.org",[]) else @@ -213,10 +206,14 @@ DownloadFpo -> do let url = URL downloadServer +//+ top ++ path rurl = renderUrl url - redir <- fmap B.unpack <$> httpRedirect mgr rurl + redir <- + if dynredir + then return Nothing + else fmap B.unpack <$> httpRedirect mgr rurl case redir of Nothing -> do - warning $ "no redirect for" +-+ rurl + unless dynredir $ + warning $ "no redirect for" +-+ rurl return (URL downloadServer +//+ top,path) Just actual -> do when debug $ do diff --git a/src/Types.hs b/src/Types.hs --- a/src/Types.hs +++ b/src/Types.hs @@ -8,7 +8,7 @@ channel, Verbosity(..), Release(..), - readRelease + eitherRelease ) where import Data.Char (isDigit) @@ -48,7 +48,6 @@ -- | Read a Release name, otherwise return an error message eitherRelease :: String -> Either String Release --- FIXME: alias "raw" or "r"?? eitherRelease "rawhide" = Right Rawhide -- FIXME add proper parsing: eitherRelease "epel8-next" = Right $ EPELNext 8 @@ -65,14 +64,7 @@ LT -> Centos r EQ -> ELN GT -> Fedora r -eitherRelease cs = Left $ cs ++ " is not a known os release" - --- | Read a Fedora Release name -readRelease :: String -> Maybe Release -readRelease bs = - case eitherRelease bs of - Left _ -> Nothing - Right br -> Just br +eitherRelease cs = Left cs instance Show Release where show Rawhide = "rawhide" diff --git a/test/tests.hs b/test/tests.hs --- a/test/tests.hs +++ b/test/tests.hs @@ -9,7 +9,7 @@ let debug = False (ok, out, err) <- cmdFull "fdrq" (["-d" | debug] ++ args ++ maybeToList mpkg) "" if null err - then unless (isNothing mpkg) $ putStrLn "stderr empty" + then unless (isNothing mpkg || "-T" `notElem` args) $ putStrLn "stderr empty" else putStrLn err if null out then unless (isNothing mpkg) $ error' "no output" @@ -31,9 +31,9 @@ ,([current], Nothing) ,([previous], Nothing) ,([prevprev], Nothing) - ,(["rawhide"], Just "coreutils") - ,([rawhide], Just "coreutils") - ,([current], Just "gtk4") + ,(["-T", "rawhide"], Just "coreutils") + ,(["--dynamic", rawhide], Just "coreutils") + ,(["-T", current], Just "gtk4") ,([previous], Just "bash") ,([prevprev], Just "fontconfig") ,(["-t", previous], Just "podman")