diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,9 @@ # Revision history for fedora-repoquery +## 0.7 (2024-07-18) +- use curl library for HTTP instead of http-directory +- rename the executable back to fedora-repoquery to avoid confusion with fedrq + ## 0.6 (2024-07-16) - defaults to no datestamp output: --time replaces --no-check - further speed improvements by reducing http checks diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,79 +1,99 @@ # fedora-repoquery -A fedora release version wrapper of dnf repoquery, -which caches repodata separately per release. +A dnf wrapper to repoquery different Fedora releases, +caching repodata separately per release. +It also supports EPEL and Centos Stream repos. ## Usage Usage examples: -`$ fdrq rawhide firefox` +`$ fedora-repoquery rawhide firefox` ``` firefox-128.0-2.fc41.x86_64 (fedora-rawhide) ``` -`$ fdrq 40 --requires filesystem` +`$ fedora-repoquery 40 --requires filesystem` ``` setup ``` -`$ fdrq epel9 ghc` +`$ fedora-repoquery epel9 ghc` ``` ghc-8.10.7-116.el9.x86_64 (epel9) ``` -`$ fdrq c10 bash` +`$ fedora-repoquery c10 bash` ``` bash-5.2.26-4.el10.x86_64 (c10s-BaseOS) ``` -`$ fdrq eln kernel` +`$ fedora-repoquery eln kernel` ``` -kernel-6.10.0-64.eln141.x86_64 (eln-BaseOS) +kernel-6.11.0-0.rc0.20240716gitd67978318827.2.eln141.x86_64 (eln-BaseOS) ``` -etc. +Without a release argument, the system yum repo configuration is used: -With the --time option the timestamp of repos is also shown: +`$ fedora-repoquery pandoc` + ``` -$ 2024-07-16 19:45:44 +08 +pandoc-3.1.3-29.fc41.x86_64 (rawhide) +``` +Use the --time (-T) option to display repo timestamps: +``` +$ fedora-repoquery -T 41 fedrq +2024-07-16 19:45:44 +08 + fedrq-1.1.0-3.fc41.noarch (fedora-rawhide) ``` -The repo timestamp(s) can also be output with no args after -the release version: -`$ fdrq 40` +Repo timestamp(s) are output when there are no query args +after the release version: + +`$ fedora-repoquery 40` + ``` -2024-04-20 02:22:34 +08 -2024-07-16 12:29:35 +08 +2024-04-20 02:22:34 +08 +2024-07-18 12:15:13 +08 ``` -Also note that dnf5 currently still outputs repo update messages to stdout -but it is a lot faster than dnf4. +One can also query multiple releases (or arch's): +`$ fedora-repoquery 40 41 python3 | grep x86_64` +``` +python3-3.12.2-2.fc40.x86_64 (f40) +python3-3.12.4-1.fc40.x86_64 (f40-updates) +python3-3.13.0~b3-2.fc41.x86_64 (fedora-rawhide) +``` + + ## Help -`$ fdrq --version` +`$ fedora-repoquery --version` ``` -0.6 +0.7 ``` -`$ fdrq --help` +`$ fedora-repoquery --help` + ``` fedora-repoquery tool for querying Fedora repos for packages. -Usage: fdrq [--version] [-4|--dnf4] [(-q|--quiet) | (-v|--verbose)] [--dynamic] - [-T|--time] [-K|--koji] [--devel-channel | --test-channel] - [(-m|--mirror URL) | (-D|--dl)] - [(-s|--source) | (-A|--all-archs) | [-a|--arch ARCH]] [-t|--testing] - [-d|--debug] - ((-z|--cache-size) | (-e|--cache-clean-empty) | (-l|--list) | - [RELEASE]... [REPOQUERY_OPTS]... [PACKAGE]...) +Usage: fedora-repoquery [--version] [-4|--dnf4] [(-q|--quiet) | (-v|--verbose)] + [--dynamic] [-T|--time] [-K|--koji] + [--devel-channel | --test-channel] + [(-m|--mirror URL) | (-D|--dl)] + [(-s|--source) | (-A|--all-archs) | [-a|--arch ARCH]] + [-t|--testing] [-d|--debug] + ((-z|--cache-size) | (-e|--cache-clean-empty) | + (-l|--list) | + [RELEASE]... [REPOQUERY_OPTS]... [PACKAGE]...) where RELEASE is {fN or N (fedora), 'rawhide', epelN, epelN-next, cN (centos stream), 'eln'}, with N the release version number. diff --git a/fedora-repoquery.cabal b/fedora-repoquery.cabal --- a/fedora-repoquery.cabal +++ b/fedora-repoquery.cabal @@ -1,15 +1,15 @@ name: fedora-repoquery -version: 0.6 -synopsis: Fedora repoquery tool +version: 0.7 +synopsis: Fedora release repos package query tool description: - A CLI tool for repoquerying Fedora packages: + A CLI tool for repoquerying Fedora and Centos Stream release packages: by default it displays their version and repo location - with compose timestamp + and optionally the repo compose timestamps license: GPL-3 license-file: COPYING -author: Jens Petersen -maintainer: Jens Petersen -copyright: 2021-2024 Jens Petersen +author: Jens Petersen +maintainer: Jens Petersen +copyright: 2021-2024 Jens Petersen category: Utility homepage: https://github.com/juhp/fedora-repoquery bug-reports: https://github.com/juhp/fedora-repoquery/issues @@ -30,7 +30,7 @@ type: git location: https://github.com/juhp/fedora-repoquery.git -executable fdrq +executable fedora-repoquery main-is: Main.hs autogen-modules: Paths_fedora_repoquery other-modules: Paths_fedora_repoquery @@ -46,18 +46,18 @@ hs-source-dirs: src build-depends: base < 5, bodhi, - bytestring, cached-json-file, + case-insensitive, + curl, directory, extra, filepath, Glob, - http-client, - http-directory >= 0.1.4 && (< 0.1.6 || >= 0.1.8), regex-compat, safe, simple-cmd, simple-cmd-args, + text, time, xdg-basedir @@ -108,4 +108,4 @@ ghc-options: -Wall build-depends: base >= 4 && < 5 , simple-cmd - build-tool-depends: fedora-repoquery:fdrq + build-tool-depends: fedora-repoquery:fedora-repoquery diff --git a/src/Common.hs b/src/Common.hs --- a/src/Common.hs +++ b/src/Common.hs @@ -1,17 +1,11 @@ {-# LANGUAGE CPP #-} module Common ( - Manager, warning, -#if !MIN_VERSION_http_directory(0,1,6) noTrailingSlash -#endif ) where -import Network.HTTP.Client (Manager) -#if !MIN_VERSION_http_directory(0,1,6) import qualified Data.Text as T -#endif #if MIN_VERSION_simple_cmd(0,2,0) import SimpleCmd (warning) @@ -22,7 +16,5 @@ warning = hPutStrLn stderr #endif -#if !MIN_VERSION_http_directory(0,1,6) noTrailingSlash :: T.Text -> T.Text noTrailingSlash = T.dropWhileEnd (== '/') -#endif diff --git a/src/Query.hs b/src/Query.hs --- a/src/Query.hs +++ b/src/Query.hs @@ -17,7 +17,7 @@ import Common (warning) import ShowRelease (showRelease) import Types -import URL (URL, renderUrl) +import URL (FileDir(Dir), URL, renderUrl) -- from dnf5 repoquery.cpp pkg_attrs_options pkgAttrsOptions :: [String] @@ -95,4 +95,4 @@ renderRepoConfig :: (String, URL) -> [String] renderRepoConfig (name, url) = - ["--repofrompath=" ++ name ++ "," ++ renderUrl url, "--repo=" ++ name, "--setopt=" ++ name ++ ".metadata_expire=6h" ] + ["--repofrompath=" ++ name ++ "," ++ renderUrl Dir url, "--repo=" ++ name, "--setopt=" ++ name ++ ".metadata_expire=6h" ] diff --git a/src/ShowRelease.hs b/src/ShowRelease.hs --- a/src/ShowRelease.hs +++ b/src/ShowRelease.hs @@ -10,12 +10,13 @@ where import Control.Monad.Extra (forM_, unless, void, when, whenJust) -import qualified Data.ByteString.Char8 as B +import Data.Bifunctor (first) +import qualified Data.CaseInsensitive as CI import Data.List.Extra import Data.Maybe (fromMaybe) +import Data.Time.Format (defaultTimeLocale, parseTimeM, rfc822DateFormat) import Data.Time.LocalTime (utcToLocalZonedTime) -import Network.HTTP.Directory (httpLastModified, httpManager, - httpRedirect, Manager) +import Network.Curl (curlHead, CurlOption(..)) import SimpleCmd (error', (+-+)) import Text.Regex (mkRegex, subRegex) @@ -33,11 +34,10 @@ showRelease :: Bool -> Bool -> Bool -> Bool -> RepoSource -> Release -> Arch -> Maybe Arch -> Bool -> IO [(String, URL)] showRelease debug dynredir warn checkdate reposource@(RepoSource koji _chan _mirror) release sysarch march testing = do - mgr <- httpManager let arch = fromMaybe sysarch march - (url,path) <- getURL debug dynredir mgr reposource release arch + (url,path) <- getURL debug dynredir reposource release arch let urlpath = url +//+ path - when debug $ print $ renderUrl urlpath + when debug $ print $ renderUrl Dir urlpath (basicrepos,morerepos) <- case release of -- RepoKoji -> ["koji-fedora"] @@ -68,36 +68,37 @@ map (repoConfigArgs reposource sysarch march rawhide release) morerepos forM_ basicrepourls $ \(reponame,(url',path')) -> do let baserepo = url' +//+ path' - when debug $ print $ renderUrl baserepo + when debug $ print $ renderUrl Dir baserepo unless (not checkdate || release == System) $ do let composeinfo = + url' +//+ if koji - then url' +//+ ["repo.json"] + then ["repo.json"] else case release of - Centos 10 -> url' +//+ ["metadata","composeinfo.json"] - Centos _ -> url' +//+ ["COMPOSE_ID"] -- ["metadata","composeinfo.json"] - ELN -> url' +//+ ["metadata","composeinfo.json"] - EPEL _ -> url' +//+ ["Everything", "state"] - EPELNext _ -> url' +//+ ["Everything", "state"] - Fedora _ -> url' +//+ - if "updates" `isInfixOf` reponame || + Centos 10 -> ["metadata","composeinfo.json"] + Centos _ -> ["COMPOSE_ID"] -- ["metadata","composeinfo.json"] + ELN -> ["metadata","composeinfo.json"] + EPEL _ -> ["Everything", "state"] + EPELNext _ -> ["Everything", "state"] + Fedora _ -> if "updates" `isInfixOf` reponame || "updates-testing" `isInfixOf` reponame then ["Everything", "state"] else ["COMPOSE_ID"] - Rawhide -> url' +//+ ["COMPOSE_ID"] + Rawhide -> ["COMPOSE_ID"] System -> error' "system not supported" - let composeUrl = renderUrl composeinfo + let composeUrl = renderUrl File composeinfo when debug $ print composeUrl - mtime <- httpLastModified mgr composeUrl - whenJust mtime $ \utc -> do + mtimestr <- curlGetHeader "Last-Modified" composeUrl + whenJust mtimestr $ \timestr -> do + utc <- parseTimeM False defaultTimeLocale rfc822DateFormat timestr date <- utcToLocalZonedTime utc - (if warn then warning else putStrLn) $ show date +-+ "<" ++ renderUrl url' ++ ">" + (if warn then warning else putStrLn) $ show date +-+ "<" ++ renderUrl Dir url' ++ ">" return $ map (fmap (uncurry (+//+))) $ basicrepourls ++ morerepourls -getURL :: Bool -> Bool -> Manager -> RepoSource -> Release -> Arch +getURL :: Bool -> Bool -> RepoSource -> Release -> Arch -> IO (URL,[String]) -getURL debug dynredir mgr reposource@(RepoSource koji chan _mirror) release arch = +getURL debug dynredir reposource@(RepoSource koji chan _mirror) release arch = case release of Centos n -> case n of @@ -120,8 +121,8 @@ EPEL n | n < 7 -> return (URL "https://archives.fedoraproject.org/pub/archive/epel", [show n]) - EPEL n -> getFedoraServer debug dynredir mgr reposource ["epel"] [show n] - EPELNext n -> getFedoraServer debug dynredir mgr reposource ["epel","next"] [show n] + EPEL n -> getFedoraServer debug dynredir reposource ["epel"] [show n] + EPELNext n -> getFedoraServer debug dynredir reposource ["epel","next"] [show n] Fedora n -> do ebodhirelease <- activeFedoraRelease n case ebodhirelease of @@ -135,9 +136,9 @@ let pending = releaseState rel == "pending" rawhide = pending && releaseBranch rel == "rawhide" releasestr = if rawhide then "rawhide" else show n - in getFedoraServer debug dynredir mgr reposource fedoraTop + in getFedoraServer debug dynredir reposource fedoraTop [if pending then "development" else "releases", releasestr] - Rawhide -> getFedoraServer debug dynredir mgr reposource fedoraTop ["development", "rawhide"] + Rawhide -> getFedoraServer debug dynredir reposource fedoraTop ["development", "rawhide"] System -> error' "getURL: system unsupported" where fedoraTop = @@ -196,24 +197,26 @@ -- reponame = repo ++ "-Centos-" ++ show release ++ "-Stream" ++ "-" ++ show chan ++ if arch == sysarch then "" else "-" ++ showArch arch -- in (reponame, (url +//+ compose, [path, showArch arch, "os/"])) -getFedoraServer :: Bool -> Bool -> Manager -> RepoSource -> [String] -> [String] +getFedoraServer :: Bool -> Bool -> RepoSource -> [String] -> [String] -> IO (URL,[String]) -getFedoraServer debug dynredir mgr (RepoSource koji _ mirror) top path = +getFedoraServer debug dynredir (RepoSource koji _ mirror) top path = if koji then return (URL "https://kojipkgs.fedoraproject.org",[]) else case mirror of DownloadFpo -> do let url = URL downloadServer +//+ top ++ path - rurl = renderUrl url + rurl = renderUrl Dir url + when debug $ print rurl redir <- if dynredir then return Nothing - else fmap B.unpack <$> httpRedirect mgr rurl + else curlGetHeader "location" rurl + when debug $ print redir case redir of Nothing -> do unless dynredir $ - warning $ "no redirect for" +-+ rurl + warning $ "slow redirection for" +-+ rurl return (URL downloadServer +//+ top,path) Just actual -> do when debug $ do @@ -232,3 +235,10 @@ downloadServer :: String downloadServer = "https://download.fedoraproject.org/pub" + +-- FIXME check CurlCode(CurlOK) status +curlGetHeader :: CI.CI String -> String -> IO (Maybe String) +curlGetHeader field url = do + (_status,headers) <- curlHead url [CurlConnectTimeoutMS 5000] + -- tail because curl leaves space before field value + return $ tail <$> lookup field (map (first CI.mk) headers) diff --git a/src/URL.hs b/src/URL.hs --- a/src/URL.hs +++ b/src/URL.hs @@ -4,27 +4,28 @@ (+//+), URL(..), renderUrl, + FileDir(..), removeSubpath ) where import Data.List.Extra (dropSuffix) -#if MIN_VERSION_http_directory(0,1,9) -import Network.HTTP.Directory((+/+)) -#endif - newtype URL = URL String deriving Eq -renderUrl :: URL -> String -renderUrl (URL url) = url +data FileDir = File | Dir + deriving Eq +renderUrl :: FileDir -> URL -> String +renderUrl Dir (URL url) = trailingSlash url +renderUrl File (URL url) = url + foldSlash :: [String] -> String foldSlash = foldr (+/+) "" infixr 5 +//+ (+//+) :: URL -> [String] -> URL -(URL base) +//+ rel = +URL base +//+ rel = URL $ base +/+ foldSlash rel -- piecesToPath :: [String] -> String @@ -36,7 +37,7 @@ -- Nothing -> error $ "failed to parse url: " ++ show u -- Just uri -> uri -#if !MIN_VERSION_http_directory(0,1,9) +-- from http-directory infixr 5 +/+ (+/+) :: String -> String -> String "" +/+ s = s @@ -44,9 +45,14 @@ s +/+ t | last s == '/' = init s +/+ t | head t == '/' = s +/+ tail t s +/+ t = s ++ "/" ++ t -#endif removeSubpath :: [String] -> String -> String removeSubpath path url = let url' = (if last url == '/' then init else id) url in dropSuffix (foldSlash path) url' + +-- from http-directory +trailingSlash :: String -> String +trailingSlash "" = "" +trailingSlash str = + if last str == '/' then str else str ++ "/" diff --git a/test/tests.hs b/test/tests.hs --- a/test/tests.hs +++ b/test/tests.hs @@ -7,7 +7,7 @@ putStrLn "" putStrLn $ "# " ++ unwords args +-+ fromMaybe "" mpkg let debug = False - (ok, out, err) <- cmdFull "fdrq" (["-d" | debug] ++ args ++ maybeToList mpkg) "" + (ok, out, err) <- cmdFull "fedora-repoquery" (["-d" | debug] ++ args ++ maybeToList mpkg) "" if null err then unless (isNothing mpkg || "-T" `notElem` args) $ putStrLn "stderr empty" else putStrLn err @@ -28,6 +28,7 @@ tests :: [([String],Maybe String)] tests = [(["rawhide"], Nothing) + ,(["-D","rawhide"], Nothing) ,([current], Nothing) ,([previous], Nothing) ,([prevprev], Nothing)