diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,17 @@ # Version history of koji-tool +## 1.3.1 (2025-12-23) +- 'install': use select-rpms-0.3.1 + - allows selecting debuginfo + - add --allow-erasing + - rpm package manager options +- 'install': do not print download dir for --list mode +- 'tasks': detect dynbr failure and show root.log +- 'tasks': for build.log print the full log url too +- 'tasks': use "Accept: text/plain" for getting logs +- 'tasks': less whitespace for compact task output +- use http-directory-0.1.12 for Accept text/html + ## 1.3 (2025-05-03) - 'builds': add --tail - 'builds': building state aliases: build, open, run diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ `$ koji-tool --version` ``` -1.3 +1.3.1 ``` `$ koji-tool --help` @@ -79,13 +79,13 @@ `$ koji-tool builds --help` ``` -Usage: koji-tool builds [-H|--hub HUB] [(-u|--user USER) | (-M|--mine)] - [(-L|--latest) | (-U|--unlimited) | (-l|--limit INT)] - [-s|--state STATE] - [(-B|--before TIMESTAMP) | (-F|--from TIMESTAMP)] - [--type TYPE] [(-d|--details) | (-t|--tasks)] - [-T|--tail] [-i|--install INSTALLOPTS] [-D|--debug] - [(-b|--build NVR/BUILDID) | (-p|--pattern NVRPAT) | +Usage: koji-tool builds [-H|--hub HUB] [(-u|--user USER) | (-M|--mine)] + [(-L|--latest) | (-U|--unlimited) | (-l|--limit INT)] + [-s|--state STATE] + [(-B|--before TIMESTAMP) | (-F|--from TIMESTAMP)] + [--type TYPE] [(-d|--details) | (-t|--tasks)] + [-T|--tail] [-i|--install INSTALLOPTS] [-D|--debug] + [(-b|--build NVR/BUILDID) | (-p|--pattern NVRPAT) | PACKAGE] Query Koji builds (by default lists the most recent builds) @@ -128,8 +128,8 @@ redhat-rpm-config-342-4.fc42 BuildComplete (churchyard) https://koji.fedoraproject.org/koji/buildinfo?buildID=2696410 https://koji.fedoraproject.org/koji/taskinfo?taskID=131402281 -start: Fri Apr 11 12:30:43 CEST 2025 -completion: Fri Apr 11 12:32:00 CEST 2025 +start: Fri Apr 11 18:30:43 +08 2025 +completion: Fri Apr 11 18:32:00 +08 2025 duration: 1 min 17 sec https://kojipkgs.fedoraproject.org/packages/redhat-rpm-config/342/4.fc42 ``` @@ -163,16 +163,16 @@ `$ koji-tool tasks --help` ``` -Usage: koji-tool tasks [-H|--hub HUB] [(-u|--user USER) | (-M|--mine)] - [(-L|--latest) | (-U|--unlimited) | (-l|--limit INT)] - [-s|--state STATE] [-a|--arch ARCH] - [(-B|--before TIMESTAMP) | (-F|--from TIMESTAMP)] - [-m|--method METHOD] [-D|--debug] - [(-P|--only-package PKG) | (-N|--only-nvr PREFIX)] - [(-d|--details) | (-c|--concise)] [-T|--tail] - [--hw-info | --rootlog] [-g|--grep STRING] - [-i|--install INSTALLOPTS] - [(-b|--build BUILD) | (-p|--pattern NVRPAT) | +Usage: koji-tool tasks [-H|--hub HUB] [(-u|--user USER) | (-M|--mine)] + [(-L|--latest) | (-U|--unlimited) | (-l|--limit INT)] + [-s|--state STATE] [-a|--arch ARCH] + [(-B|--before TIMESTAMP) | (-F|--from TIMESTAMP)] + [-m|--method METHOD] [-D|--debug] + [(-P|--only-package PKG) | (-N|--only-nvr PREFIX)] + [(-d|--details) | (-c|--concise)] [-T|--tail] + [--hw-info | --rootlog] [-g|--grep STRING] + [-i|--install INSTALLOPTS] + [(-b|--build BUILD) | (-p|--pattern NVRPAT) | --children TASKID | --parent TASKID | PACKAGE|TASKID] Query Koji tasks (by default lists the most recent buildArch tasks) @@ -221,11 +221,12 @@ `$ koji-tool tasks --method newrepo --latest` ``` -ValueStruct [("id",ValueInt 106124),("name",ValueString "epel10.1-build")] ValueStruct [("opts",ValueStruct []),("__starstar",ValueBool True)] newRepo TaskOpen (kojira) -https://koji.fedoraproject.org/koji/taskinfo?taskID=133511105 -create: Tue Jun 3 18:23:56 CEST 2025 -start: Tue Jun 3 18:24:47 CEST 2025 -current duration: 1 min 46 sec (start delay 51 sec) +ValueStruct [("id",ValueInt 45727),("name",ValueString "epel9-build")] ValueStruct [("opts",ValueStruct []),("__starstar",ValueBool True)] newRepo TaskClosed (kojira) +https://koji.fedoraproject.org/koji/taskinfo?taskID=140300590 +create: Tue Dec 23 14:41:11 +08 2025 +start: Tue Dec 23 14:42:03 +08 2025 +completion: Tue Dec 23 14:45:38 +08 2025 +duration: 3 min 35 sec (start delay 52 sec) ``` List latest package build's tasks: @@ -233,13 +234,13 @@ `$ koji-tool tasks --latest redhat-rpm-config` ``` -redhat-rpm-config-343-6.eln148.noarch TaskClosed (distrobuildsync-eln/jenkins-continuous-infra.apps.ci.centos.org) -https://koji.fedoraproject.org/koji/taskinfo?taskID=133375186 (parent: 133374938) -create: Sat May 31 12:36:50 CEST 2025 -start: Sat May 31 12:37:34 CEST 2025 -completion: Sat May 31 12:38:37 CEST 2025 -duration: 1 min 3 sec (start delay 44 sec) -https://kojipkgs.fedoraproject.org/work/tasks/5186/133375186/build.log (16kB) +redhat-rpm-config-343-19.eln153.noarch TaskClosed (distrobuildsync-eln/jenkins-continuous-infra.apps.ci.centos.org) +https://koji.fedoraproject.org/koji/taskinfo?taskID=140205053 (parent: 140204866) +create: Sat Dec 20 23:40:05 +08 2025 +start: Sat Dec 20 23:41:17 +08 2025 +completion: Sat Dec 20 23:42:19 +08 2025 +duration: 1 min 2 sec (start delay 1 min 12 sec) +https://kojipkgs.fedoraproject.org/work/tasks/5053/140205053/build.log (16kB) ``` It is also possible to install packages from a task using @@ -313,16 +314,17 @@ `$ koji-tool install --help` ``` -Usage: koji-tool install [-n|--dry-run] [-D|--debug] [-y|--yes] [-H|--hub HUB] - [-P|--packages-url URL] [-l|--list] [-L|--latest] - [-t|--check-remote-time] - [--rpm | --rpm-ostree | --dnf5 | --dnf3] - [-a|--arch ARCH] - [(-N|--no-reinstall) | (-S|--skip-existing) | - (-O|--only-existing)] [-b|--prefix SUBPKGPREFIX] - [--all | --ask | [-p|--package SUBPKG] - [-e|--except SUBPKG] [-x|--exclude SUBPKG] - [-i|--include SUBPKG]] [-d|--disttag DISTTAG] +Usage: koji-tool install [-n|--dry-run] [-D|--debug] [-y|--yes] [-H|--hub HUB] + [-P|--packages-url URL] [-l|--list] [-L|--latest] + [-t|--check-remote-time] + [--rpm | --rpm-ostree | --dnf5 | --dnf3] + [--allow-erasing] [-a|--arch ARCH] + [(-N|--no-reinstall) | (-S|--skip-existing) | + (-O|--only-existing) | (-E|--error-existing)] + [-b|--prefix SUBPKGPREFIX] + [--all | --ask | [-p|--package SUBPKG] + [-e|--except SUBPKG] [-x|--exclude SUBPKG] + [-i|--include SUBPKG]] [-d|--disttag DISTTAG] [--tagged | (-R|--nvr) | (-V|--nv)] PKG|NVR|TASKID|TAG... @@ -335,18 +337,20 @@ -H,--hub HUB KojiHub shortname or url (HUB = fedora, stream, rpmfusion, or URL) [default: fedora] -P,--packages-url URL KojiFiles packages url [default: Fedora] - -l,--list List builds + -l,--list List RPMs -L,--latest Latest build -t,--check-remote-time Check remote rpm timestamps --rpm Use rpm instead of dnf --rpm-ostree Use rpm-ostree instead of dnf --dnf5 Use dnf5 to install --dnf3 Use dnf-3 to install [default dnf unless ostree] + --allow-erasing Use dnf --allowerasing -a,--arch ARCH Task arch -N,--no-reinstall Do not reinstall existing NVRs -S,--skip-existing Ignore already installed subpackages (implies --no-reinstall) -O,--only-existing Only update existing installed subpackages + -E,--error-existing Abort for existing installed subpackages -b,--prefix SUBPKGPREFIX Prefix to use for subpackages [default: base package] --all all subpackages [default if not installed] --ask ask for each subpackage diff --git a/koji-tool.cabal b/koji-tool.cabal --- a/koji-tool.cabal +++ b/koji-tool.cabal @@ -1,5 +1,5 @@ name: koji-tool -version: 1.3 +version: 1.3.1 synopsis: Koji CLI tool for querying tasks and installing builds description: koji-tool is a CLI interface to Koji with commands to query @@ -27,7 +27,8 @@ || == 9.4.8 || == 9.6.7 || == 9.8.4 - || == 9.10.2 + || == 9.10.3 + || == 9.12.2 source-repository head type: git @@ -54,12 +55,13 @@ filepath, formatting >= 7, http-conduit, - http-directory >= 0.1.10, + http-directory >= 0.1.12, + http-types, koji >= 0.0.2, pretty-simple, rpm-nvr >= 0.1.2, safe, - select-rpms >= 0.2, + select-rpms >= 0.3.1, simple-cmd >= 0.2.2, simple-cmd-args >= 0.1.8, text, diff --git a/src/Builds.hs b/src/Builds.hs --- a/src/Builds.hs +++ b/src/Builds.hs @@ -25,7 +25,6 @@ import Distribution.Koji import Distribution.Koji.API import Safe (headMay) -import SelectRPMs (selectDefault) import SimpleCmd import Text.Pretty.Simple @@ -206,7 +205,7 @@ void $ cmdBool "koji" ["watch-task", show taskid] whenJust minstall $ \installopts -> do putStrLn "" - installCmd False debug No (Just hub) Nothing False False False Nothing [] Nothing Nothing installopts Nothing (Right ReqNVR) [showNVR (buildNVR build)] + installCmd False debug No (Just hub) Nothing False False False Nothing False [] Nothing Nothing installopts Nothing (Right ReqNVR) [showNVR (buildNVR build)] formatBuildResult :: String -> Bool -> TimeZone -> BuildResult -> [String] formatBuildResult hub ended tz (BuildResult nvr state buildid mtaskid start mendtime owner) = @@ -268,7 +267,7 @@ -- printBuild hub tz (Just Detailed) False debug Nothing mapM_ putKojiBuild kblds when install $ - installCmd False False No (Just hub) Nothing False False False Nothing [] Nothing Nothing selectDefault Nothing (Right ReqNVR) $ map kbNvr kblds + installCmd False False No (Just hub) Nothing False False False Nothing False [] Nothing Nothing selectDefault Nothing (Right ReqNVR) $ map kbNvr kblds where putKojiBuild :: KojiBuild -> IO () putKojiBuild (KojiBuild _ _ owner nvr) = diff --git a/src/Find.hs b/src/Find.hs --- a/src/Find.hs +++ b/src/Find.hs @@ -19,7 +19,7 @@ import qualified Builds import Common (Limit(Limit)) -import Install (Select(PkgsReq)) +import Install (selectDefault) import qualified Tasks import User ( UserOpt(User, UserSelf) ) @@ -106,7 +106,7 @@ "you can only specify one package - too many unknown words: " ++ unwords other installation = if install - then Just (PkgsReq [] [] [] []) + then Just selectDefault else Nothing if build then diff --git a/src/Install.hs b/src/Install.hs --- a/src/Install.hs +++ b/src/Install.hs @@ -4,9 +4,10 @@ module Install ( Select(..), + selectDefault, Request(..), installCmd, - ExistingStrategy(..), + ExistingStrategy, PkgMgr(..), knownHubs, Yes(..), @@ -24,9 +25,9 @@ import Distribution.Koji import qualified Distribution.Koji.API as Koji import Network.HTTP.Directory (httpFileSize', httpLastModified', (+/+)) -import SelectRPMs (Yes(..), PkgMgr(..), ExistingStrategy(..), Select(..), - ExistNVRA, checkSelection, decideRPMs, installArgs, - installRPMs, nvraToRPM, rpmsToNVRAs) +import SelectRPMs (Yes(..), PkgMgr(..), ExistingStrategy, Select(..), + ExistNVRA, decideRPMs, installArgs, installRPMsAllowErasing, + nvraToRPM, rpmsToNVRAs, selectDefault) import SimpleCmd import System.Directory import System.FilePath @@ -52,11 +53,10 @@ -- FIXME offer to download subpackage deps -- FIXME is --check-remote-time really needed? installCmd :: Bool -> Bool -> Yes -> Maybe String -> Maybe String -> Bool - -> Bool -> Bool -> Maybe PkgMgr -> [String] + -> Bool -> Bool -> Maybe PkgMgr -> Bool -> [String] -> Maybe ExistingStrategy -> Maybe String -> Select -> Maybe String -> Either () Request -> [String] -> IO () -installCmd dryrun debug yes mhuburl mpkgsurl listmode latest checkremotetime mmgr archs mstrategy mprefix select mdisttag erequest pkgbldtsktag = do - checkSelection select +installCmd dryrun debug yes mhuburl mpkgsurl listmode latest checkremotetime mmgr allowerasing archs mstrategy mprefix select mdisttag erequest pkgbldtsktag = do let huburl = maybe fedoraKojiHub hubURL mhuburl pkgsurl = fromMaybe (hubToPkgsURL huburl) mpkgsurl when debug $ do @@ -78,8 +78,8 @@ return (req,pkgbldtsktag) buildrpms <- mapM (kojiRPMs huburl pkgsurl request) $ nubOrd pkgbldtsks - printDlDir - installRPMs dryrun debug mmgr yes buildrpms + unless listmode printDlDir + installRPMsAllowErasing dryrun debug mmgr allowerasing yes buildrpms where kojiRPMs :: String -> String -> Request -> String -> IO (FilePath,[ExistNVRA]) diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -6,7 +6,7 @@ import Data.Char (isDigit) import Data.List.Extra -import SelectRPMs (existingStrategyOption, selectRpmsOptions) +import SelectRPMs (existingStrategyOption, pkgMgrOpt, selectRpmsOptions) import SimpleCmd import SimpleCmdArgs @@ -80,10 +80,11 @@ <*> hubOpt <*> optional (strOptionWith 'P' "packages-url" "URL" "KojiFiles packages url [default: Fedora]") - <*> switchWith 'l' "list" "List builds" + <*> switchWith 'l' "list" "List RPMs" <*> switchWith 'L' "latest" "Latest build" <*> switchWith 't' "check-remote-time" "Check remote rpm timestamps" <*> optional pkgMgrOpt + <*> switchLongWith "allow-erasing" "Use dnf --allowerasing" <*> many archOpt <*> optional existingStrategyOption <*> optional (strOptionWith 'b' "prefix" "SUBPKGPREFIX" "Prefix to use for subpackages [default: base package]") @@ -156,13 +157,6 @@ readTaskReq :: String -> Maybe TaskReq readTaskReq cs = Just $ if all isDigit cs then Task (read cs) else Package cs - - pkgMgrOpt :: Parser PkgMgr - pkgMgrOpt = - flagLongWith' RPM "rpm" "Use rpm instead of dnf" <|> - flagLongWith' OSTREE "rpm-ostree" "Use rpm-ostree instead of dnf" <|> - flagLongWith' DNF5 "dnf5" "Use dnf5 to install" <|> - flagLongWith' DNF3 "dnf3" "Use dnf-3 to install [default dnf unless ostree]" -- FIXME check valid arch (eg i686 not i386) archOpt = strOptionWith 'a' "arch" "ARCH" "Task arch" diff --git a/src/Tasks.hs b/src/Tasks.hs --- a/src/Tasks.hs +++ b/src/Tasks.hs @@ -36,6 +36,7 @@ import Formatting hiding (now) import Network.HTTP.Directory import Network.HTTP.Simple +import Network.HTTP.Types (hAccept) import Safe (headMay, lastMay) import SimpleCmd import System.FilePath @@ -156,7 +157,7 @@ buildlogSize qDebug tz tail' logfile mgrep hub task whenJust minstall $ \installopts -> do putStrLn "" - installCmd False qDebug No (Just hub) Nothing False False False Nothing [] Nothing Nothing installopts Nothing (Right ReqName) [show (taskId task)] + installCmd False qDebug No (Just hub) Nothing False False False Nothing False [] Nothing Nothing installopts Nothing (Right ReqName) [show (taskId task)] maybeTaskResult :: Struct -> Maybe TaskResult maybeTaskResult st = do @@ -331,7 +332,7 @@ taskinfoUrl hub taskid] ++ ["(" ++ renderDuration True dur ++ ")" | Just start <- [mstart], Just end <- [mend], let dur = diffUTCTime end start] ++ [show state, - showPackage pkg ++ if method == "buildArch" then '.' : arch ++ replicate (8 - length arch) ' ' else ' ' : method] + showPackage pkg ++ if method == "buildArch" then '.' : arch else ' ' : method] formatTaskResult :: String -> Maybe UTCTime -> TimeZone -> TaskResult -> [String] formatTaskResult hub @@ -443,7 +444,6 @@ whenJust mtime $ \time -> putStr $ " (" ++ compactZonedTime tz time ++ ")" putChar '\n' - -- FIXME check if short build.log ends with srpm let file = if logfile == BuildLog && size < 4000 then RootLog @@ -451,59 +451,69 @@ unless (file == BuildLog) $ putStrLn $ url +/+ logFile file when (tail' || logfile /= BuildLog || isJust mgrep) $ do - displayLog url file + displayLog size url file (putStrLn . compactTaskResult hub tz) task else do let rootlog = url +/+ logFile RootLog whenM (httpExists' rootlog) $ putStrLn rootlog where - displayLog :: String -> LogFile -> IO () - displayLog url file = do + displayLog :: Integer -> String -> LogFile -> IO () + displayLog buildlogsize url file = do let logurl = case file of BuildLog -> tailLogUrl hub (taskId task) file Nothing _ -> url +/+ logFile file req <- parseRequest logurl - resp <- httpLBS req - let out = U.toString $ getResponseBody resp - ls = lines out - putStrLn "" - let output - | file == RootLog = - let excluded = ["Executing command:", - "Child return code was: 0", - "child environment: None", - "ensuring that dir exists:", - "touching file:", - "creating dir:", - "kill orphans"] - in - map (dropPrefix "DEBUG ") $ takeEnd 30 $ - filter (\l -> not (any (`isInfixOf` l) excluded)) ls - | lastMay ls == Just "Child return code was: 0" = ls - | otherwise = - case breakOnEnd ["Child return code was: 1"] ls of - ([],ls') -> ls' - (ls',_) -> ls' - putStr $ unlines $ - case mgrep of - Nothing -> output - Just needle -> - filter (match needle) ls - putStrLn $ '\n' : logurl ++ "\n" - where - match :: String -> String -> Bool - match "" _ = error' "empty grep string not allowed" - match _ "" = False - match ('^':needle) ls = - if lastMay needle == Just '$' - then needle == ls - else needle `isPrefixOf` ls - match needle ls = - if lastMay needle == Just '$' - then needle `isSuffixOf` ls - else needle `isInfixOf` ls + resp <- httpLBS $ addRequestHeader hAccept "text/plain" req + let ls = lines . U.toString $ getResponseBody resp + -- detect dynbr failure (build.log ~ 7400B): + -- Wrote: /builddir/build/SRPMS/bustle-0.13.0-1.fc42.buildreqs.nosrc.rpm + -- Child return code was: 11 + -- Dynamic buildrequires detected + -- Going to install missing buildrequires. See root.log for details. + if file == BuildLog && buildlogsize < 8000 && + lastMay ls == Just "Going to install missing buildrequires. See root.log for details." + then displayLog buildlogsize url RootLog + else do + putStrLn "" + let output + | file == RootLog = + let excluded = ["Executing command:", + "Child return code was: 0", + "child environment: None", + "ensuring that dir exists:", + "touching file:", + "creating dir:", + "kill orphans"] + in + map (dropPrefix "DEBUG ") $ takeEnd 30 $ + filter (\l -> not (any (`isInfixOf` l) excluded)) ls + | lastMay ls == Just "Child return code was: 0" = ls + | otherwise = + case breakOnEnd ["Child return code was: 1"] ls of + ([],ls') -> ls' + (ls',_) -> ls' + putStr $ unlines $ + case mgrep of + Nothing -> output + Just needle -> + filter (match needle) ls + putStrLn $ '\n' : logurl + when (file == BuildLog) $ + putStrLn $ url +/+ logFile file ++ "\n" + where + match :: String -> String -> Bool + match "" _ = error' "empty grep string not allowed" + match _ "" = False + match ('^':needle) ls = + if lastMay needle == Just '$' + then needle == ls + else needle `isPrefixOf` ls + match needle ls = + if lastMay needle == Just '$' + then needle `isSuffixOf` ls + else needle `isInfixOf` ls -- FIXME turn into a type? kojiMethods :: [String] diff --git a/test/tests.hs b/test/tests.hs --- a/test/tests.hs +++ b/test/tests.hs @@ -1,44 +1,44 @@ import SimpleCmd import System.IO -program :: ([String], [[String]]) -> IO () -program (c, argsv) = - putStrLn ("\n# " ++ head c) >> +program :: (String, [String], [[String]]) -> IO () +program (c, cs, argsv) = + putStrLn ("\n# " ++ c) >> mapM_ run argsv where run args = do putStrLn "" - cmdLog "koji-tool" (c ++ args) + cmdLog_ "koji-tool" (c:cs ++ args) -tests :: Bool -> [([String], [[String]])] +tests :: Bool -> [(String, [String], [[String]])] tests havedist = [ - (["builds"], + ("builds", [], [["-L", "rust"] ,["-l", "3"] ,["-L", "-p", "rpm-ostree*.fc40"]]) , - (["tasks"], + ("tasks", [], [["-L"] ,["-l", "3"] ,["-L", "rpm-ostree"]]) , - (["latest"], + ("latest", [], [["rawhide", "ghc"]]) , - (["find"], + ("find", [], [["last", "failed", "build"]]) , - (["install", "-n", "-y"], + ("install", ["-n", "-y"], [["podman", "-p", "podman"] ++ sysdist ,["-l", "coreutils"] ++ sysdist ,["-l", "-R", "rpmlint-2.5.0-5.fc40"] ,["-H", "https://kojihub.stream.centos.org/kojihub", "-d", "el9", "bash", "-p", "bash"] ,["-H", "stream", "-d", "el9", "kernel", "-x", "kernel-devel*", "-x", "*-debug*"] ,["-l", "-H", "stream", "-d", "el9", "grep"] - ,["-H", "rpmfusion", "ffmpeg", "-p", "ffmpeg", "-p", "ffmpeg-libs"] ++ sysdist - ,["-l", "-H", "rpmfusion", "ffmpeg"] ++ sysdist - ,["ghc9.4", + -- ,["-H", "rpmfusion", "ffmpeg", "-p", "ffmpeg", "-p", "ffmpeg-libs"] ++ sysdist + -- ,["-l", "-H", "rpmfusion", "ffmpeg"] ++ sysdist + ,["ghc9.6", "-e", "*-devel", "-x", "*-prof", "-x", "*-doc", @@ -49,7 +49,7 @@ ) ] where - sysdist = if havedist then [] else ["-d", "fc40"] + sysdist = if havedist then [] else ["-d", "fc42"] main :: IO () main = do