diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,9 @@ # Release history for koji-install +## 0.5 (2021-12-27) +- --package and --exclude filters can now be combined +- --package and --exclude now also check subpackage names without base prefix + ## 0.4 (2021-12-20) - support installing/listing by koji taskid - select subpackages with --package and --exclude, by name or globbing diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -3,19 +3,34 @@ A CLI tool to download and install rpms from a Koji build or task. [Koji](https://pagure.io/koji/) is a RPM package buildsystem used by -Fedora, Centos, and some other projects. +Fedora, CentOS, and some other projects. +By default it only downloads binaries of already-installed subpackages, +but there are options to list and select or exclude specific subpackages. + +Note koji-install is intended for development and testing purposes +and should not be necessary/used normally on production systems, +but it can be very helpful for quickly testing an specific package build or +update. + ## Usage By default it uses Fedora Koji. +``` $ koji-install podman - -Will download the latest build for your Fedora version, +``` +will download the latest build for your Fedora version, and try to install it. +Use `--disttag` suffix to select a different Fedora version. -You can specify a different Koji hub service with `--hub`. +``` +$ koji-install TASKID --exclude "*-devel" +``` +will install all the non-devel subpackages from the task. +One can use `--hub` to specify a different Koji hub build service. + ### Selecting subpackages By default only installed subpackages are downloaded and updated, @@ -36,9 +51,9 @@ Usage: koji-install [--version] [-n|--dry-run] [-D|--debug] [-H|--hub HUB] [-P|--packages-url URL] [-l|--list] - [(-a|--all) | (-A|--ask) | (-p|--package SUBPKG) | - (-x|--exclude SUBPKG)] [-d|--disttag DISTTAG] - [(-R|--nvr) | (-V|--nv)] PACKAGE|TASKID... + [(-a|--all) | (-A|--ask) | [-p|--package SUBPKG] + [-x|--exclude SUBPKG]] [-d|--disttag DISTTAG] + [(-R|--nvr) | (-V|--nv)] PKG|NVR|TASKID... HUB = fedora, stream, rpmfusion, or URL Available options: diff --git a/koji-install.cabal b/koji-install.cabal --- a/koji-install.cabal +++ b/koji-install.cabal @@ -1,11 +1,9 @@ name: koji-install -version: 0.4 +version: 0.5 synopsis: CLI tool for installing rpms directly from Fedora Koji description: koji-install can install a koji build/task of a package locally. - By default it only downloads newer binaries of already-installed - subpackages, but there are options to list and select other packages. - Koji is the RPM-based buildsystem of Fedora Linux and Centos. + Koji is the RPM-based buildsystem of Fedora Linux and CentOS. license: BSD3 license-file: LICENSE author: Jens Petersen @@ -40,7 +38,7 @@ Glob, http-directory >= 0.1.9, koji >= 0.0.2, - rpm-nvr, + rpm-nvr >= 0.1.1, simple-cmd, simple-cmd-args, xdg-userdirs diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -22,10 +22,38 @@ import DownloadDir import Paths_koji_install (version) -data Mode = Update | All | Ask | PkgsReq SubPackages - deriving Eq +-- mbox kojihub is locked +knownHubs :: [String] +knownHubs = ["fedora","stream","rpmfusion", "or URL"] -data SubPackages = Subpkgs [String] | ExclPkgs [String] +hubURL :: String -> String +hubURL "fedora" = fedoraKojiHub +-- later use centosKojiHub +hubURL "stream" = "https://kojihub.stream.centos.org/kojihub" +--hubURL "mbox" = "https://koji.mbox.centos.org/kojihub" +hubURL "rpmfusion" = "https://koji.rpmfusion.org/kojihub" +hubURL "fusion" = "https://koji.rpmfusion.org/kojihub" +hubURL hub = + if "http" `isPrefixOf` hub + then hub + else error' $ "unknown hub: try " ++ show knownHubs + +defaultPkgsURL :: String -> String +defaultPkgsURL url = + case dropSuffix "/" url of + "https://koji.fedoraproject.org/kojihub" -> + "https://kojipkgs.fedoraproject.org/packages" + "https://kojihub.stream.centos.org/kojihub" -> + "https://kojihub.stream.centos.org/kojifiles/packages" + _ -> + if "kojihub" `isSuffixOf` url + then replace "kojihub" "kojifiles" url +/+ "packages" + else error' $ "use --files-url to specify kojifiles url for " ++ url + +data Mode = All + | Ask + -- distinguish except and exclude + | PkgsReq [String] [String] -- ^ include, except/exclude deriving Eq data Request = ReqName | ReqNV | ReqNVR @@ -37,7 +65,6 @@ -- FIXME --arch (including src) -- FIXME --debuginfo -- FIXME --delete after installing - main :: IO () main = do sysdisttag <- do @@ -58,16 +85,17 @@ <*> disttagOpt sysdisttag <*> (flagWith' ReqNVR 'R' "nvr" "Give an N-V-R instead of package name" <|> flagWith ReqName ReqNVR 'V' "nv" "Give an N-V instead of package name") - <*> some (strArg "PACKAGE|TASKID...") + <*> some (strArg "PKG|NVR|TASKID...") where modeOpt :: Parser Mode modeOpt = flagWith' All 'a' "all" "all subpackages" <|> flagWith' Ask 'A' "ask" "ask for each subpackge [default if not installed]" <|> - PkgsReq <$> (Subpkgs <$> some (strOptionWith 'p' "package" "SUBPKG" "Subpackage (glob) to install") <|> - ExclPkgs <$> some (strOptionWith 'x' "exclude" "SUBPKG" "Subpackage (glob) not to install")) <|> - pure Update + pkgsReqOpts + pkgsReqOpts = PkgsReq + <$> many (strOptionWith 'p' "package" "SUBPKG" "Subpackage (glob) to install") <*> many (strOptionWith 'x' "exclude" "SUBPKG" "Subpackage (glob) not to install") + disttagOpt :: String -> Parser String disttagOpt disttag = startingDot <$> strOptionalWith 'd' "disttag" "DISTTAG" ("Use a different disttag [default: " ++ disttag ++ "]") disttag @@ -76,37 +104,9 @@ "" -> error' "empty disttag" (c:_) -> if c == '.' then cs else '.' : cs --- mbox kojihub is locked -knownHubs :: [String] -knownHubs = ["fedora","stream","rpmfusion", "or URL"] - -hubURL :: String -> String -hubURL "fedora" = fedoraKojiHub --- later use centosKojiHub -hubURL "stream" = "https://kojihub.stream.centos.org/kojihub" ---hubURL "mbox" = "https://koji.mbox.centos.org/kojihub" -hubURL "rpmfusion" = "https://koji.rpmfusion.org/kojihub" -hubURL "fusion" = "https://koji.rpmfusion.org/kojihub" -hubURL hub = - if "http" `isPrefixOf` hub - then hub - else error' $ "unknown hub: try " ++ show knownHubs - -defaultPkgsURL :: String -> String -defaultPkgsURL url = - case dropSuffix "/" url of - "https://koji.fedoraproject.org/kojihub" -> - "https://kojipkgs.fedoraproject.org/packages" - "https://kojihub.stream.centos.org/kojihub" -> - "https://kojihub.stream.centos.org/kojifiles/packages" - _ -> - if "kojihub" `isSuffixOf` url - then replace "kojihub" "kojifiles" url +/+ "packages" - else error' $ "use --files-url to specify kojifiles url for " ++ url - program :: Bool -> Bool -> Maybe String -> Maybe String -> Bool -> Mode -> String -> Request -> [String] -> IO () -program dryrun debug mhuburl mpkgsurl listmode mode disttag request pkgs = do +program dryrun debug mhuburl mpkgsurl listmode mode disttag request pkgbldtsks = do let huburl = maybe fedoraKojiHub hubURL mhuburl pkgsurl = fromMaybe (defaultPkgsURL huburl) mpkgsurl when debug $ do @@ -116,39 +116,39 @@ dlDir <- setDownloadDir dryrun "rpms" when debug $ putStrLn dlDir setNoBuffering - mapM (kojiRPMs huburl pkgsurl dlDir) pkgs + mapM (kojiRPMs huburl pkgsurl dlDir) pkgbldtsks >>= if listmode then mapM_ putStrLn . mconcat else installRPMs dryrun . mconcat where - kojiRPMs :: String -> String -> String -> String -> IO [String] - kojiRPMs huburl pkgsurl dlDir pkg = - if all isDigit pkg - then kojiTaskRPMs dryrun debug huburl pkgsurl listmode mode dlDir pkg - else kojiBuildRPMs huburl pkgsurl dlDir pkg + kojiRPMs :: String -> String -> String -> String -> IO [String] -- ([String],String) + kojiRPMs huburl pkgsurl dlDir bldtask = + if all isDigit bldtask + then kojiTaskRPMs dryrun debug huburl pkgsurl listmode mode dlDir bldtask + else kojiBuildRPMs huburl pkgsurl dlDir bldtask kojiBuildRPMs :: String -> String -> String -> String -> IO [String] - kojiBuildRPMs huburl pkgsurl dlDir pkg = do - nvrs <- kojiBuildOSBuilds debug huburl listmode disttag request pkg + kojiBuildRPMs huburl pkgsurl dlDir pkgbld = do + nvrs <- kojiBuildOSBuilds debug huburl listmode disttag request pkgbld if listmode - then if mode /= Update + then if mode /= PkgsReq [] [] then error' "modes not supported for listing build" else return nvrs else case nvrs of - [] -> error' $ pkg ++ " not found for " ++ disttag + [] -> error' $ pkgbld ++ " not found for " ++ disttag [nvr] -> do putStrLn $ nvr ++ "\n" allRpms <- map (<.> "rpm") . sort . filter (not . debugPkg) <$> kojiGetBuildRPMs huburl nvr when debug $ print allRpms - dlRpms <- decideRpms listmode mode (Just pkg) allRpms + dlRpms <- decideRpms listmode mode (maybeNVRName nvr) allRpms when debug $ print dlRpms unless (dryrun || null dlRpms) $ do mapM_ (downloadBuildRpm debug pkgsurl (readNVR nvr)) dlRpms -- FIXME once we check file size - can skip if no downloads putStrLn $ "Packages downloaded to " ++ dlDir return dlRpms - _ -> error $ "multiple build founds for " ++ pkg ++ ": " ++ + _ -> error $ "multiple build founds for " ++ pkgbld ++ ": " ++ unwords nvrs debugPkg :: String -> Bool @@ -178,7 +178,12 @@ return [] else do when debug $ print rpms - dlRpms <- decideRpms listmode mode Nothing rpms + let srpm = + case filter (".src.rpm" `isExtensionOf`) rpms of + [src] -> src + _ -> error' "could not determine nvr from any srpm" + nvr = dropSuffix ".src.rpm" srpm + dlRpms <- decideRpms listmode mode (maybeNVRName nvr) $ rpms \\ [srpm] when debug $ print dlRpms unless (dryrun || null dlRpms) $ do mapM_ (downloadTaskRpm debug pkgsurl task) dlRpms @@ -187,11 +192,14 @@ where getTaskRPMs :: Int -> IO [String] getTaskRPMs taskid = - sort . filter isBinaryRpm . map fst <$> + sort . filter (".rpm" `isExtensionOf`) . map fst <$> Koji.listTaskOutput huburl taskid False True False +maybeNVRName :: String -> Maybe String +maybeNVRName = fmap nvrName . maybeNVR + decideRpms :: Bool -> Mode -> Maybe String -> [String] -> IO [String] -decideRpms listmode mode mpkg allRpms = +decideRpms listmode mode mbase allRpms = case mode of All -> if listmode then error' "cannot use --list and --all together" @@ -199,43 +207,57 @@ Ask -> if listmode then error' "cannot use --list and --ask together" else mapMaybeM rpmPrompt allRpms - Update -> + PkgsReq [] [] -> if listmode then return allRpms else do - rpms <- filterM (isInstalled . rpmName . readNVRA) allRpms + rpms <- filterM (isInstalled . nvraName) $ + filter isBinaryRpm allRpms if null rpms - then decideRpms listmode Ask mpkg allRpms + then decideRpms listmode Ask mbase allRpms else return rpms - PkgsReq pkgsreq -> - return $ selectRPMs pkgsreq allRpms + PkgsReq subpkgs exclpkgs -> + return $ selectRPMs mbase (subpkgs,exclpkgs) allRpms isInstalled :: String -> IO Bool isInstalled rpm = cmdBool "rpm" ["--quiet", "-q", rpm] -selectRPMs :: SubPackages -> [String] -> [String] -selectRPMs pkgsreq allRpms = - case pkgsreq of - Subpkgs subpkgs -> - sort . mconcat $ - flip map subpkgs $ \ pkgpat -> - case filter (match (compile pkgpat) . nvraName) allRpms of - [] -> error' $ "no subpackage match for " ++ pkgpat - result -> result - ExclPkgs subpkgs -> - -- FIXME somehow determine unused excludes - foldl' (exclude subpkgs) [] allRpms +selectRPMs :: Maybe String -> ([String],[String]) -> [String] -> [String] +selectRPMs mbase (subpkgs,[]) rpms = + sort . mconcat $ + flip map subpkgs $ \ pkgpat -> + case filter (match (compile pkgpat) . nvraName) rpms of + [] -> case mbase of + Just base | head pkgpat /= '*' -> + selectRPMs Nothing ([base ++ '-' : pkgpat],[]) rpms + _ -> error' $ "no subpackage match for " ++ pkgpat + result -> result +selectRPMs mbase ([], subpkgs) rpms = + -- FIXME somehow determine unused excludes + foldl' (exclude subpkgs) [] rpms where - nvraName :: String -> String - nvraName = rpmName . readNVRA - exclude :: [String] -> [String] -> String -> [String] - exclude [] acc pkg = acc ++ [pkg] - exclude (p:ps) acc pkg = - if match (compile p) (nvraName pkg) - then acc - else exclude ps acc pkg + exclude [] acc rpm = acc ++ [rpm] + exclude (pat:pats) acc rpm = + if checkMatch (nvraName rpm) + then acc + else exclude pats acc rpm + where + checkMatch :: String -> Bool + checkMatch rpmname = + let comppat = compile pat + in if isLiteral comppat + then pat == rpmname || + maybe False (\b -> (b ++ '-' : pat) == rpmname) mbase + else match comppat rpmname +selectRPMs mbase (subpkgs,exclpkgs) rpms = + let needed = selectRPMs mbase (subpkgs,[]) rpms + excluded = selectRPMs mbase ([], exclpkgs) rpms + in nub . sort $ needed ++ excluded +nvraName :: String -> String +nvraName = rpmName . readNVRA + rpmPrompt :: String -> IO (Maybe String) rpmPrompt rpm = do putStr $ rpm ++ " [y/n]: " @@ -324,13 +346,13 @@ installRPMs :: Bool -> [FilePath] -> IO () installRPMs _ [] = return () -installRPMs dryrun pkgs = do - installed <- filterM (isInstalled . rpmName . readNVRA) pkgs +installRPMs dryrun rpms = do + installed <- filterM (isInstalled . dropExtension) rpms unless (null installed) $ if dryrun then mapM_ putStrLn $ "would update:" : installed else sudo_ "dnf" ("reinstall" : installed) - let rest = pkgs \\ installed + let rest = rpms \\ installed unless (null rest) $ if dryrun then mapM_ putStrLn $ "would install:" : rest diff --git a/test/tests.hs b/test/tests.hs --- a/test/tests.hs +++ b/test/tests.hs @@ -10,7 +10,7 @@ tests havedist = [["podman", "-p", "podman"] ++ sysdist ,["-l", "coreutils"] ++ sysdist - ,["-l", "79802560", "-p", "rpmlint"] + ,["-l", "-R", "rpmlint-2.2.0-1.fc36"] ,["-H", "https://kojihub.stream.centos.org/kojihub", "-d", "el9", "bash", "-p", "bash"] ,["-H", "stream", "-d", "el9", "kernel", "-x", "kernel-devel*", "-x", "*-debug*"] ,["-l", "-H", "stream", "-d", "el9", "grep"]