diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.9.6 (2023-08-18) +- --no-dl option to avoid dl.fp.o even if newer +- support c8s and add options for centos-stream channels +- update eln path +- downloadFile: add showdestdir to downloading message and curl debug +- add --debug option + ## 0.9.5 (2023-04-13) - add F38 Sericea sway ostree image diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -5,28 +5,33 @@ [![Stackage Lts](http://stackage.org/package/dl-fedora/badge/lts)](http://stackage.org/lts/package/dl-fedora) [![Stackage Nightly](http://stackage.org/package/dl-fedora/badge/nightly)](http://stackage.org/nightly/package/dl-fedora) -A tool for downloading Fedora images. +A tool for downloading Fedora, ELN, and Centos Stream images. By default it targets the Workstation edition of Fedora. Usage examples: `dl-fedora rawhide` : downloads the latest Fedora Rawhide Workstation Live iso -`dl-fedora 38 silverblue` : downloads the Fedora Silverblue iso +`dl-fedora 39 silverblue` : downloads the Fedora Silverblue iso `dl-fedora respin kde` : downloads the latest KDE Live respin -`dl-fedora 37 server --arch aarch64` : will download the Server iso for armv8 +`dl-fedora 38 server --arch aarch64` : will download the Server iso for armv8 -`dl-fedora --run 38` : will download Fedora Workstation and boot the Live image with qemu-kvm. +`dl-fedora --run 39` : will download Fedora Workstation and boot the Live image with qemu-kvm. `dl-fedora --local rawhide` : shows the current locally available image (as well as the latest one). With `--dryrun` it doesn't check for newest iso. +`dl-fedora c9s` : downloads a Centos Stream 9 net installer. + By default dl-fedora downloads to `~/Downloads/` (correctly the XDG user "DOWNLOADS" directory), -but if you create an `iso` subdirectory (`~/Downloads/iso/`) +but if you create an `iso` subdirectory there (`~/Downloads/iso/`) it will use that directory instead. +`dl-fedora` downloads the latest image redirected from `download.fedoraproject.org` by default, after checking for the latest image at `dl.fedoraproject.org`. +However if the mirror is not synced yet it will download from `dl.fedoraproject.org` instead. You can use `--no-dl` to avoid checking and downloading from the main dl.fedoraproject.org mirror (in this case a nightly image might not be the very latest). + If the image is already found to be downloaded it will not be re-downloaded of course. Curl is used to do the downloading: partial downloads will continue. @@ -39,18 +44,18 @@ ## Usage ```shellsession $ dl-fedora --version -0.9.5.1 +0.9.6 $ dl-fedora --help Fedora iso downloader Usage: dl-fedora [--version] [-g|--gpg-keys] - [(-C|--no-checksum) | (-c|--checksum)] [-n|--dry-run] + [(-C|--no-checksum) | (-c|--checksum)] [-n|--dry-run] [--debug] [-T|--no-http-timeout] [-l|--local] [-r|--run] [-R|--replace] - [(-d|--dl) | (-k|--koji) | (-m|--mirror URL)] [-a|--arch ARCH] - RELEASE [EDITION] + [(-d|--dl) | (-D|--no-dl) | (-k|--koji) | (-m|--mirror URL)] + [--cs-devel | --cs-test] [-a|--arch ARCH] RELEASE [EDITION] Tool for downloading Fedora iso file images. - RELEASE = release number, respin, rawhide, test (Beta), stage (RC), eln, c9s + RELEASE = release number, respin, rawhide, test (Beta), stage (RC), eln, c8s, c9s EDITION = {cloud,container,everything,server,workstation,silverblue,kinoite, sericea,budgie,cinnamon,i3,kde,lxde,lxqt,mate,soas,sway, xfce} [default: workstation] @@ -65,16 +70,21 @@ -C,--no-checksum Do not check checksum -c,--checksum Do checksum even if already downloaded -n,--dry-run Don't actually download anything + --debug Debug output -T,--no-http-timeout Do not timeout for http response -l,--local Show current local image via symlink -r,--run Boot image in Qemu -R,--replace Delete previous snapshot image after downloading latest one - -d,--dl Use dl.fedoraproject.org + -d,--dl Use dl.fedoraproject.org (dl.fp.o) + -D,--no-dl Do not use dl.fp.o (even if newer) -k,--koji Use koji.fedoraproject.org -m,--mirror URL Mirror url for /pub [default https://download.fedoraproject.org/pub] - -a,--arch ARCH Architecture [default: x86_64] + --cs-devel Use centos-stream development compose + --cs-test Use centos-stream test compose (default is + production) + -a,--arch ARCH Specify arch [default: x86_64] ``` ## Contribution diff --git a/dl-fedora.cabal b/dl-fedora.cabal --- a/dl-fedora.cabal +++ b/dl-fedora.cabal @@ -1,8 +1,9 @@ cabal-version: 1.18 name: dl-fedora -version: 0.9.5.1 +version: 0.9.6 synopsis: Fedora image download tool -description: Tool to download Fedora iso and image files +description: + Tool to download Fedora and Centos Stream iso and image files. -- can change to GPL-3.0-or-later with Cabal-2.2 license: GPL-3 license-file: LICENSE @@ -15,7 +16,7 @@ build-type: Simple extra-doc-files: README.md , CHANGELOG.md -tested-with: GHC == 9.4.4, GHC == 9.2.7, GHC == 9.0.2, GHC == 8.10.7, +tested-with: GHC == 9.4.5, GHC == 9.2.8, GHC == 9.0.2, GHC == 8.10.7, GHC == 8.8.4, GHC == 8.6.5, GHC == 8.4.4, GHC == 8.2.2 source-repository head @@ -41,7 +42,7 @@ optparse-applicative, regex-posix, simple-cmd >= 0.2.0, - simple-cmd-args >= 0.1.2, + simple-cmd-args >= 0.1.8, text, time, unix, diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -28,8 +28,8 @@ import Paths_dl_fedora (version) -import SimpleCmd (cmd_, cmdN, error', grep_, pipe_, pipeBool, pipeFile_, - {-removePrefix,-} warning) +import SimpleCmd (cmd, cmd_, cmdN, error', grep_, pipe_, pipeBool, pipeFile_, + {-removePrefix,-} warning, (+-+)) import SimpleCmdArgs import System.Directory (createDirectory, doesDirectoryExist, doesFileExist, @@ -96,46 +96,60 @@ data CheckSum = AutoCheckSum | NoCheckSum | CheckSum deriving Eq -dlFpo, downloadFpo, kojiPkgs, odcsFpo, c9sComposes, odcsStream :: String +dlFpo, downloadFpo, kojiPkgs, odcsFpo, csComposes, odcsStream :: String dlFpo = "https://dl.fedoraproject.org/pub" downloadFpo = "https://download.fedoraproject.org/pub" kojiPkgs = "https://kojipkgs.fedoraproject.org/compose" odcsFpo = "https://odcs.fedoraproject.org/composes" -c9sComposes = "https://composes.stream.centos.org" +csComposes = "https://composes.stream.centos.org" odcsStream = "https://odcs.stream.centos.org" -data Mirror = DlFpo | KojiFpo | Mirror String | DefaultMirror +data Mirror = DlFpo | UseMirror | KojiFpo | Mirror String | DefaultLatest + deriving Eq +data CentosChannel = CSProduction | CSTest | CSDevelopment + deriving Eq + +showChannel :: CentosChannel -> String +showChannel CSProduction = "production" +showChannel CSTest = "test" +showChannel CSDevelopment = "development" + main :: IO () main = do let pdoc = Just $ P.vcat [ P.text "Tool for downloading Fedora iso file images.", - P.text ("RELEASE = " <> intercalate ", " ["release number", "respin", "rawhide", "test (Beta)", "stage (RC)", "eln", "c9s"]), + P.text ("RELEASE = " <> intercalate ", " ["release number", "respin", "rawhide", "test (Beta)", "stage (RC)", "eln", "c8s", "c9s"]), P.text "EDITION = " <> P.lbrace <> P.align (P.fillCat (P.punctuate P.comma (map (P.text . lowerEdition) [(minBound :: FedoraEdition)..maxBound])) <> P.rbrace) <> P.text " [default: workstation]" , P.text "", P.text "See ", P.text "and also ." ] + sysarch <- readArch <$> cmd "rpm" ["--eval", "%{_arch}"] simpleCmdArgsWithMods (Just version) (fullDesc <> header "Fedora iso downloader" <> progDescDoc pdoc) $ program <$> switchWith 'g' "gpg-keys" "Import Fedora GPG keys for verifying checksum file" <*> checkSumOpts <*> switchWith 'n' "dry-run" "Don't actually download anything" + <*> switchLongWith "debug" "Debug output" <*> switchWith 'T' "no-http-timeout" "Do not timeout for http response" <*> switchWith 'l' "local" "Show current local image via symlink" <*> switchWith 'r' "run" "Boot image in Qemu" <*> switchWith 'R' "replace" "Delete previous snapshot image after downloading latest one" <*> mirrorOpt - <*> strOptionalWith 'a' "arch" "ARCH" "Architecture [default: x86_64]" "x86_64" + <*> (flagLongWith' CSDevelopment "cs-devel" "Use centos-stream development compose" <|> + flagLongWith CSProduction CSTest "cs-test" "Use centos-stream test compose (default is production)") + <*> (optionWith (eitherReader eitherArch) 'a' "arch" "ARCH" ("Specify arch [default:" +-+ showArch sysarch ++ "]") <|> pure sysarch) <*> strArg "RELEASE" <*> (fromMaybe Workstation <$> optional (argumentWith auto "EDITION")) where mirrorOpt :: Parser Mirror mirrorOpt = - flagWith' DlFpo 'd' "dl" "Use dl.fedoraproject.org" <|> + flagWith' DlFpo 'd' "dl" "Use dl.fedoraproject.org (dl.fp.o)" <|> + flagWith' UseMirror 'D' "no-dl" "Do not use dl.fp.o (even if newer)" <|> flagWith' KojiFpo 'k' "koji" "Use koji.fedoraproject.org" <|> Mirror <$> strOptionWith 'm' "mirror" "URL" ("Mirror url for /pub [default " ++ downloadFpo ++ "]") <|> - pure DefaultMirror + pure DefaultLatest checkSumOpts :: Parser CheckSum checkSumOpts = @@ -146,18 +160,21 @@ primarySize :: Maybe Integer, primaryTime :: Maybe UTCTime} -program :: Bool -> CheckSum -> Bool -> Bool -> Bool -> Bool -> Bool -> Mirror -> String -> String -> FedoraEdition -> IO () -program gpg checksum dryrun notimeout local run removeold mirror arch tgtrel edition = do +program :: Bool -> CheckSum -> Bool -> Bool -> Bool -> Bool -> Bool -> Bool + -> Mirror -> CentosChannel -> Arch -> String -> FedoraEdition + -> IO () +program gpg checksum dryrun debug notimeout local run removeold mirror channel arch tgtrel edition = do let mirrorUrl = case mirror of Mirror m -> m KojiFpo -> kojiPkgs DlFpo -> dlFpo - DefaultMirror + _ -- UseMirror or DefaultLatest | tgtrel == "eln" -> odcsFpo - | tgtrel == "c9s" -> c9sComposes + | tgtrel `elem` ["c8s","c9s"] -> csComposes | otherwise -> downloadFpo showdestdir <- setDownloadDir dryrun "iso" + when debug $ putStrLn showdestdir mgr <- if notimeout then newManager (tlsManagerSettings {managerResponseTimeout = responseTimeoutNone}) else httpManager @@ -168,21 +185,22 @@ then do filePrefix <- getFilePrefix showdestdir -- FIXME support non-iso - return $ filePrefix <> (if tgtrel == "eln" then "-" <> arch else "") <> "-latest" <.> "iso" + return $ filePrefix <> (if tgtrel == "eln" then "-" <> showArch arch else "") <> "-latest" <.> "iso" else do (fileurl, filenamePrefix, prime, _mchecksum, _done) <- findURL mgr mirrorUrl showdestdir tz <- getCurrentTimeZone putStrLn $ unwords ["Newest:", takeFileName fileurl, renderTime tz (primaryTime prime)] putStrLn fileurl - return $ filenamePrefix <> (if tgtrel == "eln" then "-" <> arch else "") <> "-latest" <.> takeExtension fileurl + return $ filenamePrefix <> (if tgtrel == "eln" then "-" <> showArch arch else "") <> "-latest" <.> takeExtension fileurl if run then bootImage symlink showdestdir else showSymlink symlink showdestdir else do (fileurl, filenamePrefix, prime, mchecksum, done) <- findURL mgr mirrorUrl showdestdir - let symlink = filenamePrefix <> (if tgtrel == "eln" then "-" <> arch else "") <> "-latest" <.> takeExtension fileurl - downloadFile dryrun done mgr fileurl prime >>= fileChecksum mgr mchecksum showdestdir + let symlink = filenamePrefix <> (if tgtrel == "eln" then "-" <> showArch arch else "") <> "-latest" <.> takeExtension fileurl + downloadFile dryrun debug done mgr fileurl prime showdestdir >>= + fileChecksum mgr mchecksum showdestdir unless dryrun $ do let localfile = takeFileName fileurl updateSymlink localfile symlink showdestdir @@ -195,11 +213,15 @@ (path,mrelease) <- urlPathMRel mgr -- use http-directory trailingSlash (0.1.7) let primaryDir = - (case tgtrel of - "koji" -> kojiPkgs - "eln" -> odcsFpo - "c9s" -> odcsStream - _ -> dlFpo) +/+ path <> "/" + case tgtrel of + "koji" -> kojiPkgs + "eln" -> odcsFpo + "c8s" -> odcsStream + "c9s" -> odcsStream + _ -> if mirror == DefaultLatest + then dlFpo + else downloadFpo + +/+ path <> "/" hrefs <- httpDirectory mgr primaryDir let prefixPat = makeFilePrefix mrelease selector = if '*' `elem` prefixPat then (=~ prefixPat) else (prefixPat `isPrefixOf`) @@ -210,7 +232,7 @@ error' $ "no match for " <> prefixPat <> " in " <> primaryDir Just file -> do let prefix = if '*' `elem` prefixPat - then (file =~ prefixPat) ++ if arch `isInfixOf` prefixPat then "" else arch + then (file =~ prefixPat) ++ if showArch arch `isInfixOf` prefixPat then "" else showArch arch else prefixPat primeUrl = primaryDir +/+ file (primeSize,primeTime) <- httpFileSizeTime mgr primeUrl @@ -263,7 +285,7 @@ else prefixPat Just file -> do let prefix = if '*' `elem` prefixPat - then (file =~ prefixPat) ++ if arch `isInfixOf` prefixPat then "" else arch + then (file =~ prefixPat) ++ if showArch arch `isInfixOf` prefixPat then "" else showArch arch else prefixPat return prefix @@ -273,6 +295,7 @@ "rawhide" -> Just "Rawhide" "respin" -> Nothing "eln" -> Nothing + "c8s" -> Nothing "c9s" -> Nothing rel | all isDigit rel -> Just rel _ -> error' $ tgtrel ++ " is unsupported with --dryrun" @@ -302,15 +325,16 @@ urlPathMRel mgr = do let subdir = if edition `elem` fedoraSpins - then joinPath ["Spins", arch, "iso"] - else joinPath [showEdition edition, arch, editionMedia edition] + then joinPath ["Spins", showArch arch, "iso"] + else joinPath [showEdition edition, showArch arch, editionMedia edition] case tgtrel of "respin" -> return ("alt/live-respins", Nothing) "rawhide" -> return ("fedora/linux/development/rawhide" +/+ subdir, Just "Rawhide") "test" -> testRelease mgr subdir "stage" -> stageRelease mgr subdir - "eln" -> return ("production/latest-Fedora-ELN/compose" +/+ "Everything" +/+ arch +/+ "iso", Nothing) - "c9s" -> return ("production/latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ arch +/+ "iso", Nothing) + "eln" -> return ("production/latest-Fedora-ELN/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) + "c8s" -> return ("stream-8" +/+ showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) + "c9s" -> return (showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) rel | all isDigit rel -> released mgr rel subdir _ -> error' "Unknown release" @@ -358,15 +382,16 @@ makeFilePrefix mrelease = case tgtrel of "respin" -> "F[1-9][0-9]*-" <> liveRespin edition <> "-x86_64" <> "-LIVE" - "eln" -> "Fedora-ELN-Rawhide" + "eln" -> "Fedora-ELN" + "c8s" -> "CentOS-Stream-8" "c9s" -> "CentOS-Stream-9" _ -> let showRel r = if last r == '/' then init r else r rel = maybeToList (showRel <$> mrelease) middle = if edition `elem` [Cloud, Container] - then rel ++ [".*" <> arch] - else arch : rel + then rel ++ [".*" <> showArch arch] + else showArch arch : rel in intercalate "-" (["Fedora", showEdition edition, editionType edition] ++ middle) @@ -467,8 +492,9 @@ renderTime tz mprimeTime = "(" ++ maybe "" (show . utcToZonedTime tz) mprimeTime ++ ")" -downloadFile :: Bool -> Bool -> Manager -> URL -> Primary -> IO (Maybe Bool) -downloadFile dryrun done mgr url prime = do +downloadFile :: Bool -> Bool -> Bool -> Manager -> URL -> Primary -> String + -> IO (Maybe Bool) +downloadFile dryrun debug done mgr url prime showdestdir = do putStrLn url if done then return (Just False) @@ -483,11 +509,14 @@ putStrLn "Warning! Mirror timestamp differs from primary file" return mirrorTime else return $ primaryTime prime - if dryrun then return Nothing + if dryrun + then return Nothing else do tz <- getCurrentTimeZone - putStrLn $ unwords ["downloading", takeFileName url, renderTime tz mtime] - cmd_ "curl" ["-C", "-", "-O", url] + putStrLn $ unwords ["downloading", takeFileName url, renderTime tz mtime, "to", showdestdir] + let args = ["-C", "-", "-O", url] + when debug $ putStrLn $ unwords $ "curl" : "--output-dir" : showdestdir : args + cmd_ "curl" args return (Just True) editionType :: FedoraEdition -> String @@ -536,3 +565,32 @@ | head t == '/' = s +/+ tail t s +/+ t = s ++ "/" ++ t #endif + +-- derived from fedora-repoquery Types +data Arch = Source + | X86_64 + | AARCH64 + | S390X + | PPC64LE + deriving Eq + +eitherArch :: String -> Either String Arch +eitherArch s = + case lower s of + "source" -> Right Source + "x86_64" -> Right X86_64 + "aarch64" -> Right AARCH64 + "s390x" -> Right S390X + "ppc64le" -> Right PPC64LE + _ -> Left $ "unknown arch: " ++ s + +readArch :: String -> Arch +readArch = + either error' id . eitherArch + +showArch :: Arch -> String +showArch Source = "source" +showArch X86_64 = "x86_64" +showArch AARCH64 = "aarch64" +showArch S390X = "s390x" +showArch PPC64LE = "ppc64le" diff --git a/test/tests.hs b/test/tests.hs --- a/test/tests.hs +++ b/test/tests.hs @@ -9,7 +9,7 @@ -- FIXME automate me branched :: Int -branched = 38 +branched = 39 current, previous :: String current = show branched previous = show (branched - 1) @@ -28,6 +28,9 @@ ,["-n", previous, "kde"] ,["-T", "-n", current, "everything"] ,["-n", previous, "server", "--arch", "aarch64"] + ,["-n", "c8s"] + ,["-n", "c9s"] + ,["-n", "eln"] ] main :: IO ()