diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 1.1 (2024-05-23) +- add c10s and Fedora IoT edition +- print multiple matches and download latest +- '--dvd' option to select dvd image rather than boot/netinst iso +- handle empty mirror directory by falling back to dl.fp.o +- better debug output +- improve handling of empty checksum file +- filename tweaks for Container and Cloud + ## 1.0 (2023-09-15) - default to download.fp.o: replace --no-dl with --latest - new --check (-c) command mode: checks for newer image diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ `dl-fedora rawhide` : downloads the latest Fedora Rawhide Workstation Live iso -`dl-fedora 39 silverblue` : downloads the Fedora Silverblue iso +`dl-fedora 40 silverblue` : downloads the Fedora Silverblue iso `dl-fedora respin kde` : downloads the latest KDE Live respin -`dl-fedora 38 server --arch aarch64` : will download the Server iso for armv8 +`dl-fedora 39 server --arch aarch64` : will download the Server iso for armv8 -`dl-fedora --run 39` : will download Fedora Workstation and boot the Live image with qemu-kvm. +`dl-fedora --run 40` : will download Fedora Workstation and boot the Live image with qemu-kvm. `dl-fedora --check respin` : checks if there is a newer respin iso image available. @@ -50,25 +50,30 @@ It also tries to check the iso checksum and its gpg signature. ## Usage -```shellsession -$ dl-fedora --version -1.0 -$ dl-fedora --help +`$ dl-fedora --version` + +``` +1.1 +``` +`$ dl-fedora --help` + +``` Fedora iso downloader -Usage: dl-fedora [--version] [-g|--gpg-keys] [--no-checksum | --checksum] - [--debug] [-T|--no-http-timeout] - [(-c|--check) | (-l|--local) | (-R|--replace)] [-n|--dry-run] - [-r|--run] - [(-L|--latest) | (-d|--dl) | (-k|--koji) | (-m|--mirror URL)] - [--cs-devel | --cs-test] [-a|--arch ARCH] RELEASE [EDITION] +Usage: dl-fedora [--version] [-g|--gpg-keys] [--no-checksum | --checksum] + [--debug] [-T|--no-http-timeout] + [(-c|--check) | (-l|--local) | (-R|--replace)] [-n|--dry-run] + [-r|--run] + [(-L|--latest) | (-d|--dl) | (-k|--koji) | (-m|--mirror URL)] + [--dvd] [--cs-devel | --cs-test] [-a|--arch ARCH] RELEASE + [EDITION] Tool for downloading Fedora iso file images. - RELEASE = release number, respin, rawhide, test (Beta), stage (RC), eln, c8s, c9s + RELEASE = release number, respin, rawhide, test (Beta), stage (RC), eln, c9s, c10s EDITION = {cloud,container,everything,server,workstation,budgie,cinnamon,i3, - kde,lxde,lxqt,mate,soas,sway,xfce,silverblue,kinoite,onyx, - sericea} [default: workstation] - + kde,lxde,lxqt,mate,soas,sway,xfce,silverblue,kinoite,onyx,sericea, + iot} [default: workstation] + See Available options: @@ -91,6 +96,8 @@ -k,--koji Use koji.fedoraproject.org -m,--mirror URL Mirror url for /pub [default https://download.fedoraproject.org/pub] + --dvd Download dvd iso instead of boot netinst (for Server, + eln, centos) --cs-devel Use centos-stream development compose --cs-test Use centos-stream test compose (default is production) diff --git a/dl-fedora.cabal b/dl-fedora.cabal --- a/dl-fedora.cabal +++ b/dl-fedora.cabal @@ -1,6 +1,6 @@ cabal-version: 1.18 name: dl-fedora -version: 1.0 +version: 1.1 synopsis: Fedora image download tool description: Tool to download Fedora and Centos Stream iso and image files. @@ -11,14 +11,14 @@ bug-reports: https://github.com/juhp/dl-fedora/issues author: Jens Petersen maintainer: juhpetersen@gmail.com -copyright: 2019-2023 Jens Petersen +copyright: 2019-2024 Jens Petersen category: Utility build-type: Simple extra-doc-files: README.md , CHANGELOG.md -tested-with: GHC == 9.6.2, GHC == 9.4.7, GHC == 9.2.8, GHC == 9.0.2, - GHC == 8.10.7, GHC == 8.8.4, GHC == 8.6.5, GHC == 8.4.4, - GHC == 8.2.2 +tested-with: GHC == 8.2.2, GHC == 8.4.4, GHC == 8.6.5, GHC == 8.8.4, + GHC == 8.10.7, GHC == 9.0.2, GHC == 9.2.8, GHC == 9.4.8, + GHC == 9.6.5, GHC == 9.8.2 source-repository head type: git diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -28,7 +28,8 @@ import Paths_dl_fedora (version) -import SimpleCmd (cmd, cmd_, cmdN, error', grep_, pipe_, pipeBool, pipeFile_, +import SimpleCmd (cmd, cmd_, cmdLog_, cmdN, error', grep_, + pipe_, pipeBool, pipeFile_, sudoLog, warning, (+-+)) import SimpleCmdArgs import SimplePrompt (yesNo) @@ -70,6 +71,7 @@ | Kinoite | Onyx | Sericea + | IoT deriving (Show, Enum, Bounded, Eq) showEdition :: FedoraEdition -> String @@ -125,7 +127,7 @@ main = do let pdoc = Just $ P.vcat [ P.text "Tool for downloading Fedora iso file images.", - P.text ("RELEASE = " <> intercalate ", " ["release number", "respin", "rawhide", "test (Beta)", "stage (RC)", "eln", "c8s", "c9s"]), + P.text ("RELEASE = " <> intercalate ", " ["release number", "respin", "rawhide", "test (Beta)", "stage (RC)", "eln", "c9s", "c10s"]), P.text "EDITION = " <> P.lbrace <> P.align (P.fillCat (P.punctuate P.comma (map (P.text . lowerEdition) [(minBound :: FedoraEdition)..maxBound])) <> P.rbrace) <> P.text " [default: workstation]" , P.text "", P.text "See " @@ -143,6 +145,7 @@ <*> switchWith 'n' "dry-run" "Don't actually download anything" <*> switchWith 'r' "run" "Boot image in QEMU" <*> mirrorOpt + <*> switchLongWith "dvd" "Download dvd iso instead of boot netinst (for Server, eln, centos)" <*> (flagLongWith' CSDevelopment "cs-devel" "Use centos-stream development compose" <|> flagLongWith CSProduction CSTest "cs-test" "Use centos-stream test compose (default is production)") <*> (optionWith (eitherReader eitherArch) 'a' "arch" "ARCH" ("Specify arch [default:" +-+ showArch sysarch ++ "]") <|> pure sysarch) @@ -167,9 +170,9 @@ primaryTime :: Maybe UTCTime} program :: Bool -> CheckSum -> Bool -> Bool -> Mode -> Bool -> Bool - -> Mirror -> CentosChannel -> Arch -> String -> FedoraEdition + -> Mirror -> Bool -> CentosChannel -> Arch -> String -> FedoraEdition -> IO () -program gpg checksum debug notimeout mode dryrun run mirror channel arch tgtrel edition = do +program gpg checksum debug notimeout mode dryrun run mirror dvdnet channel arch tgtrel edition = do let mirrorUrl = case mirror of Mirror m -> m @@ -177,10 +180,10 @@ DlFpo -> dlFpo _ -- UseMirror or DefaultLatest | tgtrel == "eln" -> odcsFpo - | tgtrel `elem` ["c8s","c9s"] -> csComposes + | tgtrel `elem` ["c8s","c9s","c10s"] -> csComposes | otherwise -> downloadFpo showdestdir <- setDownloadDir dryrun "iso" - when debug $ putStrLn showdestdir + when debug $ print showdestdir mgr <- if notimeout then newManager (tlsManagerSettings {managerResponseTimeout = responseTimeoutNone}) else httpManager @@ -191,8 +194,7 @@ if done then do putStrLn $ "Local:" +-+ takeFileName fileurl +-+ "is latest" - when debug $ - putStrLn fileurl + when debug $ print fileurl else do let symlink = filenamePrefix <> (if tgtrel == "eln" then "-" <> showArch arch else "") <> "-latest" <.> takeExtension fileurl mtarget <- derefSymlink symlink @@ -229,55 +231,70 @@ -> IO (URL, String, Primary, Maybe String, Bool) findURL mgr mirrorUrl showdestdir quiet = do (path,mrelease) <- urlPathMRel mgr - -- use http-directory trailingSlash (0.1.7) - let primaryDir = - case tgtrel of - "koji" -> kojiPkgs - "eln" -> odcsFpo - "c8s" -> odcsStream - "c9s" -> odcsStream - _ -> if mirror `elem` [DefaultLatest, DlFpo] - then dlFpo - else downloadFpo - +/+ path <> "/" - hrefs <- httpDirectory mgr primaryDir + let path' = trailingSlash path + (primeDir,hrefs) <- + case tgtrel of + "koji" -> getUrlDirectory kojiPkgs path' + "eln" -> getUrlDirectory odcsFpo path' + "c8s" -> getUrlDirectory odcsStream path' + "c9s" -> getUrlDirectory odcsStream path' + "c10s" -> getUrlDirectory odcsStream path' + _ -> + if mirror `elem` [DefaultLatest, DlFpo] + then getUrlDirectory dlFpo path' + else do + (url,ls) <- getUrlDirectory downloadFpo path' + if null ls + then getUrlDirectory dlFpo path' + else return (url,ls) + when (null hrefs) $ error' $ primeDir +-+ "is empty" let prefixPat = makeFilePrefix mrelease selector = if '*' `elem` prefixPat then (=~ prefixPat) else (prefixPat `isPrefixOf`) - mfile = find selector $ map T.unpack hrefs + fileslen = groupSortOn length $ filter selector $ map T.unpack hrefs mchecksum = find ((if tgtrel == "respin" then T.isPrefixOf else T.isSuffixOf) (T.pack "CHECKSUM")) hrefs - case mfile of - Nothing -> - error' $ "no match for " <> prefixPat <> " in " <> primaryDir - Just file -> do - let prefix = if '*' `elem` prefixPat - then (file =~ prefixPat) ++ if showArch arch `isInfixOf` prefixPat then "" else showArch arch - else prefixPat - primeUrl = primaryDir +/+ file - (primeSize,primeTime) <- httpFileSizeTime mgr primeUrl - (finalurl, already) <- do - let localfile = takeFileName primeUrl - exists <- doesFileExist localfile - if exists - then do - done <- checkLocalFileSize localfile primeSize primeTime showdestdir quiet - if done - then return (primeUrl,True) - else do - unlessM (writable <$> getPermissions localfile) $ do - putStrLn $ localfile <> " does not have write permission!" - yes <- yesNo "Fix ownership" - if yes - then do - user <- getLoginName - sudoLog "chown" [user,localfile] - else - error' $ localfile <> " does have write permission, aborting!" - findMirror primeUrl path file - else findMirror primeUrl path file - let finalDir = dropFileName finalurl - return (finalurl, prefix, Primary primeUrl primeSize primeTime, - (finalDir +/+) . T.unpack <$> mchecksum, already) + if null fileslen + then error' $ "no match for " <> prefixPat <> " in " <> primeDir + else do + let files = head fileslen + when (length files > 1) $ mapM_ putStrLn files + let file = last files + prefix = if '*' `elem` prefixPat + then (file =~ prefixPat) ++ if showArch arch `isInfixOf` prefixPat then "" else showArch arch + else prefixPat + primeUrl = primeDir +/+ file + (primeSize,primeTime) <- httpFileSizeTime mgr primeUrl + (finalurl, already) <- do + let localfile = takeFileName primeUrl + exists <- doesFileExist localfile + if exists + then do + localsize <- toInteger . fileSize <$> getFileStatus localfile + done <- checkLocalFileSize localsize localfile primeSize primeTime showdestdir quiet + if done + then return (primeUrl,True) + else do + unlessM (writable <$> getPermissions localfile) $ do + putStrLn $ localfile <> " does not have write permission!" + yes <- yesNo "Fix ownership" + if yes + then do + user <- getLoginName + sudoLog "chown" [user,localfile] + else + error' $ localfile <> " does have write permission, aborting!" + findMirror primeUrl path file + else findMirror primeUrl path file + let finalDir = dropFileName finalurl + return (finalurl, prefix, Primary primeUrl primeSize primeTime, + (finalDir +/+) . T.unpack <$> mchecksum, already) where + getUrlDirectory :: String -> FilePath -> IO (String, [T.Text]) + getUrlDirectory top path = do + let url = top +/+ path + when debug $ print url + ls <- httpDirectory mgr url + return (url, ls) + findMirror primeUrl path file = do url <- if mirrorUrl `elem` [dlFpo,kojiPkgs,odcsFpo] @@ -294,6 +311,7 @@ Just u -> do let url = B.unpack u exists <- httpExists mgr url + when debug $ print (url, exists) if exists then return url else return primeUrl return (url,False) @@ -317,6 +335,7 @@ getRelease :: Maybe String getRelease = case tgtrel of + -- FIXME IoT uses version for instead of Rawhide "rawhide" -> Just "Rawhide" "respin" -> Nothing "eln" -> Nothing @@ -325,10 +344,9 @@ rel | all isDigit rel -> Just rel _ -> error' $ tgtrel ++ " is unsupported with --dryrun" - checkLocalFileSize :: FilePath -> Maybe Integer -> Maybe UTCTime + checkLocalFileSize :: Integer -> FilePath -> Maybe Integer -> Maybe UTCTime -> String -> Bool -> IO Bool - checkLocalFileSize localfile mprimeSize mprimeTime showdestdir quiet = do - localsize <- toInteger . fileSize <$> getFileStatus localfile + checkLocalFileSize localsize localfile mprimeSize mprimeTime showdestdir quiet = do if Just localsize == mprimeSize then do unless quiet $ do @@ -352,16 +370,20 @@ if edition `elem` fedoraSpins then joinPath ["Spins", showArch arch, "iso"] else joinPath [showEdition edition, showArch arch, editionMedia edition] - case tgtrel of - "respin" -> return ("alt/live-respins", Nothing) - "rawhide" -> return ("fedora/linux/development/rawhide" +/+ subdir, Just "Rawhide") - "test" -> testRelease mgr subdir - "stage" -> stageRelease mgr subdir - "eln" -> return ("production/latest-Fedora-ELN/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) - "c8s" -> return ("stream-8" +/+ showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) - "c9s" -> return (showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) - rel | all isDigit rel -> released mgr rel subdir - _ -> error' "Unknown release" + if edition == IoT && isFedora tgtrel + then return ("alt/iot" +/+ tgtrel +/+ subdir, Nothing) + else + case tgtrel of + "respin" -> return ("alt/live-respins", Nothing) + "rawhide" -> return ("fedora/linux/development/rawhide" +/+ subdir, Just "Rawhide") + "test" -> testRelease mgr subdir + "stage" -> stageRelease mgr subdir + "eln" -> return ("production/latest-Fedora-ELN/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) + "c8s" -> return ("stream-8" +/+ showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) + "c9s" -> return (showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) + "c10s" -> return ("stream-10" +/+ showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing) + rel | all isDigit rel -> released mgr rel subdir + _ -> error' "Unknown release" testRelease :: Manager -> FilePath -> IO (FilePath, Maybe String) testRelease mgr subdir = do @@ -375,7 +397,6 @@ stageRelease mgr subdir = do let path = "alt/stage" url = dlFpo +/+ path - -- use http-directory-0.1.7 noTrailingSlash rels <- reverse . map (T.unpack . noTrailingSlash) <$> httpDirectory mgr url let mrel = listToMaybe rels return (path +/+ fromMaybe (error' ("staged release not found in " <> url)) mrel +/+ subdir, takeWhile (/= '_') <$> mrel) @@ -395,31 +416,52 @@ let dir = "fedora/linux/releases" url = dlFpo +/+ dir exists <- httpExists mgr $ url +/+ rel - if exists then return (dir +/+ rel +/+ subdir, Just rel) + when debug $ print (exists, url +/+ rel) + if exists + then return (dir +/+ rel +/+ subdir, Just rel) else do let dir' = "fedora/linux/development" url' = dlFpo +/+ dir' exists' <- httpExists mgr $ url' +/+ rel - if exists' then return (dir' +/+ rel +/+ subdir, Just rel) + if exists' + then return (dir' +/+ rel +/+ subdir, Just rel) else error' "release not found in releases/ or development/" + renderdvdboot = if dvdnet then "dvd1" else "boot" + makeFilePrefix :: Maybe String -> String makeFilePrefix mrelease = case tgtrel of "respin" -> "F[1-9][0-9]*-" <> liveRespin edition <> "-x86_64" <> "-LIVE" - "eln" -> "Fedora-ELN" - "c8s" -> "CentOS-Stream-8" - "c9s" -> "CentOS-Stream-9" + "eln" -> "Fedora-ELN-.*" ++ renderdvdboot + "c8s" -> "CentOS-Stream-8-.*" ++ renderdvdboot + "c9s" -> "CentOS-Stream-9-.*" ++ renderdvdboot + "c10s" -> "CentOS-Stream-10-.*" ++ renderdvdboot _ -> let showRel r = if last r == '/' then init r else r rel = maybeToList (showRel <$> mrelease) - middle = - if edition `elem` [Cloud, Container] - then rel ++ [".*" <> showArch arch] - else showArch arch : rel + (midpref,middle) = + case edition of + -- https://github.com/fedora-iot/iot-distro/issues/1 + IoT -> ("", rel ++ [".*" <> showArch arch]) + Cloud -> ('.' : showArch arch, rel) + Container -> ('.' : showArch arch, rel) + _ -> ("", showArch arch : rel) in - intercalate "-" (["Fedora", showEdition edition, editionType edition] ++ middle) + intercalate "-" (["Fedora", showEdition edition, editionType edition ++ midpref] ++ middle) + editionType :: FedoraEdition -> String + editionType Server = if dvdnet then "dvd" else "netinst" + editionType IoT = "ostree" + editionType Kinoite = "ostree" + editionType Onyx = "ostree" + editionType Sericea = "ostree" + editionType Silverblue = "ostree" + editionType Everything = "netinst" + editionType Cloud = "Base-Generic" + editionType Container = "Base-Generic" + editionType _ = "Live" + fileChecksum :: Manager -> Maybe URL -> String -> Maybe Bool -> IO () fileChecksum _ Nothing _ _ = return () fileChecksum mgr (Just url) showdestdir mneedChecksum = @@ -432,11 +474,17 @@ then checkChecksumfile mgr url checksumfile showdestdir else createDirectory checksumdir >> return False putStrLn "" - unless exists $ - whenM (httpExists mgr url) $ - withCurrentDirectory checksumdir $ - cmd_ "curl" ["-C", "-", "-s", "-S", "-O", url] - haveChksum <- doesFileExist checksumfile + haveChksum <- + if exists + then return True + else do + remoteExists <- httpExists mgr url + if remoteExists + then do + withCurrentDirectory checksumdir $ + curl debug $ (if debug then ["--silent", "--show-error"] else []) ++ ["--remote-name", url] + doesFileExist checksumfile + else return False if not haveChksum then putStrLn "No checksum file found" else do @@ -446,7 +494,7 @@ unless havekey $ do putStrLn "Importing Fedora GPG keys:\n" -- https://fedoramagazine.org/verify-fedora-iso-file/ - pipe_ ("curl",["-s", "-S", "https://getfedora.org/static/fedora.gpg"]) ("gpg",["--import"]) + pipe_ ("curl",["--silent", "--show-error", "https://getfedora.org/static/fedora.gpg"]) ("gpg",["--import"]) putStrLn "" chkgpg <- if pgp then checkForFedoraKeys @@ -466,8 +514,11 @@ exists <- doesFileExist checksumfile if not exists then return False else do + filesize <- toInteger . fileSize <$> getFileStatus checksumfile + when (filesize == 0) $ error' $ checksumfile +-+ "empty!" (primeSize,primeTime) <- httpFileSizeTime mgr url - ok <- checkLocalFileSize checksumfile primeSize primeTime showdestdir True + when debug $ print (primeSize,primeTime,url) + ok <- checkLocalFileSize filesize checksumfile primeSize primeTime showdestdir True unless ok $ error' $ "Checksum file filesize mismatch for " ++ checksumfile return ok @@ -515,7 +566,7 @@ downloadFile :: Bool -> Bool -> Bool -> Manager -> URL -> Primary -> String -> IO (Maybe Bool) downloadFile dryrun debug done mgr url prime showdestdir = do - putStrLn url + unless debug $ putStrLn url if done then return (Just False) else do @@ -534,22 +585,9 @@ else do tz <- getCurrentTimeZone putStrLn $ unwords ["downloading", takeFileName url, renderTime tz mtime, "to", showdestdir] - let args = ["-C", "-", "-O", url] - when debug $ putStrLn $ unwords $ "curl" : "--output-dir" : showdestdir : args - cmd_ "curl" args + curl debug ["--fail", "--remote-name", url] return (Just True) -editionType :: FedoraEdition -> String -editionType Server = "dvd" -editionType Kinoite = "ostree" -editionType Onyx = "ostree" -editionType Sericea = "ostree" -editionType Silverblue = "ostree" -editionType Everything = "netinst" -editionType Cloud = "Base" -editionType Container = "Base" -editionType _ = "Live" - editionMedia :: FedoraEdition -> String editionMedia Cloud = "images" editionMedia Container = "images" @@ -616,3 +654,10 @@ showArch AARCH64 = "aarch64" showArch S390X = "s390x" showArch PPC64LE = "ppc64le" + +isFedora :: String -> Bool +isFedora tgtrel = tgtrel == "rawhide" || all isDigit tgtrel + +curl :: Bool -> [String] -> IO () +curl debug args = + (if debug then cmdLog_ else cmd_) "curl" $ ["--location", "--continue-at", "-"] ++ args diff --git a/test/tests.hs b/test/tests.hs --- a/test/tests.hs +++ b/test/tests.hs @@ -9,7 +9,7 @@ -- FIXME automate me branched :: Int -branched = 39 +branched = 40 current, previous :: String current = show branched previous = show (branched - 1) @@ -21,15 +21,17 @@ ,["-c", "respin"] ,["-l", current] ,["-l", "rawhide", "-n"] + ,["40", "iot", "-n"] ] ++ if ghAction then [] else [["-c", current, "silverblue"] ,["-n", previous, "kde"] ,["-T", "-n", current, "everything"] - ,["-n", previous, "server", "--arch", "aarch64"] + ,["-n", previous, "server", "--dvd", "--arch", "aarch64"] ,["-n", "c8s"] - ,["-n", "c9s"] + ,["-n", "c9s", "--dvd"] + ,["-n", "c10s"] ,["-c", "eln"] ]