dl-fedora 1.0 → 1.1
raw patch · 5 files changed
+187/−124 lines, 5 files
Files
- CHANGELOG.md +9/−0
- README.md +24/−17
- dl-fedora.cabal +5/−5
- src/Main.hs +144/−99
- test/tests.hs +5/−3
CHANGELOG.md view
@@ -1,5 +1,14 @@ # Changelog +## 1.1 (2024-05-23)+- add c10s and Fedora IoT edition+- print multiple matches and download latest+- '--dvd' option to select dvd image rather than boot/netinst iso+- handle empty mirror directory by falling back to dl.fp.o+- better debug output+- improve handling of empty checksum file+- filename tweaks for Container and Cloud+ ## 1.0 (2023-09-15) - default to download.fp.o: replace --no-dl with --latest - new --check (-c) command mode: checks for newer image
README.md view
@@ -12,13 +12,13 @@ `dl-fedora rawhide` : downloads the latest Fedora Rawhide Workstation Live iso -`dl-fedora 39 silverblue` : downloads the Fedora Silverblue iso+`dl-fedora 40 silverblue` : downloads the Fedora Silverblue iso `dl-fedora respin kde` : downloads the latest KDE Live respin -`dl-fedora 38 server --arch aarch64` : will download the Server iso for armv8+`dl-fedora 39 server --arch aarch64` : will download the Server iso for armv8 -`dl-fedora --run 39` : will download Fedora Workstation and boot the Live image with qemu-kvm.+`dl-fedora --run 40` : will download Fedora Workstation and boot the Live image with qemu-kvm. `dl-fedora --check respin` : checks if there is a newer respin iso image available.@@ -50,25 +50,30 @@ It also tries to check the iso checksum and its gpg signature. ## Usage-```shellsession-$ dl-fedora --version-1.0-$ dl-fedora --help+`$ dl-fedora --version`++```+1.1+```+`$ dl-fedora --help`++``` Fedora iso downloader -Usage: dl-fedora [--version] [-g|--gpg-keys] [--no-checksum | --checksum]- [--debug] [-T|--no-http-timeout]- [(-c|--check) | (-l|--local) | (-R|--replace)] [-n|--dry-run]- [-r|--run]- [(-L|--latest) | (-d|--dl) | (-k|--koji) | (-m|--mirror URL)]- [--cs-devel | --cs-test] [-a|--arch ARCH] RELEASE [EDITION]+Usage: dl-fedora [--version] [-g|--gpg-keys] [--no-checksum | --checksum] + [--debug] [-T|--no-http-timeout] + [(-c|--check) | (-l|--local) | (-R|--replace)] [-n|--dry-run] + [-r|--run] + [(-L|--latest) | (-d|--dl) | (-k|--koji) | (-m|--mirror URL)] + [--dvd] [--cs-devel | --cs-test] [-a|--arch ARCH] RELEASE + [EDITION] Tool for downloading Fedora iso file images.- RELEASE = release number, respin, rawhide, test (Beta), stage (RC), eln, c8s, c9s+ RELEASE = release number, respin, rawhide, test (Beta), stage (RC), eln, c9s, c10s EDITION = {cloud,container,everything,server,workstation,budgie,cinnamon,i3,- kde,lxde,lxqt,mate,soas,sway,xfce,silverblue,kinoite,onyx,- sericea} [default: workstation]-+ kde,lxde,lxqt,mate,soas,sway,xfce,silverblue,kinoite,onyx,sericea,+ iot} [default: workstation]+ See <https://github.com/juhp/dl-fedora/#readme> Available options:@@ -91,6 +96,8 @@ -k,--koji Use koji.fedoraproject.org -m,--mirror URL Mirror url for /pub [default https://download.fedoraproject.org/pub]+ --dvd Download dvd iso instead of boot netinst (for Server,+ eln, centos) --cs-devel Use centos-stream development compose --cs-test Use centos-stream test compose (default is production)
dl-fedora.cabal view
@@ -1,6 +1,6 @@ cabal-version: 1.18 name: dl-fedora-version: 1.0+version: 1.1 synopsis: Fedora image download tool description: Tool to download Fedora and Centos Stream iso and image files.@@ -11,14 +11,14 @@ bug-reports: https://github.com/juhp/dl-fedora/issues author: Jens Petersen maintainer: juhpetersen@gmail.com-copyright: 2019-2023 Jens Petersen+copyright: 2019-2024 Jens Petersen category: Utility build-type: Simple extra-doc-files: README.md , CHANGELOG.md-tested-with: GHC == 9.6.2, GHC == 9.4.7, GHC == 9.2.8, GHC == 9.0.2,- GHC == 8.10.7, GHC == 8.8.4, GHC == 8.6.5, GHC == 8.4.4,- GHC == 8.2.2+tested-with: GHC == 8.2.2, GHC == 8.4.4, GHC == 8.6.5, GHC == 8.8.4,+ GHC == 8.10.7, GHC == 9.0.2, GHC == 9.2.8, GHC == 9.4.8,+ GHC == 9.6.5, GHC == 9.8.2 source-repository head type: git
src/Main.hs view
@@ -28,7 +28,8 @@ import Paths_dl_fedora (version) -import SimpleCmd (cmd, cmd_, cmdN, error', grep_, pipe_, pipeBool, pipeFile_,+import SimpleCmd (cmd, cmd_, cmdLog_, cmdN, error', grep_,+ pipe_, pipeBool, pipeFile_, sudoLog, warning, (+-+)) import SimpleCmdArgs import SimplePrompt (yesNo)@@ -70,6 +71,7 @@ | Kinoite | Onyx | Sericea+ | IoT deriving (Show, Enum, Bounded, Eq) showEdition :: FedoraEdition -> String@@ -125,7 +127,7 @@ main = do let pdoc = Just $ P.vcat [ P.text "Tool for downloading Fedora iso file images.",- P.text ("RELEASE = " <> intercalate ", " ["release number", "respin", "rawhide", "test (Beta)", "stage (RC)", "eln", "c8s", "c9s"]),+ P.text ("RELEASE = " <> intercalate ", " ["release number", "respin", "rawhide", "test (Beta)", "stage (RC)", "eln", "c9s", "c10s"]), P.text "EDITION = " <> P.lbrace <> P.align (P.fillCat (P.punctuate P.comma (map (P.text . lowerEdition) [(minBound :: FedoraEdition)..maxBound])) <> P.rbrace) <> P.text " [default: workstation]" , P.text "", P.text "See <https://github.com/juhp/dl-fedora/#readme>"@@ -143,6 +145,7 @@ <*> switchWith 'n' "dry-run" "Don't actually download anything" <*> switchWith 'r' "run" "Boot image in QEMU" <*> mirrorOpt+ <*> switchLongWith "dvd" "Download dvd iso instead of boot netinst (for Server, eln, centos)" <*> (flagLongWith' CSDevelopment "cs-devel" "Use centos-stream development compose" <|> flagLongWith CSProduction CSTest "cs-test" "Use centos-stream test compose (default is production)") <*> (optionWith (eitherReader eitherArch) 'a' "arch" "ARCH" ("Specify arch [default:" +-+ showArch sysarch ++ "]") <|> pure sysarch)@@ -167,9 +170,9 @@ primaryTime :: Maybe UTCTime} program :: Bool -> CheckSum -> Bool -> Bool -> Mode -> Bool -> Bool- -> Mirror -> CentosChannel -> Arch -> String -> FedoraEdition+ -> Mirror -> Bool -> CentosChannel -> Arch -> String -> FedoraEdition -> IO ()-program gpg checksum debug notimeout mode dryrun run mirror channel arch tgtrel edition = do+program gpg checksum debug notimeout mode dryrun run mirror dvdnet channel arch tgtrel edition = do let mirrorUrl = case mirror of Mirror m -> m@@ -177,10 +180,10 @@ DlFpo -> dlFpo _ -- UseMirror or DefaultLatest | tgtrel == "eln" -> odcsFpo- | tgtrel `elem` ["c8s","c9s"] -> csComposes+ | tgtrel `elem` ["c8s","c9s","c10s"] -> csComposes | otherwise -> downloadFpo showdestdir <- setDownloadDir dryrun "iso"- when debug $ putStrLn showdestdir+ when debug $ print showdestdir mgr <- if notimeout then newManager (tlsManagerSettings {managerResponseTimeout = responseTimeoutNone}) else httpManager@@ -191,8 +194,7 @@ if done then do putStrLn $ "Local:" +-+ takeFileName fileurl +-+ "is latest"- when debug $- putStrLn fileurl+ when debug $ print fileurl else do let symlink = filenamePrefix <> (if tgtrel == "eln" then "-" <> showArch arch else "") <> "-latest" <.> takeExtension fileurl mtarget <- derefSymlink symlink@@ -229,55 +231,70 @@ -> IO (URL, String, Primary, Maybe String, Bool) findURL mgr mirrorUrl showdestdir quiet = do (path,mrelease) <- urlPathMRel mgr- -- use http-directory trailingSlash (0.1.7)- let primaryDir =- case tgtrel of- "koji" -> kojiPkgs- "eln" -> odcsFpo- "c8s" -> odcsStream- "c9s" -> odcsStream- _ -> if mirror `elem` [DefaultLatest, DlFpo]- then dlFpo- else downloadFpo- +/+ path <> "/"- hrefs <- httpDirectory mgr primaryDir+ let path' = trailingSlash path+ (primeDir,hrefs) <-+ case tgtrel of+ "koji" -> getUrlDirectory kojiPkgs path'+ "eln" -> getUrlDirectory odcsFpo path'+ "c8s" -> getUrlDirectory odcsStream path'+ "c9s" -> getUrlDirectory odcsStream path'+ "c10s" -> getUrlDirectory odcsStream path'+ _ ->+ if mirror `elem` [DefaultLatest, DlFpo]+ then getUrlDirectory dlFpo path'+ else do+ (url,ls) <- getUrlDirectory downloadFpo path'+ if null ls+ then getUrlDirectory dlFpo path'+ else return (url,ls)+ when (null hrefs) $ error' $ primeDir +-+ "is empty" let prefixPat = makeFilePrefix mrelease selector = if '*' `elem` prefixPat then (=~ prefixPat) else (prefixPat `isPrefixOf`)- mfile = find selector $ map T.unpack hrefs+ fileslen = groupSortOn length $ filter selector $ map T.unpack hrefs mchecksum = find ((if tgtrel == "respin" then T.isPrefixOf else T.isSuffixOf) (T.pack "CHECKSUM")) hrefs- case mfile of- Nothing ->- error' $ "no match for " <> prefixPat <> " in " <> primaryDir- Just file -> do- let prefix = if '*' `elem` prefixPat- then (file =~ prefixPat) ++ if showArch arch `isInfixOf` prefixPat then "" else showArch arch- else prefixPat- primeUrl = primaryDir +/+ file- (primeSize,primeTime) <- httpFileSizeTime mgr primeUrl- (finalurl, already) <- do- let localfile = takeFileName primeUrl- exists <- doesFileExist localfile- if exists- then do- done <- checkLocalFileSize localfile primeSize primeTime showdestdir quiet- if done- then return (primeUrl,True)- else do- unlessM (writable <$> getPermissions localfile) $ do- putStrLn $ localfile <> " does not have write permission!"- yes <- yesNo "Fix ownership"- if yes- then do- user <- getLoginName- sudoLog "chown" [user,localfile]- else- error' $ localfile <> " does have write permission, aborting!"- findMirror primeUrl path file- else findMirror primeUrl path file- let finalDir = dropFileName finalurl- return (finalurl, prefix, Primary primeUrl primeSize primeTime,- (finalDir +/+) . T.unpack <$> mchecksum, already)+ if null fileslen+ then error' $ "no match for " <> prefixPat <> " in " <> primeDir+ else do+ let files = head fileslen+ when (length files > 1) $ mapM_ putStrLn files+ let file = last files+ prefix = if '*' `elem` prefixPat+ then (file =~ prefixPat) ++ if showArch arch `isInfixOf` prefixPat then "" else showArch arch+ else prefixPat+ primeUrl = primeDir +/+ file+ (primeSize,primeTime) <- httpFileSizeTime mgr primeUrl+ (finalurl, already) <- do+ let localfile = takeFileName primeUrl+ exists <- doesFileExist localfile+ if exists+ then do+ localsize <- toInteger . fileSize <$> getFileStatus localfile+ done <- checkLocalFileSize localsize localfile primeSize primeTime showdestdir quiet+ if done+ then return (primeUrl,True)+ else do+ unlessM (writable <$> getPermissions localfile) $ do+ putStrLn $ localfile <> " does not have write permission!"+ yes <- yesNo "Fix ownership"+ if yes+ then do+ user <- getLoginName+ sudoLog "chown" [user,localfile]+ else+ error' $ localfile <> " does have write permission, aborting!"+ findMirror primeUrl path file+ else findMirror primeUrl path file+ let finalDir = dropFileName finalurl+ return (finalurl, prefix, Primary primeUrl primeSize primeTime,+ (finalDir +/+) . T.unpack <$> mchecksum, already) where+ getUrlDirectory :: String -> FilePath -> IO (String, [T.Text])+ getUrlDirectory top path = do+ let url = top +/+ path+ when debug $ print url+ ls <- httpDirectory mgr url+ return (url, ls)+ findMirror primeUrl path file = do url <- if mirrorUrl `elem` [dlFpo,kojiPkgs,odcsFpo]@@ -294,6 +311,7 @@ Just u -> do let url = B.unpack u exists <- httpExists mgr url+ when debug $ print (url, exists) if exists then return url else return primeUrl return (url,False)@@ -317,6 +335,7 @@ getRelease :: Maybe String getRelease = case tgtrel of+ -- FIXME IoT uses version for instead of Rawhide "rawhide" -> Just "Rawhide" "respin" -> Nothing "eln" -> Nothing@@ -325,10 +344,9 @@ rel | all isDigit rel -> Just rel _ -> error' $ tgtrel ++ " is unsupported with --dryrun" - checkLocalFileSize :: FilePath -> Maybe Integer -> Maybe UTCTime+ checkLocalFileSize :: Integer -> FilePath -> Maybe Integer -> Maybe UTCTime -> String -> Bool -> IO Bool- checkLocalFileSize localfile mprimeSize mprimeTime showdestdir quiet = do- localsize <- toInteger . fileSize <$> getFileStatus localfile+ checkLocalFileSize localsize localfile mprimeSize mprimeTime showdestdir quiet = do if Just localsize == mprimeSize then do unless quiet $ do@@ -352,16 +370,20 @@ if edition `elem` fedoraSpins then joinPath ["Spins", showArch arch, "iso"] else joinPath [showEdition edition, showArch arch, editionMedia edition]- case tgtrel of- "respin" -> return ("alt/live-respins", Nothing)- "rawhide" -> return ("fedora/linux/development/rawhide" +/+ subdir, Just "Rawhide")- "test" -> testRelease mgr subdir- "stage" -> stageRelease mgr subdir- "eln" -> return ("production/latest-Fedora-ELN/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing)- "c8s" -> return ("stream-8" +/+ showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing)- "c9s" -> return (showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing)- rel | all isDigit rel -> released mgr rel subdir- _ -> error' "Unknown release"+ if edition == IoT && isFedora tgtrel+ then return ("alt/iot" +/+ tgtrel +/+ subdir, Nothing)+ else+ case tgtrel of+ "respin" -> return ("alt/live-respins", Nothing)+ "rawhide" -> return ("fedora/linux/development/rawhide" +/+ subdir, Just "Rawhide")+ "test" -> testRelease mgr subdir+ "stage" -> stageRelease mgr subdir+ "eln" -> return ("production/latest-Fedora-ELN/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing)+ "c8s" -> return ("stream-8" +/+ showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing)+ "c9s" -> return (showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing)+ "c10s" -> return ("stream-10" +/+ showChannel channel +/+ "latest-CentOS-Stream/compose" +/+ "BaseOS" +/+ showArch arch +/+ "iso", Nothing)+ rel | all isDigit rel -> released mgr rel subdir+ _ -> error' "Unknown release" testRelease :: Manager -> FilePath -> IO (FilePath, Maybe String) testRelease mgr subdir = do@@ -375,7 +397,6 @@ stageRelease mgr subdir = do let path = "alt/stage" url = dlFpo +/+ path- -- use http-directory-0.1.7 noTrailingSlash rels <- reverse . map (T.unpack . noTrailingSlash) <$> httpDirectory mgr url let mrel = listToMaybe rels return (path +/+ fromMaybe (error' ("staged release not found in " <> url)) mrel +/+ subdir, takeWhile (/= '_') <$> mrel)@@ -395,31 +416,52 @@ let dir = "fedora/linux/releases" url = dlFpo +/+ dir exists <- httpExists mgr $ url +/+ rel- if exists then return (dir +/+ rel +/+ subdir, Just rel)+ when debug $ print (exists, url +/+ rel)+ if exists+ then return (dir +/+ rel +/+ subdir, Just rel) else do let dir' = "fedora/linux/development" url' = dlFpo +/+ dir' exists' <- httpExists mgr $ url' +/+ rel- if exists' then return (dir' +/+ rel +/+ subdir, Just rel)+ if exists'+ then return (dir' +/+ rel +/+ subdir, Just rel) else error' "release not found in releases/ or development/" + renderdvdboot = if dvdnet then "dvd1" else "boot"+ makeFilePrefix :: Maybe String -> String makeFilePrefix mrelease = case tgtrel of "respin" -> "F[1-9][0-9]*-" <> liveRespin edition <> "-x86_64" <> "-LIVE"- "eln" -> "Fedora-ELN"- "c8s" -> "CentOS-Stream-8"- "c9s" -> "CentOS-Stream-9"+ "eln" -> "Fedora-ELN-.*" ++ renderdvdboot+ "c8s" -> "CentOS-Stream-8-.*" ++ renderdvdboot+ "c9s" -> "CentOS-Stream-9-.*" ++ renderdvdboot+ "c10s" -> "CentOS-Stream-10-.*" ++ renderdvdboot _ -> let showRel r = if last r == '/' then init r else r rel = maybeToList (showRel <$> mrelease)- middle =- if edition `elem` [Cloud, Container]- then rel ++ [".*" <> showArch arch]- else showArch arch : rel+ (midpref,middle) =+ case edition of+ -- https://github.com/fedora-iot/iot-distro/issues/1+ IoT -> ("", rel ++ [".*" <> showArch arch])+ Cloud -> ('.' : showArch arch, rel)+ Container -> ('.' : showArch arch, rel)+ _ -> ("", showArch arch : rel) in- intercalate "-" (["Fedora", showEdition edition, editionType edition] ++ middle)+ intercalate "-" (["Fedora", showEdition edition, editionType edition ++ midpref] ++ middle) + editionType :: FedoraEdition -> String+ editionType Server = if dvdnet then "dvd" else "netinst"+ editionType IoT = "ostree"+ editionType Kinoite = "ostree"+ editionType Onyx = "ostree"+ editionType Sericea = "ostree"+ editionType Silverblue = "ostree"+ editionType Everything = "netinst"+ editionType Cloud = "Base-Generic"+ editionType Container = "Base-Generic"+ editionType _ = "Live"+ fileChecksum :: Manager -> Maybe URL -> String -> Maybe Bool -> IO () fileChecksum _ Nothing _ _ = return () fileChecksum mgr (Just url) showdestdir mneedChecksum =@@ -432,11 +474,17 @@ then checkChecksumfile mgr url checksumfile showdestdir else createDirectory checksumdir >> return False putStrLn ""- unless exists $- whenM (httpExists mgr url) $- withCurrentDirectory checksumdir $- cmd_ "curl" ["-C", "-", "-s", "-S", "-O", url]- haveChksum <- doesFileExist checksumfile+ haveChksum <-+ if exists+ then return True+ else do+ remoteExists <- httpExists mgr url+ if remoteExists+ then do+ withCurrentDirectory checksumdir $+ curl debug $ (if debug then ["--silent", "--show-error"] else []) ++ ["--remote-name", url]+ doesFileExist checksumfile+ else return False if not haveChksum then putStrLn "No checksum file found" else do@@ -446,7 +494,7 @@ unless havekey $ do putStrLn "Importing Fedora GPG keys:\n" -- https://fedoramagazine.org/verify-fedora-iso-file/- pipe_ ("curl",["-s", "-S", "https://getfedora.org/static/fedora.gpg"]) ("gpg",["--import"])+ pipe_ ("curl",["--silent", "--show-error", "https://getfedora.org/static/fedora.gpg"]) ("gpg",["--import"]) putStrLn "" chkgpg <- if pgp then checkForFedoraKeys@@ -466,8 +514,11 @@ exists <- doesFileExist checksumfile if not exists then return False else do+ filesize <- toInteger . fileSize <$> getFileStatus checksumfile+ when (filesize == 0) $ error' $ checksumfile +-+ "empty!" (primeSize,primeTime) <- httpFileSizeTime mgr url- ok <- checkLocalFileSize checksumfile primeSize primeTime showdestdir True+ when debug $ print (primeSize,primeTime,url)+ ok <- checkLocalFileSize filesize checksumfile primeSize primeTime showdestdir True unless ok $ error' $ "Checksum file filesize mismatch for " ++ checksumfile return ok @@ -515,7 +566,7 @@ downloadFile :: Bool -> Bool -> Bool -> Manager -> URL -> Primary -> String -> IO (Maybe Bool) downloadFile dryrun debug done mgr url prime showdestdir = do- putStrLn url+ unless debug $ putStrLn url if done then return (Just False) else do@@ -534,22 +585,9 @@ else do tz <- getCurrentTimeZone putStrLn $ unwords ["downloading", takeFileName url, renderTime tz mtime, "to", showdestdir]- let args = ["-C", "-", "-O", url]- when debug $ putStrLn $ unwords $ "curl" : "--output-dir" : showdestdir : args- cmd_ "curl" args+ curl debug ["--fail", "--remote-name", url] return (Just True) -editionType :: FedoraEdition -> String-editionType Server = "dvd"-editionType Kinoite = "ostree"-editionType Onyx = "ostree"-editionType Sericea = "ostree"-editionType Silverblue = "ostree"-editionType Everything = "netinst"-editionType Cloud = "Base"-editionType Container = "Base"-editionType _ = "Live"- editionMedia :: FedoraEdition -> String editionMedia Cloud = "images" editionMedia Container = "images"@@ -616,3 +654,10 @@ showArch AARCH64 = "aarch64" showArch S390X = "s390x" showArch PPC64LE = "ppc64le"++isFedora :: String -> Bool+isFedora tgtrel = tgtrel == "rawhide" || all isDigit tgtrel++curl :: Bool -> [String] -> IO ()+curl debug args =+ (if debug then cmdLog_ else cmd_) "curl" $ ["--location", "--continue-at", "-"] ++ args
test/tests.hs view
@@ -9,7 +9,7 @@ -- FIXME automate me branched :: Int-branched = 39+branched = 40 current, previous :: String current = show branched previous = show (branched - 1)@@ -21,15 +21,17 @@ ,["-c", "respin"] ,["-l", current] ,["-l", "rawhide", "-n"]+ ,["40", "iot", "-n"] ] ++ if ghAction then [] else [["-c", current, "silverblue"] ,["-n", previous, "kde"] ,["-T", "-n", current, "everything"]- ,["-n", previous, "server", "--arch", "aarch64"]+ ,["-n", previous, "server", "--dvd", "--arch", "aarch64"] ,["-n", "c8s"]- ,["-n", "c9s"]+ ,["-n", "c9s", "--dvd"]+ ,["-n", "c10s"] ,["-c", "eln"] ]