oglyad-0.1.6.0: src/Oglyad/Source/Repodata.hs
module Oglyad.Source.Repodata
( RpmEntry (..),
fetchRepo,
decompressAndParse,
parseMetalink,
parseRepomd,
parsePrimary,
srpmName,
latestBySource,
)
where
import Conduit (foldlC, runConduit, sourceHandle, sourceLazy, (.|))
import Control.DeepSeq (deepseq)
import Control.Exception (SomeException, try)
import Control.Monad ((<$!>), (>=>))
import Control.Monad.Reader (liftIO)
import Crypto.Hash.SHA256 qualified as SHA256
import Data.Bifunctor (first)
import Data.ByteString.Base16 qualified as B16
import Data.ByteString.Lazy qualified as LBS
import Data.Char (isAlphaNum)
import Data.List (sortOn)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (mapMaybe)
import Data.Ord (Down (..))
import Data.Text (Text)
import Data.Text qualified as T
import Data.Text.Encoding qualified as TE
import Data.Text.Read qualified as TR
import Data.Time (UTCTime)
import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
import Data.XML.Types (Content (..), Event (..))
import Oglyad.App (App, HttpError (..), httpGetE, note)
import Oglyad.Process (childEnv)
import Oglyad.Source.Cache (loadJson, saveJson)
import Oglyad.Types (RepoBuild (..))
import Oglyad.Version (EVR (..))
import System.Environment (getEnvironment)
import System.Exit (ExitCode (..))
import System.IO (Handle)
import System.Process.Typed (byteStringInput, createPipe, getStdout, nullStream, proc, setEnv, setStderr, setStdin, setStdout, waitExitCode, withProcessTerm)
import Text.XML (Name (..), def, parseLBS)
import Text.XML.Cursor (attribute, attributeIs, content, element, fromDocument, ($/), ($//), (&/))
import Text.XML.Stream.Parse (parseBytes)
-- | strict fields with copied text (see 'finish' in 'parsePrimary'), so an
-- entry never keeps the decompressed file alive
data RpmEntry = RpmEntry
{ rpmName :: !Text,
rpmArch :: !Text,
rpmEVR :: !EVR,
-- | absent on src rpms
rpmSourceRpm :: !(Maybe Text),
rpmBuildTime :: !UTCTime
}
deriving stock (Eq, Show)
-- | the primary.xml href in repomd.xml embeds a checksum, so an unchanged
-- href means the cached parse is still good
fetchRepo :: Text -> App (Either String (Map Text RepoBuild))
fetchRepo url = do
located <- fetchRepomd url
case located >>= \(base, repomd) -> (base,) <$> parseRepomd repomd of
Left err -> pure (Left err)
Right (base, href) -> do
let file = "repodata-" <> T.unpack (T.map (\c -> if isAlphaNum c then c else '_') base) <> ".json"
liftIO (loadJson file) >>= \case
Just (cachedHref, builds) | cachedHref == href -> pure (Right builds)
_ -> do
note ("downloading " <> T.unpack base <> "/" <> T.unpack href)
r <- fmap latestBySource <$> fetchPrimaryAt base href
case r of
Right builds -> liftIO (saveJson file (href, builds))
Left _ -> pure ()
pure r
-- | metalinks are read like dnf reads them: mirrors by preference, taking the
-- first repomd.xml whose sha256 the metalink lists, so a mirror that hasn't
-- synced yet is skipped
fetchRepomd :: Text -> App (Either String (Text, LBS.ByteString))
fetchRepomd url
| "metalink" `T.isInfixOf` url =
httpGetE url >>= \r -> case first httpErr r >>= parseMetalink of
Left err -> pure (Left ("can't read metalink " <> T.unpack url <> ": " <> err))
Right (_, []) -> pure (Left ("metalink " <> T.unpack url <> " lists no mirrors"))
Right (hashes, mirrors) -> tryMirrors hashes ("no mirror in " <> T.unpack url) mirrors
| otherwise = fmap (url,) . first httpErr <$> httpGetE (url <> "/repodata/repomd.xml")
where
tryMirrors _ lastErr [] = pure (Left lastErr)
tryMirrors hashes _ (m : rest) = case T.stripSuffix "/repodata/repomd.xml" m of
Nothing -> tryMirrors hashes ("metalink mirror " <> T.unpack m <> " isn't a repomd.xml") rest
Just base ->
httpGetE m >>= \case
Left e -> tryMirrors hashes (httpErr e) rest
Right body
| null hashes || sha256 body `elem` hashes -> pure (Right (base, body))
| otherwise -> tryMirrors hashes (T.unpack m <> " doesn't match the metalink's checksum, is the mirror behind?") rest
sha256 = TE.decodeUtf8 . B16.encode . SHA256.hashlazy
-- | the hashes include alternates, recent repomd.xml versions mirrors may
-- still serve
parseMetalink :: LBS.ByteString -> Either String ([Text], [Text])
parseMetalink bs = do
doc <- first show (parseLBS def bs)
let cur = fromDocument doc
hashes = cur $// (element (metalinkNs "hash") >=> attributeIs "type" "sha256") &/ content
urls =
[ (pref, T.strip u)
| el <- cur $// element (metalinkNs "url"),
let pref = case attribute "preference" el of
p : _ | Right (n, "") <- TR.decimal p -> n
_ -> 0 :: Int,
u <- [T.concat (el $/ content)]
]
Right (map T.strip hashes, map snd (sortOn (Down . fst) urls))
fetchPrimaryAt :: Text -> Text -> App (Either String [RpmEntry])
fetchPrimaryAt base href = do
download <- httpGetE (base <> "/" <> href)
case first httpErr download of
Left err -> pure (Left err)
Right compressed -> liftIO (decompressAndParse href compressed)
httpErr :: HttpError -> String
httpErr = \case
RateLimited why -> T.unpack why
HttpFailed why -> T.unpack why
-- | streams from the decompressor's pipe, so even Fedora's ~250MB file never
-- sits in memory. it shells out to zstd, gzip and xz because Fedora doesn't
-- package a Haskell zstd binding, and oglyad should build from Fedora's own
-- libraries
decompressAndParse :: Text -> LBS.ByteString -> IO (Either String [RpmEntry])
decompressAndParse href compressed
| ".zst" `T.isSuffixOf` href = viaCommand "zstd"
| ".gz" `T.isSuffixOf` href = viaCommand "gzip"
| ".xz" `T.isSuffixOf` href = viaCommand "xz"
| ".xml" `T.isSuffixOf` href = pure (parsePrimary compressed)
| otherwise = pure (Left ("don't know how to decompress " <> T.unpack href))
where
-- a corrupt archive shows up as a parse error or a non-zero exit, never a
-- crash
viaCommand cmd = do
env <- childEnv <$> getEnvironment
let decompressToStdout = ["--decompress", "--stdout", "--quiet"]
cfg = setEnv env (setStdin (byteStringInput compressed) (setStdout createPipe (setStderr nullStream (proc cmd decompressToStdout))))
r <- try $ withProcessTerm cfg $ \p -> do
parsed <- parsePrimaryHandle (getStdout p)
code <- waitExitCode p
pure $ case (code, parsed) of
(ExitFailure n, _) -> Left (cmd <> " exited with " <> show n <> " decompressing " <> T.unpack href)
(ExitSuccess, _) -> parsed
pure (either (\e -> Left (cmd <> ": " <> show (e :: SomeException))) id r)
repoNs, commonNs, rpmNs, metalinkNs :: Text -> Name
repoNs n = Name n (Just "http://linux.duke.edu/metadata/repo") Nothing
commonNs n = Name n (Just "http://linux.duke.edu/metadata/common") Nothing
rpmNs n = Name n (Just "http://linux.duke.edu/metadata/rpm") Nothing
metalinkNs n = Name n (Just "http://www.metalinker.org/") Nothing
-- | builds the whole document in memory, which is fine for a 2KB file
parseRepomd :: LBS.ByteString -> Either String Text
parseRepomd bs = do
doc <- first show (parseLBS def bs)
let hrefs =
fromDocument doc
$// (element (repoNs "data") >=> attributeIs "type" "primary")
&/ (element (repoNs "location") >=> attribute "href")
case hrefs of
(h : _) -> Right h
[] -> Left "no primary entry in repomd.xml"
data PkgAcc = PkgAcc
{ accName :: Text,
accArch :: Text,
accEVR :: Maybe EVR,
accTime :: Maybe UTCTime,
accSrpm :: Maybe Text
}
emptyAcc :: PkgAcc
emptyAcc = PkgAcc "" "" Nothing Nothing Nothing
data Capture = CapName | CapArch | CapSrpm | CapNone
deriving stock (Eq)
data St = St
{ stAcc :: !PkgAcc,
stCap :: !Capture,
stOut :: [RpmEntry],
stOpenElements :: !Int
}
-- | for small inputs and tests. real runs stream with 'parsePrimaryHandle'.
-- primary.xml is too big for a DOM (Fedora's is ~200MB), so XML events are
-- folded over instead, keeping memory proportional to the results
parsePrimary :: LBS.ByteString -> Either String [RpmEntry]
parsePrimary bs =
first (\e -> show (e :: SomeException)) (runConduit (sourceLazy bs .| parseBytes def .| foldlC stepPrimary st0))
>>= finishPrimary
-- | reads chunk by chunk because a lazy ByteString would let its thunk hold
-- on to the start of the stream, keeping the whole decompressed file alive
-- until parsing ends
parsePrimaryHandle :: Handle -> IO (Either String [RpmEntry])
parsePrimaryHandle h = do
r <- try (runConduit (sourceHandle h .| parseBytes def .| foldlC stepPrimary st0))
pure (either (\e -> Left (show (e :: SomeException))) finishPrimary r)
st0 :: St
st0 = St emptyAcc CapNone [] 0
finishPrimary :: St -> Either String [RpmEntry]
finishPrimary final
| stOpenElements final /= 0 = Left "primary.xml ends in the middle of an element, truncated download?"
| otherwise = Right (reverse (stOut final))
stepPrimary :: St -> Event -> St
stepPrimary cur ev = case ev of
EventBeginElement _ _ -> step cur {stOpenElements = stOpenElements cur + 1} ev
EventEndElement _ -> step cur {stOpenElements = stOpenElements cur - 1} ev
_ -> step cur ev
where
step st = \case
EventBeginElement name attrs
| name == commonNs "package" -> st {stAcc = emptyAcc, stCap = CapNone}
| name == commonNs "name" -> st {stCap = CapName}
| name == commonNs "arch" -> st {stCap = CapArch}
| name == rpmNs "sourcerpm" -> st {stCap = CapSrpm}
| name == commonNs "version" ->
let attr k = lookupAttr k attrs
evr = EVR <$> (attr "epoch" >>= readInt) <*> attr "ver" <*> attr "rel"
in st {stAcc = (stAcc st) {accEVR = evr}}
| name == commonNs "time" ->
let t = posixSecondsToUTCTime . fromIntegral <$> (lookupAttr "build" attrs >>= readInt)
in st {stAcc = (stAcc st) {accTime = t}}
| otherwise -> st
EventContent (ContentText t) -> case stCap st of
CapName -> st {stAcc = (stAcc st) {accName = accName (stAcc st) <> t}}
CapArch -> st {stAcc = (stAcc st) {accArch = accArch (stAcc st) <> t}}
CapSrpm -> st {stAcc = (stAcc st) {accSrpm = Just (maybe t (<> t) (accSrpm (stAcc st)))}}
CapNone -> st
EventEndElement name
| name == commonNs "package" -> case finish (stAcc st) of
Just e -> st {stOut = e : stOut st, stCap = CapNone}
Nothing -> st {stCap = CapNone}
| otherwise -> st {stCap = CapNone}
_ -> st
-- collected text is a slice of the parser's buffers and keeps the whole
-- buffer alive, so it's copied. UTCTime's fields are lazy, so the build
-- time is deep forced for the same reason
finish PkgAcc {..} = do
EVR e v r <- accEVR
t <- accTime
if T.null accName
then Nothing
else t `deepseq` (Just $! RpmEntry (T.copy accName) (T.copy accArch) (EVR e (T.copy v) (T.copy r)) (T.copy <$!> accSrpm) t)
lookupAttr :: Name -> [(Name, [Content])] -> Maybe Text
lookupAttr k attrs = do
cs <- lookup k attrs
pure (T.concat (mapMaybe contentText cs))
contentText = \case
ContentText t -> Just t
ContentEntity _ -> Nothing
readInt :: Text -> Maybe Int
readInt t = case TR.decimal t of
Right (n, "") -> Just n
_ -> Nothing
srpmName :: Text -> Maybe Text
srpmName file = do
base <- T.stripSuffix ".src.rpm" file
let parts = T.splitOn "-" base
if length parts < 3
then Nothing
else Just (T.intercalate "-" (take (length parts - 2) parts))
latestBySource :: [RpmEntry] -> Map Text RepoBuild
latestBySource = foldr step Map.empty
where
step e acc = case srcOf e of
Nothing -> acc
Just src -> Map.insertWith newer src (RepoBuild (srcEVR e) (rpmBuildTime e)) acc
srcOf e = maybe (Just (rpmName e)) srpmName (rpmSourceRpm e)
newer a b = if rbBuildTime a >= rbBuildTime b then a else b
-- binary subpackages can carry their own Version/Release, so prefer the
-- EVR spelled out in the srpm filename when there is one
srcEVR e = case rpmSourceRpm e >>= srpmEVR of
Just evr -> evr {evrEpoch = evrEpoch (rpmEVR e)}
Nothing -> rpmEVR e
srpmEVR file = do
base <- T.stripSuffix ".src.rpm" file
let parts = T.splitOn "-" base
case reverse parts of
rel : ver : _ : _ -> Just (EVR 0 ver rel)
_ -> Nothing