packages feed

oglyad-0.1.6.0: src/Oglyad/Collect.hs

module Oglyad.Collect
  ( buildSnapshot,
  )
where

import Control.Applicative ((<|>))
import Control.Concurrent.Async (concurrently)
import Control.Concurrent.MVar (withMVar)
import Control.Monad (forM, join)
import Control.Monad.Reader (ask, asks, liftIO, runReaderT)
import Data.Either (partitionEithers)
import Data.List (nub, partition)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe, isJust, isNothing, mapMaybe)
import Data.Text (Text)
import Data.Text qualified as T
import Data.Time (NominalDiffTime, UTCTime, diffUTCTime, getCurrentTime, nominalDay)
import Oglyad.App (App, Env (..), HttpError (..), forConcurrentlyN, lookupOnce, note, processSlot)
import Oglyad.Source.Anda (projectNames, reportSupported, updateProject, withScratchCopy)
import Oglyad.Source.Cache (AndaEntry (..), CacheEntry (..), UpstreamCache, loadJson, lookupFresh, saveJson)
import Oglyad.Source.Monorepo
import Oglyad.Source.Repodata (fetchRepo)
import Oglyad.Source.Rhai (parseUpdateRhaiWith, tokenize, tracksSubmodules)
import Oglyad.Source.RhaiEval (Simulated (..), simulateVersion)
import Oglyad.Source.Upstream (latestVersion)
import Oglyad.Types
import System.Directory (findExecutable, makeAbsolute)
import System.FilePath ((</>))
import System.Info qualified

-- | problems along the way become findings instead of crashes, so one broken
-- spec doesn't take down the whole run.
--
-- the monorepo work (rpmspec, git) is all local cpu and the repodata work is
-- all network, so the two halves run at the same time
buildSnapshot :: App (Snapshot, [Finding])
buildSnapshot = do
  cfg@Config {..} <- asks envConfig
  now <- liftIO getCurrentTime
  tree <- liftIO $ case cfgSpecRef of
    Nothing -> Right . (cfgMonorepo,"HEAD",) <$> headCommit cfgMonorepo
    Just ref -> fmap (\(dir, sha) -> (dir, sha, Just sha)) <$> checkoutRef cfgMonorepo ref
  let (root, logRef, hd, treeErrs) = case tree of
        Right (r, l, h) -> (Just r, l, h, [])
        Left err -> (Nothing, "HEAD", Nothing, [collectErr (fromMaybe "monorepo" cfgSpecRef) err])
  (dirErrs, dirs) <- partitionEithers <$> liftIO (maybe (pure []) (`findPackageDirs` cfgRoot) root)

  let subrepos = if cfgSubrepos then nub (map Just (mapMaybe pdSubrepo dirs)) else []
      terraRepos = [(s, repoUrl cfg s) | s <- Nothing : subrepos]
      publishedTo p = if cfgSubrepos then pkgSubrepo p else Nothing
  env <- ask
  ((specErrs, pkgs0), (terra, base)) <-
    liftIO $
      concurrently
        (runReaderT (readPackages (fromMaybe cfgMonorepo root) logRef hd dirs) env)
        (runReaderT (fetchRepos terraRepos cfgBaseRepos) env)

  let (repoErrs, repos) = partitionEithers terra
      fetched = Map.fromList repos
      pkgs1 =
        [ p
            { pkgRepoBuild = Map.lookup (publishedTo p) fetched >>= Map.lookup (pkgName p),
              pkgRepoChecked = Map.member (publishedTo p) fetched
            }
        | p <- pkgs0
        ]
  (upErrs, withUpstream) <- if cfgUpstream then upstreamVersions now root pkgs1 else pure ([], id)
  let pkgs = map withUpstream pkgs1

  let snap =
        Snapshot
          { snapPackages = Map.fromList [(pkgPath p, p) | p <- pkgs],
            snapBase = foldMap Map.keysSet base,
            snapNow = now,
            snapConfig = cfg,
            snapHead = hd,
            snapIgnoreLabels = Map.fromList [(pdPath d, pdIgnores d) | d <- dirs, not (null (pdIgnores d))]
          }
      unreadable =
        [ (collectErr (T.pack path) ("can't read package dir: " <> err)) {fPath = Just (T.pack path)}
        | (path, err) <- dirErrs
        ]
  pure (snap, treeErrs <> unreadable <> specErrs <> repoErrs <> upErrs)

upstreamVersions :: UTCTime -> Maybe FilePath -> [Package] -> App ([Finding], Package -> Package)
upstreamVersions now root pkgs = do
  env <- ask
  ((upErrs, fromApis), (andaErrs, fromAnda)) <-
    liftIO $
      concurrently
        (runReaderT (withUpstreams now pkgs) env)
        (runReaderT (maybe (pure ([], id)) (\r -> withAnda r now pkgs) root) env)
  pure (upErrs <> andaErrs, fromAnda . fromApis)

-- | Terra failures become findings, base distro ones just mean no overlap
-- data
fetchRepos ::
  [(Maybe Text, Text)] ->
  [Text] ->
  App ([Either Finding (Maybe Text, Map Text RepoBuild)], [Map Text RepoBuild])
fetchRepos terraRepos baseRepos = do
  let jobs = map Left terraRepos <> map Right baseRepos
  results <- forConcurrentlyN (length jobs) jobs $ \case
    Left (sub, url) -> Left . either (Left . collectErr url . T.pack) (\b -> Right (sub, b)) <$> fetchRepo url
    Right url -> Right . either (const Map.empty) id <$> fetchRepo url
  pure (partitionEithers results)

readPackages :: FilePath -> Text -> Maybe Text -> [PkgDir] -> App ([Finding], [Package])
readPackages root0 logRef headSha dirs = do
  Config {..} <- asks envConfig
  root <- liftIO (makeAbsolute root0)
  -- one cache file per release, so cron jobs checking different releases
  -- don't keep evicting each other's entries
  let cacheFile = "specs" <> T.unpack cfgCacheTag <> ".json"
  (commits, (macros, cache)) <-
    liftIO $
      concurrently
        (commitTimes cfgMonorepo cfgRoot logRef headSha cfgCacheTag (map pdPath dirs))
        ((,) <$> macroFingerprint <*> (fromMaybe Map.empty <$> loadJson cacheFile))
  keyed <- forConcurrentlyN cfgConcurrency dirs $ \d -> liftIO $ do
    key <- specFingerprint macros cfgMacros root (pdPath d)
    pure (key, d)
  let misses = length [() | (k, _) <- keyed, isNothing (Map.lookup k (cache :: Map Text (Either Text SpecInfo)))]
  note ("evaluating " <> show (length dirs) <> " specs, " <> show (length dirs - misses) <> " unchanged since last run")
  -- without rpmspec, cached evaluations still work. everything else is
  -- skipped and reported once, rather than as one error per package
  hasRpmspec <- liftIO (isJust <$> findExecutable "rpmspec")
  inSlot <- processSlot
  results <- forConcurrentlyN cfgConcurrency keyed $ \(key, d) -> liftIO $ do
    case (Map.lookup key cache, pdSpec d) of
      (Just cached, _) -> pure (Evaluated key d cached)
      (Nothing, Just spec)
        | hasRpmspec -> Evaluated key d <$> inSlot (querySpec cfgMacros (root </> spec))
        | otherwise -> pure Skipped
      (Nothing, Nothing) -> pure (NoSpec d)
  -- keep only entries for packages that still exist, so the file can't grow forever
  liftIO (saveJson cacheFile (Map.fromList [(k, i) | Evaluated k _ i <- results]))
  let skipped = length [() | Skipped <- results]
      missing =
        [ collectErr "rpmspec" ("rpmspec not found, install rpm-build. skipped " <> T.pack (show skipped) <> " specs with no cached result")
        | skipped > 0
        ]
      (errs, pkgs) =
        partitionEithers $
          [toPackage commits k d (Just i) | Evaluated k d i <- results, not (failedForThisHost d i)]
            <> [toPackage commits "" d Nothing | NoSpec d <- results]
  pure (missing <> errs, pkgs)

data SpecResult
  = Evaluated Text PkgDir (Either Text SpecInfo)
  | NoSpec PkgDir
  | Skipped

failedForThisHost :: PkgDir -> Either Text SpecInfo -> Bool
failedForThisHost PkgDir {pdArches} = \case
  Left err -> "No compatible architectures found for build" `T.isInfixOf` err && not buildsHere
  Right _ -> False
  where
    buildsHere = null pdArches || T.pack System.Info.arch `elem` pdArches

-- | uncommitted changes can't change any "last commit", so the commit is the
-- whole cache key. one cache file per release, since each reads its own branch
commitTimes :: FilePath -> FilePath -> Text -> Maybe Text -> Text -> [FilePath] -> IO (Map FilePath UTCTime)
commitTimes mono pkgRoot ref hd tag dirs = do
  let file = "commits" <> T.unpack tag <> ".json"
  cached <- loadJson file
  case (hd, cached) of
    (Just h, Just (h', m)) | h == h', all (`Map.member` m) dirs -> pure m
    _ -> do
      m <- lastCommits mono pkgRoot ref dirs
      mapM_ (\h -> saveJson file (h, m)) hd
      pure m

toPackage :: Map FilePath UTCTime -> Text -> PkgDir -> Maybe (Either Text SpecInfo) -> Either Finding Package
toPackage commits key PkgDir {..} = \case
  Nothing -> Left (pathErr "no spec file found")
  Just (Left err) -> Left (pathErr ("rpmspec failed: " <> firstLine err))
  Just (Right SpecInfo {..}) ->
    Right
      Package
        { pkgName = siName,
          pkgPath = pdPath,
          pkgSubrepo = pdSubrepo,
          pkgNightly = pdNightly,
          pkgUpdBranch = pdUpdBranch,
          pkgSpecPath = pdSpec,
          pkgFingerprint = key,
          pkgSpecEVR = Just siEVR,
          pkgRepoBuild = Nothing,
          pkgLastCommit = Map.lookup pdPath commits,
          pkgUpstream = parseUpdateRhaiWith pdVersionMacros <$> (pdUpdateRhai >>= unlessOnlyComments),
          pkgRhai = pdUpdateRhai,
          pkgUpstreamVer = Nothing,
          pkgUpstreamErr = Nothing,
          pkgSubmoduleEvents = [],
          pkgRepoChecked = False
        }
  where
    firstLine t = let l = T.takeWhile (/= '\n') t in if T.null l then "(no output)" else l
    unlessOnlyComments script = case tokenize script of
      Just [] -> Nothing
      _ -> Just script
    pathErr msg = (collectErr (T.pack pdPath) msg) {fPath = Just (T.pack pdPath)}

-- | crates.io allows one request a second, so its lookups run in their own
-- serial queue next to everything else. inside the shared job pool, a cluster
-- of crates packages would hold every slot while waiting their turn
withUpstreams :: UTCTime -> [Package] -> App ([Finding], Package -> Package)
withUpstreams now pkgs = do
  cfg@Config {..} <- asks envConfig
  cache <- liftIO (fromMaybe Map.empty <$> loadJson "upstream.json") :: App UpstreamCache
  let wanted = [p | p <- pkgs, Just src <- [pkgUpstream p], lookable src]
      fresh p = pkgUpstream p >>= \src -> lookupFresh (cacheMaxAge cfg now) now src cache
      (crates, others) = partition (isCrates . pkgUpstream) wanted
      lookup1 p = case (fresh p, pkgUpstream p) of
        (Just e, _) -> pure (pkgPath p, either (Left . HttpFailed) Right (ceResult e))
        (Nothing, Just src) -> (pkgPath p,) <$> lookupOnce (cacheKey src) (latestVersion src)
        (Nothing, Nothing) -> pure (pkgPath p, Left (HttpFailed "no source"))
  note ("querying upstreams for " <> show (length wanted) <> " packages, " <> show (length (filter (isJust . fresh) wanted)) <> " cached")
  env <- ask
  (r1, r2) <-
    liftIO $
      concurrently
        (runReaderT (forConcurrentlyN cfgConcurrency others lookup1) env)
        (runReaderT (forM crates lookup1) env)
  -- keyed by path, not name: two dirs can share a package name
  let byPath = Map.fromList (r1 <> r2)
      -- only cache successes. errors are often transient (rate limits, timeouts)
      new =
        Map.fromList
          [ (cacheKey src, CacheEntry now (Right v))
          | p <- wanted,
            isNothing (fresh p),
            Just src <- [pkgUpstream p],
            Just (Right v) <- [Map.lookup (pkgPath p) byPath]
          ]
      limited = [why | Left (RateLimited why) <- Map.elems byPath]
      rateErr = case limited of
        [] -> []
        why : _ ->
          [ (collectErr "upstream" (why <> ", skipped " <> T.pack (show (length limited)) <> " lookups. set GITHUB_TOKEN, or try again later"))
              { fSeverity = Warn
              }
          ]
  saveSharedCache "upstream.json" now new
  pure
    ( rateErr,
      \p -> case Map.lookup (pkgPath p) byPath of
        Just (Right v) -> p {pkgUpstreamVer = replay p v}
        Just (Left (HttpFailed e)) -> p {pkgUpstreamErr = Just e}
        -- rate limits are reported once above, not as a broken upstream
        Just (Left (RateLimited _)) -> p
        Nothing -> p
    )
  where
    cacheKey src = T.pack (show src)
    -- run the script's guards and edits over the raw value, so a tag the
    -- script would skip (or crop) doesn't show up as drift
    replay p v = case (pkgUpstream p, pkgRhai p) of
      (Just src, Just script) -> case simulateVersion src v script of
        Sets v' -> Just v'
        NoUpdate -> Nothing
        CantTell -> Just v
      _ -> Just v
    lookable = \case
      CommitTracked -> False
      NoVersion -> False
      Derived _ -> False
      Unknown _ -> False
      _ -> True
    isCrates = \case
      Just (Crates _ _) -> True
      _ -> False

withAnda :: FilePath -> UTCTime -> [Package] -> App ([Finding], Package -> Package)
withAnda root now pkgs = do
  cfg@Config {..} <- asks envConfig
  let wanted =
        [ p
        | p@Package {pkgSpecPath = Just _} <- pkgs,
          unfollowable p || maybe False tracksSubmodules (pkgRhai p),
          cfgRolling || pkgUpdBranch p
        ]
      cacheFile = "anda" <> T.unpack cfgCacheTag <> ".json"
  cache <- liftIO (fromMaybe Map.empty <$> loadJson cacheFile) :: App (Map Text AndaEntry)
  let fresh p = case Map.lookup (pkgFingerprint p) cache of
        Just e | diffUTCTime now (aeFetched e) <= cacheMaxAge cfg now -> Just e
        _ -> Nothing
      misses = [p | p <- wanted, isNothing (fresh p)]
  hasAnda <- liftIO (isJust <$> findExecutable "anda")
  andaReports <- if hasAnda && not (null misses) then liftIO reportSupported else pure False
  (problems, ran) <-
    if
      | null misses -> pure ([], Map.empty)
      | not hasAnda -> do
          note ("anda not found, so " <> show (length misses) <> " update.rhai scripts that need anda are skipped")
          pure ([], Map.empty)
      | not andaReports ->
          pure ([(collectErr "anda" ("anda is too old for `anda update --report`, so " <> T.pack (show (length misses)) <> " update.rhai scripts that need anda are skipped. update anda")) {fSeverity = Warn}], Map.empty)
      | otherwise -> do
          note ("running anda update for " <> show (length wanted) <> " scripts that need anda, " <> show (length wanted - length misses) <> " cached")
          env <- ask
          inSlot <- processSlot
          r <- liftIO $ withScratchCopy root $ \copy ->
            projectNames copy >>= \case
              Left e -> pure (Left e)
              Right names -> fmap Right . flip runReaderT env . forConcurrentlyN cfgConcurrency misses $ \p ->
                liftIO . inSlot $
                  (pkgPath p,) <$> case (Map.lookup (pkgPath p) names, pkgSpecPath p) of
                    (Just name, Just spec) ->
                      updateProject copy cfgGitHubToken cfgBranch name >>= \case
                        Left e -> pure (Left e)
                        Right events ->
                          either (Left . ("rpmspec failed on the updated spec: " <>)) (\i -> Right (evrVersion (siEVR i), events))
                            <$> querySpec cfgMacros (copy </> spec)
                    _ -> pure (Left "anda doesn't list this project")
          pure $ case join r of
            Left e -> ([(collectErr "anda" e) {fSeverity = Warn}], Map.empty)
            Right xs -> ([], Map.fromList xs)
  let answer p = maybe (Map.lookup (pkgPath p) ran) (\e -> Just ((,aeEvents e) <$> aeResult e)) (fresh p)
      new =
        Map.fromList
          [ (pkgFingerprint p, AndaEntry now (Right v) events)
          | p <- misses,
            Just (Right (v, events)) <- [Map.lookup (pkgPath p) ran]
          ]
  liftIO (saveJson cacheFile (new <> Map.filter ((< 30 * nominalDay) . diffUTCTime now . aeFetched) cache))
  pure
    ( problems,
      \p -> case answer p of
        Just (Right (v, events))
          | unfollowable p -> p {pkgUpstreamVer = Just v, pkgSubmoduleEvents = events}
          | otherwise -> p {pkgSubmoduleEvents = events}
        Just (Left e) -> p {pkgUpstreamErr = pkgUpstreamErr p <|> Just ("update.rhai failed under anda: " <> e)}
        Nothing -> p
    )
  where
    unfollowable p = case pkgUpstream p of
      Just (Unknown _) -> True
      _ -> False

-- | anything fetched since this run started is fresh, so checking several
-- releases asks each upstream once even with --cache-hours 0
cacheMaxAge :: Config -> UTCTime -> NominalDiffTime
cacheMaxAge Config {..} now = max cfgCacheFor (diffUTCTime now cfgRunStart)

fetchedThisMonth :: UTCTime -> CacheEntry -> Bool
fetchedThisMonth now e = diffUTCTime now (ceFetched e) < 30 * nominalDay

saveSharedCache :: FilePath -> UTCTime -> Map Text CacheEntry -> App ()
saveSharedCache file now new = do
  lock <- asks envCacheLock
  liftIO . withMVar lock $ \_ -> do
    onDisk <- fromMaybe Map.empty <$> loadJson file
    saveJson file (new <> Map.filter (fetchedThisMonth now) onDisk)

collectErr :: Text -> Text -> Finding
collectErr what msg = Finding {fCheck = "collect", fPackage = what, fPath = Nothing, fSeverity = Error, fMessage = msg, fReleases = []}