diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 0.3.1 - 2022-05-17 + +### Added + + - `HERCULES_CI_SECRETS_JSON` can now be used to find the secrets + in an alternate location. + + - Other improvements for running in sandboxed environments, chiefly + [`hercules-ci-effects effectVMTest`](https://docs.hercules-ci.com/hercules-ci-effects/reference/nix-functions/effectvmtest/) + ## 0.3.0 - 2022-03-15 ### Added diff --git a/hercules-ci-cli.cabal b/hercules-ci-cli.cabal --- a/hercules-ci-cli.cabal +++ b/hercules-ci-cli.cabal @@ -1,7 +1,7 @@ cabal-version: 1.12 name: hercules-ci-cli -version: 0.3.0 +version: 0.3.1 synopsis: The hci command for working with Hercules CI homepage: https://docs.hercules-ci.com bug-reports: https://github.com/hercules-ci/hercules-ci-agent/issues diff --git a/src/Hercules/CLI/Common.hs b/src/Hercules/CLI/Common.hs --- a/src/Hercules/CLI/Common.hs +++ b/src/Hercules/CLI/Common.hs @@ -1,11 +1,12 @@ module Hercules.CLI.Common ( runAuthenticated, + runAuthenticatedOrDummy, exitMsg, ) where import Hercules.CLI.Client -import Hercules.CLI.Credentials (determineDomain, readToken) +import Hercules.CLI.Credentials (determineDomain, readToken, tryReadEffectToken) import Hercules.CLI.Exception (exitMsg) import Protolude import RIO (RIO, runRIO) @@ -14,6 +15,16 @@ runAuthenticated :: RIO (HerculesClientToken, HerculesClientEnv) b -> IO b runAuthenticated m = do clientEnv <- Hercules.CLI.Client.init - domain <- determineDomain - token <- readToken domain + token <- readToken determineDomain + runRIO (HerculesClientToken $ Token $ encodeUtf8 token, clientEnv) m + +runAuthenticatedOrDummy :: + -- | Use fake credential if 'False'. + Bool -> + RIO (HerculesClientToken, HerculesClientEnv) b -> + IO b +runAuthenticatedOrDummy True = runAuthenticated +runAuthenticatedOrDummy False = \m -> do + clientEnv <- Hercules.CLI.Client.init + token <- fromMaybe "dummy-token" <$> tryReadEffectToken runRIO (HerculesClientToken $ Token $ encodeUtf8 token, clientEnv) m diff --git a/src/Hercules/CLI/Credentials.hs b/src/Hercules/CLI/Credentials.hs --- a/src/Hercules/CLI/Credentials.hs +++ b/src/Hercules/CLI/Credentials.hs @@ -109,8 +109,18 @@ Nothing -> throwIO NoCredentialException {noCredentialDomain = domain} Just cred -> pure (personalToken cred) +-- | Try to get a token from the local environment. +-- +-- 1. HERCULES_CI_API_TOKEN +-- 2. HERCULES_CI_SECRETS_JSON tryReadEffectToken :: IO (Maybe Text) -tryReadEffectToken = runMaybeT do +tryReadEffectToken = runMaybeT $ tryReadEffectTokenFromEnv <|> tryReadEffectTokenFromFile + +tryReadEffectTokenFromEnv :: MaybeT IO Text +tryReadEffectTokenFromEnv = T.pack <$> MaybeT (System.Environment.lookupEnv "HERCULES_CI_API_TOKEN") + +tryReadEffectTokenFromFile :: MaybeT IO Text +tryReadEffectTokenFromFile = do inEffect <- MaybeT $ System.Environment.lookupEnv "IN_HERCULES_CI_EFFECT" guard $ inEffect == "true" secretsJsonPath <- MaybeT $ System.Environment.lookupEnv "HERCULES_CI_SECRETS_JSON" @@ -123,8 +133,8 @@ Just x -> pure x Nothing -> throwIO $ FatalError $ "HERCULES_CI_SECRETS_JSON, " <> T.pack secretsJsonPath <> " doesn't have key hercules-ci.data.token" -readToken :: Text -> IO Text -readToken domain = do +readToken :: IO Text -> IO Text +readToken getDomain = do tryReadEffectToken >>= \case Just x -> pure x - Nothing -> readPersonalToken domain + Nothing -> readPersonalToken =<< getDomain diff --git a/src/Hercules/CLI/Effect.hs b/src/Hercules/CLI/Effect.hs --- a/src/Hercules/CLI/Effect.hs +++ b/src/Hercules/CLI/Effect.hs @@ -1,9 +1,11 @@ {-# LANGUAGE ApplicativeDo #-} {-# LANGUAGE BlockArguments #-} {-# LANGUAGE DataKinds #-} +{-# LANGUAGE ScopedTypeVariables #-} module Hercules.CLI.Effect where +import qualified Data.ByteString as BS import Data.Has (Has) import qualified Data.Text as T import Hercules.API.Id (Id (Id, idUUID)) @@ -14,7 +16,7 @@ import qualified Hercules.Agent.NixFile.HerculesCIArgs as HerculesCIArgs import Hercules.Agent.Sensitive (Sensitive (Sensitive)) import Hercules.CLI.Client (HerculesClientEnv, HerculesClientToken, determineDefaultApiBaseUrl, projectsClient, runHerculesClient) -import Hercules.CLI.Common (runAuthenticated) +import Hercules.CLI.Common (runAuthenticatedOrDummy) import Hercules.CLI.Exception (exitMsg) import Hercules.CLI.Git (getAllBranches, getHypotheticalRefs) import qualified Hercules.CLI.Git as Git @@ -23,9 +25,9 @@ import Hercules.CLI.Project (ProjectPath, getProjectIdAndPath, projectOption, projectPathOwner, projectPathProject, projectPathText) import Hercules.CLI.Secret (getSecretsFilePath) import Hercules.CNix (Store) -import Hercules.CNix.Expr (Match (IsAttrs), Value (rtValue), getAttrBool, getDrvFile, match) +import Hercules.CNix.Expr (EvalState, Match (IsAttrs), Value (rtValue), getAttrBool, getDrvFile, match) import qualified Hercules.CNix.Std.Vector as Std.Vector -import Hercules.CNix.Store (Derivation, StorePath, buildPaths, getDerivationInputs, newStorePathWithOutputs) +import Hercules.CNix.Store (Derivation, buildPaths, getDerivationInputs, newStorePathWithOutputs) import qualified Hercules.CNix.Store as CNix import Hercules.Effect (RunEffectParams (..), runEffect) import Hercules.Error (escalate) @@ -52,34 +54,31 @@ projectOptionMaybe <- optional projectOption refMaybe <- asRefOptions requireToken <- Optparse.flag True False (long "no-token" <> help "Don't get an API token. Disallows access to state files, but can run in untrusted environment or unconfigured repo.") - pure $ runAuthenticated do - withAsync (getProjectEffectData projectOptionMaybe requireToken) \projectPathAsync -> do + pure $ runAuthenticatedOrDummy requireToken do + let getProjectInfo = + case projectOptionMaybe of + Just x + | not requireToken -> + pure + ProjectData + { pdProjectPath = Just x, + pdProjectId = Nothing, + pdToken = Nothing + } + _ -> getProjectEffectData projectOptionMaybe requireToken + withAsync getProjectInfo \projectPathAsync -> do withNix \store evalState -> do ref <- liftIO $ computeRef refMaybe - isDefaultBranch <- liftIO Git.getIsDefault - args <- liftIO $ createHerculesCIArgs (Just ref) - let attrPath = T.split (== '.') attr - nixFile = GitSource.outPath $ HerculesCIArgs.primaryRepo args - uio <- askUnliftIO - valMaybe <- liftIO $ getOnPushOutputValueByPath evalState (toS nixFile) args (resolveInputs uio evalState projectOptionMaybe) (map encodeUtf8 attrPath) - -- valMaybe <- liftIO $ attrByPath evalState rootValue - attrValue <- case valMaybe of - Nothing -> do - exitMsg $ "Could not find an attribute at path " <> show attrPath <> " in " <> nixFile - Just v -> liftIO (match evalState v) >>= escalate - effectAttrs <- case attrValue of - IsAttrs attrs -> pure attrs - _ -> do - exitMsg $ "Attribute is not an Effect at path " <> show attrPath <> " in " <> nixFile + derivation <- getEffectDrv store evalState projectOptionMaybe ref attr + isDefaultBranch <- + if requireToken + then liftIO Git.getIsDefault + else pure True - isEffect <- liftIO $ getAttrBool evalState effectAttrs "isEffect" >>= escalate - when (isEffect /= Just True) do - exitMsg $ "Attribute is not an Effect at path " <> show attrPath <> " in " <> nixFile - drvPath <- getDrvFile evalState (rtValue effectAttrs) - derivation <- prepareDerivation store drvPath apiBaseURL <- liftIO determineDefaultApiBaseUrl ProjectData {pdProjectPath = projectPath, pdProjectId = projectId, pdToken = token} <- wait projectPathAsync secretsJson <- liftIO $ traverse getSecretsFilePath projectPath + logEnv <- liftIO $ initLogEnv mempty "hci" -- withSystemTempDirectory "hci": -- ERRO[0000] container_linux.go:370: starting container process caused: process_linux.go:459: container init caused: rootfs_linux.go:59: mounting "/run/user/1000/hci6017/secrets" to rootfs at "/run/user/1000/hci6017/runc-state/rootfs/secrets" caused: operation not permitted @@ -111,16 +110,53 @@ } throwIO exitCode -prepareDerivation :: MonadIO m => Store -> StorePath -> m Derivation -prepareDerivation store drvPath = do - derivation <- liftIO $ CNix.getDerivation store drvPath +getEffectDrv :: Store -> Ptr EvalState -> Maybe ProjectPath -> Text -> Text -> RIO (HerculesClientToken, HerculesClientEnv) Derivation +getEffectDrv store evalState projectOptionMaybe ref attr = do + storeDir <- liftIO $ CNix.storeDir store + derivation <- + if decodeUtf8With lenientDecode storeDir `T.isPrefixOf` attr + then liftIO $ do + -- Support derivation in arbitrary location + -- Used in hercules-ci-effects test runner + let path = attr + contents <- BS.readFile $ toS path + let stripDrv s = fromMaybe s (T.stripSuffix ".drv" s) + CNix.getDerivationFromString store (path & T.takeWhileEnd (/= '/') & stripDrv & encodeUtf8) contents + else evaluateEffectDerivation evalState store projectOptionMaybe ref attr + prepareDerivation store derivation + pure derivation + +evaluateEffectDerivation :: (Has HerculesClientToken r, Has HerculesClientEnv r) => Ptr EvalState -> Store -> Maybe ProjectPath -> Text -> Text -> RIO r Derivation +evaluateEffectDerivation evalState store projectOptionMaybe ref attr = do + args <- liftIO $ createHerculesCIArgs (Just ref) + let attrPath = T.split (== '.') attr + nixFile = GitSource.outPath $ HerculesCIArgs.primaryRepo args + uio <- askUnliftIO + valMaybe <- liftIO $ getOnPushOutputValueByPath evalState (toS nixFile) args (resolveInputs uio evalState projectOptionMaybe) (map encodeUtf8 attrPath) + -- valMaybe <- liftIO $ attrByPath evalState rootValue + attrValue <- case valMaybe of + Nothing -> do + exitMsg $ "Could not find an attribute at path " <> show attrPath <> " in " <> nixFile + Just v -> liftIO (match evalState v) >>= escalate + effectAttrs <- case attrValue of + IsAttrs attrs -> pure attrs + _ -> do + exitMsg $ "Attribute is not an Effect at path " <> show attrPath <> " in " <> nixFile + + isEffect <- liftIO $ getAttrBool evalState effectAttrs "isEffect" >>= escalate + when (isEffect /= Just True) do + exitMsg $ "Attribute is not an Effect at path " <> show attrPath <> " in " <> nixFile + drvPath <- getDrvFile evalState (rtValue effectAttrs) + liftIO $ CNix.getDerivation store drvPath + +prepareDerivation :: MonadIO m => Store -> Derivation -> m () +prepareDerivation store derivation = do inputs <- liftIO $ getDerivationInputs store derivation storePathsWithOutputs <- liftIO Std.Vector.new liftIO $ for_ inputs \(input, outputs) -> do swo <- newStorePathWithOutputs input outputs Std.Vector.pushBackFP storePathsWithOutputs swo liftIO $ buildPaths store storePathsWithOutputs - pure derivation ciAttributeArgument :: Optparse.Parser Text ciAttributeArgument = diff --git a/src/Hercules/CLI/Secret.hs b/src/Hercules/CLI/Secret.hs --- a/src/Hercules/CLI/Secret.hs +++ b/src/Hercules/CLI/Secret.hs @@ -16,6 +16,7 @@ import Hercules.UserException (UserException (UserException)) import qualified Options.Applicative as Optparse import Protolude +import System.Environment (lookupEnv) import System.FilePath (takeDirectory, ()) import UnliftIO.Directory (XdgDirectory (XdgConfig), createDirectoryIfMissing, doesFileExist, getXdgDirectory) @@ -108,6 +109,12 @@ getSecretsFilePath :: ProjectPath -> IO FilePath getSecretsFilePath projectPath = do + lookupEnv "HERCULES_CI_SECRETS_JSON" >>= \case + Nothing -> getSecretsFilePathXdg projectPath + Just x -> pure x + +getSecretsFilePathXdg :: ProjectPath -> IO FilePath +getSecretsFilePathXdg projectPath = do dir <- getXdgDirectory XdgConfig "hercules-ci" let toPathElement = toS . T.map (\case '/' -> '_'; x -> x) pure $ dir "secrets" toPathElement (projectPathSite projectPath) toPathElement (projectPathOwner projectPath) "secrets.json"