diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.7.5] - 2020-11-27 + +### Fixed + + - GHC 8.10.2 compatibility for NixOS unstable / NixOS 21.03 + + - Build with cachix 0.5.1. Write token support is not included due to a break configuration semantics. It will be available in >= 0.8. + +### Changed + + - The in-repo expressions have upgraded their dependency to NixOS 20.09 + + - Agent will now use `HERCULES_CI_API_BASE_URL` over `HERCULES_API_BASE_URL` if set. + + - Temporarily switch to cabal tooling due to breakage. `master` continues to be stack-based. + ## [0.7.4] - 2020-08-25 ### Fixed diff --git a/cbits/aliases.h b/cbits/aliases.h deleted file mode 100644 --- a/cbits/aliases.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include "hercules-store.hh" -#include "hercules-logger.hh" -#include "derivations.hh" - -// inline-c-cpp doesn't seem to handle namespace operator or template -// syntax so we help it a bit for now. This definition can be inlined -// when it is supported by inline-c-cpp. -typedef nix::ref refStore; - -typedef nix::ref refHerculesStore; - -typedef nix::Logger::Fields LoggerFields; - -typedef HerculesLogger::LogEntry HerculesLoggerEntry; -typedef std::queue> LogEntryQueue; - -typedef nix::Strings::iterator StringsIterator; -typedef nix::DerivationOutputs::iterator DerivationOutputsIterator; -typedef nix::DerivationInputs::iterator DerivationInputsIterator; -typedef nix::StringPairs::iterator StringPairsIterator; - -using namespace std; diff --git a/cbits/hercules-aliases.h b/cbits/hercules-aliases.h new file mode 100644 --- /dev/null +++ b/cbits/hercules-aliases.h @@ -0,0 +1,24 @@ +#pragma once + +#include "hercules-store.hh" +#include "hercules-logger.hh" +#include "derivations.hh" + +// inline-c-cpp doesn't seem to handle namespace operator or template +// syntax so we help it a bit for now. This definition can be inlined +// when it is supported by inline-c-cpp. +typedef nix::ref refStore; + +typedef nix::ref refHerculesStore; + +typedef nix::Logger::Fields LoggerFields; + +typedef HerculesLogger::LogEntry HerculesLoggerEntry; +typedef std::queue> LogEntryQueue; + +typedef nix::Strings::iterator StringsIterator; +typedef nix::DerivationOutputs::iterator DerivationOutputsIterator; +typedef nix::DerivationInputs::iterator DerivationInputsIterator; +typedef nix::StringPairs::iterator StringPairsIterator; + +using namespace std; diff --git a/cbits/hercules-logger.cc b/cbits/hercules-logger.cc deleted file mode 100644 --- a/cbits/hercules-logger.cc +++ /dev/null @@ -1,90 +0,0 @@ -#include "hercules-logger.hh" - -HerculesLogger::HerculesLogger() -{ - -} - -void HerculesLogger::push(std::unique_ptr entry) { - auto state(state_.lock()); - state->queue.push(std::move(entry)); - wakeup.notify_one(); -} - -uint64_t HerculesLogger::getMs() { - auto t = std::chrono::steady_clock::now(); - auto millis = std::chrono::duration_cast(t - t_zero).count(); - return millis; -} - -void HerculesLogger::log(nix::Verbosity lvl, const nix::FormatOrString & fs) { - push(std::make_unique(LogEntry { - .entryType = 1, - .level = lvl, - .ms = getMs(), - .text = fs.s - })); -} - -void HerculesLogger::startActivity(nix::ActivityId act, nix::Verbosity lvl, nix::ActivityType type, - const std::string & s, const Fields & fields, nix::ActivityId parent) { - push(std::make_unique(LogEntry { - .entryType = 2, - .level = lvl, - .ms = getMs(), - .text = s, - .activityId = act, - .type = type, - .parent = parent, - .fields = fields - })); -} - -void HerculesLogger::stopActivity(nix::ActivityId act) { - push(std::make_unique(LogEntry { - .entryType = 3, - .ms = getMs(), - .activityId = act - })); -} - -void HerculesLogger::result(nix::ActivityId act, nix::ResultType type, const Fields & fields) { - push(std::make_unique(LogEntry { - .entryType = 4, - .ms = getMs(), - .activityId = act, - .type = type, - .fields = fields - })); -} - -void HerculesLogger::close() { - auto state(state_.lock()); - state->queue.push(nullptr); - wakeup.notify_one(); -} - -std::unique_ptr HerculesLogger::pop () { - auto state(state_.lock()); - - while (state->queue.empty()) - state.wait(wakeup); - - auto r = std::move(state->queue.front()); - state->queue.pop(); - return r; -} - -void HerculesLogger::popMany (int max, std::queue> &out) { - auto state(state_.lock()); - - while (state->queue.empty()) - state.wait(wakeup); - - for (int i = 0; i < max && !state->queue.empty(); i++) { - out.push(std::move(state->queue.front())); - state->queue.pop(); - } -} - -HerculesLogger *herculesLogger = nullptr; diff --git a/cbits/hercules-logger.cxx b/cbits/hercules-logger.cxx new file mode 100644 --- /dev/null +++ b/cbits/hercules-logger.cxx @@ -0,0 +1,90 @@ +#include "hercules-logger.hh" + +HerculesLogger::HerculesLogger() +{ + +} + +void HerculesLogger::push(std::unique_ptr entry) { + auto state(state_.lock()); + state->queue.push(std::move(entry)); + wakeup.notify_one(); +} + +uint64_t HerculesLogger::getMs() { + auto t = std::chrono::steady_clock::now(); + auto millis = std::chrono::duration_cast(t - t_zero).count(); + return millis; +} + +void HerculesLogger::log(nix::Verbosity lvl, const nix::FormatOrString & fs) { + push(std::make_unique(LogEntry { + .entryType = 1, + .level = lvl, + .ms = getMs(), + .text = fs.s + })); +} + +void HerculesLogger::startActivity(nix::ActivityId act, nix::Verbosity lvl, nix::ActivityType type, + const std::string & s, const Fields & fields, nix::ActivityId parent) { + push(std::make_unique(LogEntry { + .entryType = 2, + .level = lvl, + .ms = getMs(), + .text = s, + .activityId = act, + .type = type, + .parent = parent, + .fields = fields + })); +} + +void HerculesLogger::stopActivity(nix::ActivityId act) { + push(std::make_unique(LogEntry { + .entryType = 3, + .ms = getMs(), + .activityId = act + })); +} + +void HerculesLogger::result(nix::ActivityId act, nix::ResultType type, const Fields & fields) { + push(std::make_unique(LogEntry { + .entryType = 4, + .ms = getMs(), + .activityId = act, + .type = type, + .fields = fields + })); +} + +void HerculesLogger::close() { + auto state(state_.lock()); + state->queue.push(nullptr); + wakeup.notify_one(); +} + +std::unique_ptr HerculesLogger::pop () { + auto state(state_.lock()); + + while (state->queue.empty()) + state.wait(wakeup); + + auto r = std::move(state->queue.front()); + state->queue.pop(); + return r; +} + +void HerculesLogger::popMany (int max, std::queue> &out) { + auto state(state_.lock()); + + while (state->queue.empty()) + state.wait(wakeup); + + for (int i = 0; i < max && !state->queue.empty(); i++) { + out.push(std::move(state->queue.front())); + state->queue.pop(); + } +} + +HerculesLogger *herculesLogger = nullptr; diff --git a/cbits/hercules-store.cc b/cbits/hercules-store.cc deleted file mode 100644 --- a/cbits/hercules-store.cc +++ /dev/null @@ -1,271 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hercules-store.hh" - -using namespace nix; - -WrappingStore::WrappingStore(const Params& params, ref storeToWrap) - : Store(params), wrappedStore(storeToWrap) {} - -WrappingStore::~WrappingStore() {} - -std::string WrappingStore::getUri() { - return "wrapped:" + wrappedStore->getUri(); -}; - -bool WrappingStore::isValidPathUncached(const Path& path) { - return wrappedStore->isValidPath(path); // not ideal -} - -PathSet WrappingStore::queryValidPaths(const PathSet& paths, - SubstituteFlag maybeSubstitute) { - return wrappedStore->queryValidPaths(paths, maybeSubstitute); -} -PathSet WrappingStore::queryAllValidPaths() { - return wrappedStore->queryAllValidPaths(); -} - -// protected: -void WrappingStore::queryPathInfoUncached( - const Path& path, - Callback> callback) noexcept { - unsupported("queryPathInfoUncached"); - /* - Callback> - callback2([&callback](std::future> vpi){ - std::shared_ptr !@#$%^&* - callback(vpi); - }); - return wrappedStore->queryPathInfo(path, callback2); // not ideal - */ -} - -// public: - -void WrappingStore::queryReferrers(const Path& path, PathSet& referrers) { - wrappedStore->queryReferrers(path, referrers); -} - -PathSet WrappingStore::queryValidDerivers(const Path& path) { - return wrappedStore->queryValidDerivers(path); -} - -PathSet WrappingStore::queryDerivationOutputs(const Path& path) { - return wrappedStore->queryDerivationOutputs(path); -} - -StringSet WrappingStore::queryDerivationOutputNames(const Path& path) { - return wrappedStore->queryDerivationOutputNames(path); -} - -Path WrappingStore::queryPathFromHashPart(const string& hashPart) { - return wrappedStore->queryPathFromHashPart(hashPart); -} - -PathSet WrappingStore::querySubstitutablePaths(const PathSet& paths) { - return wrappedStore->querySubstitutablePaths(paths); -} - -void WrappingStore::querySubstitutablePathInfos(const PathSet& paths, - SubstitutablePathInfos& infos) { - wrappedStore->querySubstitutablePathInfos(paths, infos); -} - -bool WrappingStore::wantMassQuery() { - return wrappedStore->wantMassQuery(); -} - -void WrappingStore::addToStore(const ValidPathInfo& info, - Source& narSource, - RepairFlag repair, - CheckSigsFlag checkSigs, - std::shared_ptr accessor) { - wrappedStore->addToStore(info, narSource, repair, checkSigs, accessor); -} - -void WrappingStore::addToStore(const ValidPathInfo& info, - const ref& nar, - RepairFlag repair, - CheckSigsFlag checkSigs, - std::shared_ptr accessor) { - wrappedStore->addToStore(info, nar, repair, checkSigs, accessor); -} - -Path WrappingStore::addToStore(const string& name, - const Path& srcPath, - bool recursive, - HashType hashAlgo, - PathFilter& filter, - RepairFlag repair) { - return wrappedStore->addToStore(name, srcPath, recursive, hashAlgo, filter, - repair); -} - -Path WrappingStore::addTextToStore(const string& name, - const string& s, - const PathSet& references, - RepairFlag repair) { - return wrappedStore->addTextToStore(name, s, references, repair); -} - -void WrappingStore::narFromPath(const Path& path, Sink& sink) { - wrappedStore->narFromPath(path, sink); -} - -void WrappingStore::buildPaths(const PathSet& paths, BuildMode buildMode) { - wrappedStore->buildPaths(paths, buildMode); -} - -BuildResult WrappingStore::buildDerivation(const Path& drvPath, - const BasicDerivation& drv, - BuildMode buildMode) { - return wrappedStore->buildDerivation(drvPath, drv, buildMode); -} - -void WrappingStore::ensurePath(const Path& path) { - wrappedStore->ensurePath(path); -} - -void WrappingStore::addTempRoot(const Path& path) { - wrappedStore->addTempRoot(path); -} - -void WrappingStore::addIndirectRoot(const Path& path) { - wrappedStore->addIndirectRoot(path); -} - -void WrappingStore::syncWithGC() { - wrappedStore->syncWithGC(); -} - -void WrappingStore::collectGarbage(const GCOptions& options, - GCResults& results) { - wrappedStore->collectGarbage(options, results); -} - -void WrappingStore::optimiseStore() { - wrappedStore->optimiseStore(); -}; - -bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) { - return wrappedStore->verifyStore(checkContents, repair); -}; - -ref WrappingStore::getFSAccessor() { - return wrappedStore->getFSAccessor(); -} - -void WrappingStore::addSignatures(const Path& storePath, - const StringSet& sigs) { - wrappedStore->addSignatures(storePath, sigs); -}; - -void WrappingStore::computeFSClosure(const PathSet& paths, - PathSet& out, - bool flipDirection, - bool includeOutputs, - bool includeDerivers) { - wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs, - includeDerivers); -} - -void WrappingStore::queryMissing(const PathSet& targets, - PathSet& willBuild, - PathSet& willSubstitute, - PathSet& unknown, - unsigned long long& downloadSize, - unsigned long long& narSize) { - wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown, - downloadSize, narSize); -} - -std::shared_ptr WrappingStore::getBuildLog(const Path& path) { - return wrappedStore->getBuildLog(path); -} - -void WrappingStore::connect() { - wrappedStore->connect(); -}; - -int WrappingStore::getPriority() { - return wrappedStore->getPriority(); -} - -Path WrappingStore::toRealPath(const Path& storePath) { - return wrappedStore->toRealPath(storePath); -}; - -///// - -HerculesStore::HerculesStore(const Params& params, ref storeToWrap) - : WrappingStore(params, storeToWrap) {} - -void HerculesStore::ensurePath(const Path& path) { - /* We avoid asking substituters for paths, since - those would yield negative pathInfo caches on remote store. - - Instead, we only assert if path exists in the store. - - Once IFD build is performed, we ask for substitution - via ensurePath. - */ - if (!wrappedStore->isValidPath(path)) { - std::exception_ptr exceptionToThrow(nullptr); - builderCallback(strdup(path.c_str()), &exceptionToThrow); - if (exceptionToThrow != nullptr) { - std::rethrow_exception(exceptionToThrow); - } - wrappedStore->ensurePath(path); - } - ensuredPaths.insert(path); -}; - -// Avoid substituting in evaluator, see `ensurePath` for more details -void HerculesStore::queryMissing(const PathSet& targets, - PathSet& willBuild, - PathSet& willSubstitute, - PathSet& unknown, - unsigned long long& downloadSize, - unsigned long long& narSize) { -}; - -void HerculesStore::buildPaths(const PathSet& paths, BuildMode buildMode) { - for (Path path : paths) { - std::exception_ptr exceptionToThrow(nullptr); - builderCallback(strdup(path.c_str()), &exceptionToThrow); - if (exceptionToThrow != nullptr) { - std::rethrow_exception(exceptionToThrow); - } - } -} - -BuildResult HerculesStore::buildDerivation(const Path& drvPath, - const BasicDerivation& drv, - BuildMode buildMode) { - unsupported("buildDerivation"); - - std::cerr << "building derivation " << drvPath << std::endl; - auto r = wrappedStore->buildDerivation(drvPath, drv, buildMode); - std::cerr << "built derivation " << drvPath << std::endl; - return r; -} - -void HerculesStore::printDiagnostics() { - for (std::string path : ensuredPaths) { - std::cerr << path << std::endl; - } -} - -void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(const char *, std::exception_ptr *exceptionToThrow)) { - builderCallback = newBuilderCallback; -} diff --git a/cbits/hercules-store.cxx b/cbits/hercules-store.cxx new file mode 100644 --- /dev/null +++ b/cbits/hercules-store.cxx @@ -0,0 +1,271 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hercules-store.hh" + +using namespace nix; + +WrappingStore::WrappingStore(const Params& params, ref storeToWrap) + : Store(params), wrappedStore(storeToWrap) {} + +WrappingStore::~WrappingStore() {} + +std::string WrappingStore::getUri() { + return "wrapped:" + wrappedStore->getUri(); +}; + +bool WrappingStore::isValidPathUncached(const Path& path) { + return wrappedStore->isValidPath(path); // not ideal +} + +PathSet WrappingStore::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + return wrappedStore->queryValidPaths(paths, maybeSubstitute); +} +PathSet WrappingStore::queryAllValidPaths() { + return wrappedStore->queryAllValidPaths(); +} + +// protected: +void WrappingStore::queryPathInfoUncached( + const Path& path, + Callback> callback) noexcept { + unsupported("queryPathInfoUncached"); + /* + Callback> + callback2([&callback](std::future> vpi){ + std::shared_ptr !@#$%^&* + callback(vpi); + }); + return wrappedStore->queryPathInfo(path, callback2); // not ideal + */ +} + +// public: + +void WrappingStore::queryReferrers(const Path& path, PathSet& referrers) { + wrappedStore->queryReferrers(path, referrers); +} + +PathSet WrappingStore::queryValidDerivers(const Path& path) { + return wrappedStore->queryValidDerivers(path); +} + +PathSet WrappingStore::queryDerivationOutputs(const Path& path) { + return wrappedStore->queryDerivationOutputs(path); +} + +StringSet WrappingStore::queryDerivationOutputNames(const Path& path) { + return wrappedStore->queryDerivationOutputNames(path); +} + +Path WrappingStore::queryPathFromHashPart(const string& hashPart) { + return wrappedStore->queryPathFromHashPart(hashPart); +} + +PathSet WrappingStore::querySubstitutablePaths(const PathSet& paths) { + return wrappedStore->querySubstitutablePaths(paths); +} + +void WrappingStore::querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + wrappedStore->querySubstitutablePathInfos(paths, infos); +} + +bool WrappingStore::wantMassQuery() { + return wrappedStore->wantMassQuery(); +} + +void WrappingStore::addToStore(const ValidPathInfo& info, + Source& narSource, + RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr accessor) { + wrappedStore->addToStore(info, narSource, repair, checkSigs, accessor); +} + +void WrappingStore::addToStore(const ValidPathInfo& info, + const ref& nar, + RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr accessor) { + wrappedStore->addToStore(info, nar, repair, checkSigs, accessor); +} + +Path WrappingStore::addToStore(const string& name, + const Path& srcPath, + bool recursive, + HashType hashAlgo, + PathFilter& filter, + RepairFlag repair) { + return wrappedStore->addToStore(name, srcPath, recursive, hashAlgo, filter, + repair); +} + +Path WrappingStore::addTextToStore(const string& name, + const string& s, + const PathSet& references, + RepairFlag repair) { + return wrappedStore->addTextToStore(name, s, references, repair); +} + +void WrappingStore::narFromPath(const Path& path, Sink& sink) { + wrappedStore->narFromPath(path, sink); +} + +void WrappingStore::buildPaths(const PathSet& paths, BuildMode buildMode) { + wrappedStore->buildPaths(paths, buildMode); +} + +BuildResult WrappingStore::buildDerivation(const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + return wrappedStore->buildDerivation(drvPath, drv, buildMode); +} + +void WrappingStore::ensurePath(const Path& path) { + wrappedStore->ensurePath(path); +} + +void WrappingStore::addTempRoot(const Path& path) { + wrappedStore->addTempRoot(path); +} + +void WrappingStore::addIndirectRoot(const Path& path) { + wrappedStore->addIndirectRoot(path); +} + +void WrappingStore::syncWithGC() { + wrappedStore->syncWithGC(); +} + +void WrappingStore::collectGarbage(const GCOptions& options, + GCResults& results) { + wrappedStore->collectGarbage(options, results); +} + +void WrappingStore::optimiseStore() { + wrappedStore->optimiseStore(); +}; + +bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) { + return wrappedStore->verifyStore(checkContents, repair); +}; + +ref WrappingStore::getFSAccessor() { + return wrappedStore->getFSAccessor(); +} + +void WrappingStore::addSignatures(const Path& storePath, + const StringSet& sigs) { + wrappedStore->addSignatures(storePath, sigs); +}; + +void WrappingStore::computeFSClosure(const PathSet& paths, + PathSet& out, + bool flipDirection, + bool includeOutputs, + bool includeDerivers) { + wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs, + includeDerivers); +} + +void WrappingStore::queryMissing(const PathSet& targets, + PathSet& willBuild, + PathSet& willSubstitute, + PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) { + wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown, + downloadSize, narSize); +} + +std::shared_ptr WrappingStore::getBuildLog(const Path& path) { + return wrappedStore->getBuildLog(path); +} + +void WrappingStore::connect() { + wrappedStore->connect(); +}; + +int WrappingStore::getPriority() { + return wrappedStore->getPriority(); +} + +Path WrappingStore::toRealPath(const Path& storePath) { + return wrappedStore->toRealPath(storePath); +}; + +///// + +HerculesStore::HerculesStore(const Params& params, ref storeToWrap) + : WrappingStore(params, storeToWrap) {} + +void HerculesStore::ensurePath(const Path& path) { + /* We avoid asking substituters for paths, since + those would yield negative pathInfo caches on remote store. + + Instead, we only assert if path exists in the store. + + Once IFD build is performed, we ask for substitution + via ensurePath. + */ + if (!wrappedStore->isValidPath(path)) { + std::exception_ptr exceptionToThrow(nullptr); + builderCallback(strdup(path.c_str()), &exceptionToThrow); + if (exceptionToThrow != nullptr) { + std::rethrow_exception(exceptionToThrow); + } + wrappedStore->ensurePath(path); + } + ensuredPaths.insert(path); +}; + +// Avoid substituting in evaluator, see `ensurePath` for more details +void HerculesStore::queryMissing(const PathSet& targets, + PathSet& willBuild, + PathSet& willSubstitute, + PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) { +}; + +void HerculesStore::buildPaths(const PathSet& paths, BuildMode buildMode) { + for (Path path : paths) { + std::exception_ptr exceptionToThrow(nullptr); + builderCallback(strdup(path.c_str()), &exceptionToThrow); + if (exceptionToThrow != nullptr) { + std::rethrow_exception(exceptionToThrow); + } + } +} + +BuildResult HerculesStore::buildDerivation(const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + unsupported("buildDerivation"); + + std::cerr << "building derivation " << drvPath << std::endl; + auto r = wrappedStore->buildDerivation(drvPath, drv, buildMode); + std::cerr << "built derivation " << drvPath << std::endl; + return r; +} + +void HerculesStore::printDiagnostics() { + for (std::string path : ensuredPaths) { + std::cerr << path << std::endl; + } +} + +void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(const char *, std::exception_ptr *exceptionToThrow)) { + builderCallback = newBuilderCallback; +} diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker.hs @@ -58,7 +58,7 @@ import Katip import qualified Language.C.Inline.Cpp.Exceptions as C import qualified Network.URI -import Protolude hiding (bracket, catch, evalState, wait, withAsync) +import Protolude hiding (bracket, catch, evalState, wait, withAsync, yield) import qualified System.Environment as Environment import System.IO (BufferMode (LineBuffering), hSetBuffering) import System.Posix.IO (dup, fdToHandle, stdError) @@ -162,10 +162,10 @@ ) renderException :: SomeException -> Text -renderException e | Just (C.CppStdException msg) <- fromException e = toSL msg +renderException e | Just (C.CppStdException msg) <- fromException e = toS msg renderException e | Just (C.CppOtherException maybeType) <- fromException e = - "Unexpected C++ exception" <> foldMap (\t -> " of type " <> toSL t) maybeType + "Unexpected C++ exception" <> foldMap (\t -> " of type " <> toS t) maybeType renderException e | Just (FatalError msg) <- fromException e = msg renderException e = toS $ displayException e @@ -305,7 +305,7 @@ checkVersion = Socket.checkVersion', baseURL = baseURL, path = LogSettings.path l, - token = toSL $ LogSettings.reveal $ LogSettings.token l + token = encodeUtf8 $ LogSettings.reveal $ LogSettings.token l } -- TODO: test @@ -313,8 +313,8 @@ autoArgArgs kvs = do (k, v) <- M.toList kvs case v of - Eval.LiteralArg s -> ["--argstr", toS k, s] - Eval.ExprArg s -> ["--arg", toS k, s] + Eval.LiteralArg s -> ["--argstr", encodeUtf8 k, s] + Eval.ExprArg s -> ["--arg", encodeUtf8 k, s] withDrvInProgress :: MonadUnliftIO m => HerculesState -> Text -> m a -> m a withDrvInProgress HerculesState {drvsInProgress = ref} drvPath = @@ -371,16 +371,17 @@ let store = nixStore hStore s <- storeUri store UnliftIO unlift <- lift askUnliftIO - liftIO $ setBuilderCallback hStore $ \path -> unlift $ katipAddContext (sl "fullpath" (toSL path :: Text)) $ do + let decode = decodeUtf8With lenientDecode + liftIO $ setBuilderCallback hStore $ \path -> unlift $ katipAddContext (sl "fullpath" (decode path)) $ do logLocM DebugS "Building" let (plainDrv, bangOut) = BS.span (/= fromIntegral (ord '!')) path outputName = BS.dropWhile (== fromIntegral (ord '!')) bangOut - plainDrvText = toS plainDrv + plainDrvText = decode plainDrv withDrvInProgress st plainDrvText $ do - liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (toSL outputName) Nothing + liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (decode outputName) Nothing derivation <- liftIO $ getDerivation store plainDrv outputPath <- liftIO $ getDerivationOutputPath derivation outputName - katipAddContext (sl "outputPath" (toSL outputPath :: Text)) $ do + katipAddContext (sl "outputPath" (decode outputPath)) $ do logLocM DebugS "Naively calling ensurePath" liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e0 -> do katipAddContext (sl "message" (show (e0 :: SomeException) :: Text)) $ @@ -395,7 +396,7 @@ liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e1 -> do katipAddContext (sl "message" (show (e1 :: SomeException) :: Text)) $ logLocM DebugS "Recovering from fresh ensurePath" - liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (toSL outputName) (Just attempt0) + liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (decode outputName) (Just attempt0) -- TODO sync result' <- liftIO $ atomically $ do @@ -416,7 +417,7 @@ ) logLocM DebugS ("Built") withEvalState store $ \evalState -> do - katipAddContext (sl "storeURI" (toSL s :: Text)) $ + katipAddContext (sl "storeURI" (decode s)) $ logLocM DebugS "EvalState loaded." args <- liftIO $ diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs @@ -16,13 +16,13 @@ import qualified Hercules.Agent.WorkerProtocol.Event as Event import qualified Hercules.Agent.WorkerProtocol.Event.BuildResult as Event.BuildResult import Katip -import Protolude +import Protolude hiding (yield) import Unsafe.Coerce runBuild :: (MonadIO m, KatipContext m) => Ptr (Ref NixStore) -> Command.Build.Build -> ConduitT i Event m () runBuild store build = do let extraPaths = Command.Build.inputDerivationOutputPaths build - drvPath = toS $ Command.Build.drvPath build + drvPath = encodeUtf8 $ Command.Build.drvPath build x <- for extraPaths $ \input -> do liftIO $ try $ CNix.ensurePath store input materialize <- case sequenceA x of diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs @@ -18,7 +18,7 @@ import Katip import qualified Language.C.Inline.Cpp as C import qualified Language.C.Inline.Cpp.Exceptions as C -import Protolude hiding (bracket, finally, mask_, onException, tryJust, wait, withAsync) +import Protolude hiding (bracket, finally, mask_, onException, tryJust, wait, withAsync, yield) import System.IO (BufferMode (LineBuffering), hSetBuffering) import System.IO.Error (isEOFError) import System.Posix.IO (closeFd, createPipe, dup, dupTo, fdToHandle, stdError) @@ -38,7 +38,7 @@ C.include "" -C.include "aliases.h" +C.include "hercules-aliases.h" C.include "hercules-logger.hh" @@ -166,7 +166,7 @@ { i = i_, ms = ms_, level = fromIntegral level_, - msg = toSL text_ + msg = decode text_ } 2 -> do text_ <- unsafePackMallocCString =<< peek textStrPtr @@ -181,7 +181,7 @@ act = LogEntry.ActivityId act_, level = fromIntegral level_, typ = LogEntry.ActivityType typ_, - text = toSL text_, + text = decode text_, parent = LogEntry.ActivityId parent_, fields = fields_ } @@ -231,9 +231,12 @@ }|] >>= \case 0 -> LogEntry.Int <$> peek uintPtr - 1 -> LogEntry.String . toSL <$> unsafeMallocBS (peek stringPtr) + 1 -> LogEntry.String . decode <$> unsafeMallocBS (peek stringPtr) _ -> panic "convertAndDeleteFields invalid internal type" +decode :: ByteString -> Text +decode = decodeUtf8With lenientDecode + close :: IO () close = [C.throwBlock| void { @@ -314,7 +317,7 @@ Left _ -> pass Right "__%%hercules terminate log%%__" -> pass Right ln -> do - katipAddContext (sl "line" (toSL ln :: Text)) + katipAddContext (sl "line" (decode ln)) $ logLocM DebugS $ "Intercepted stderr" liftIO diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs @@ -38,7 +38,7 @@ C.include "" -C.include "aliases.h" +C.include "hercules-aliases.h" C.using "namespace nix" diff --git a/hercules-ci-agent.cabal b/hercules-ci-agent.cabal --- a/hercules-ci-agent.cabal +++ b/hercules-ci-agent.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: hercules-ci-agent -version: 0.7.4 +version: 0.7.5 synopsis: Runs Continuous Integration tasks on your machines category: Nix, CI, Testing, DevOps homepage: https://docs.hercules-ci.com @@ -13,7 +13,7 @@ build-type: Simple extra-source-files: CHANGELOG.md - cbits/aliases.h + cbits/hercules-aliases.h cbits/hercules-store.hh cbits/hercules-logger.hh testdata/vm-test-run-agent-test.drv @@ -22,6 +22,29 @@ type: git location: https://github.com/hercules-ci/hercules-ci-agent +-- match the C++ language standard Nix is using +common cxx-opts + cxx-options: + -std=c++17 + -Wall + extra-libraries: stdc++ + + if os(darwin) + -- avoid https://gitlab.haskell.org/ghc/ghc/issues/11829 + ld-options: -Wl,-keep_dwarf_unwind + + if impl(ghc >= 8.10) + ghc-options: + -optcxx-std=c++17 + -optcxx-Wall + else + ghc-options: + -optc-std=c++17 + -optc-Wall + if os(darwin) + ghc-options: -pgmc=clang++ + + library exposed-modules: Data.Fixed.Extras @@ -68,7 +91,7 @@ , mtl , network-uri , optparse-applicative - , protolude + , protolude >= 0.3 , process , safe-exceptions , stm @@ -92,6 +115,7 @@ default-language: Haskell2010 library cnix + import: cxx-opts exposed-modules: CNix CNix.Internal.Context @@ -102,15 +126,15 @@ include-dirs: cbits cxx-sources: - cbits/hercules-store.cc - cbits/hercules-logger.cc + cbits/hercules-store.cxx + cbits/hercules-logger.cxx build-depends: base , inline-c , inline-c-cpp , internal-ffi , bytestring - , cachix >= 0.3.8 + , cachix >= 0.5.1 , conduit , containers , protolude @@ -121,7 +145,6 @@ , nix-main >= 2.0 , bdw-gc extra-libraries: - stdc++ boost_context default-language: Haskell2010 default-extensions: DeriveGeneric DeriveTraversable DisambiguateRecordFields FlexibleContexts InstanceSigs LambdaCase MultiParamTypeClasses NoImplicitPrelude OverloadedStrings RankNTypes TupleSections TypeApplications TypeOperators @@ -133,13 +156,6 @@ -fwarn-missing-signatures -fwarn-name-shadowing -fwarn-incomplete-patterns - -- match what Nix is using - -optc-std=c++17 - cxx-options: -Wall -std=c++17 - if os(darwin) - -- avoid https://gitlab.haskell.org/ghc/ghc/issues/11829 - ld-options: -Wl,-keep_dwarf_unwind - ghc-options: -pgmc=clang++ executable hercules-ci-agent main-is: Main.hs @@ -243,6 +259,7 @@ default-language: Haskell2010 executable hercules-ci-agent-worker + import: cxx-opts main-is: Main.hs other-modules: Hercules.Agent.Worker @@ -266,17 +283,9 @@ -threaded -rtsopts -with-rtsopts=-maxN8 - -- match what Nix is using - -optc-std=c++17 - cxx-options: -Wall -std=c++17 - if os(darwin) - -- avoid https://gitlab.haskell.org/ghc/ghc/issues/11829 - ld-options: -Wl,-keep_dwarf_unwind - ghc-options: -pgmc=clang++ include-dirs: cbits extra-libraries: - stdc++ boost_context build-depends: aeson diff --git a/hercules-ci-agent/Hercules/Agent.hs b/hercules-ci-agent/Hercules/Agent.hs --- a/hercules-ci-agent/Hercules/Agent.hs +++ b/hercules-ci-agent/Hercules/Agent.hs @@ -173,7 +173,7 @@ report' (Left e) = withNamedContext "message" (displayException e) $ withNamedContext "exception" (show e :: Text) do logLocM ErrorS "Exception in task" - report $ TaskStatus.Exceptional $ toSL $ displayException e + report $ TaskStatus.Exceptional $ toS $ displayException e -- TODO use socket report status = retry (cap 60 exponential) diff --git a/hercules-ci-agent/Hercules/Agent/Build.hs b/hercules-ci-agent/Hercules/Agent/Build.hs --- a/hercules-ci-agent/Hercules/Agent/Build.hs +++ b/hercules-ci-agent/Hercules/Agent/Build.hs @@ -54,14 +54,14 @@ writeEvent event = case event of Event.BuildResult r -> writeIORef statusRef $ Just r Event.Exception e -> do - logLocM DebugS $ show e + logLocM DebugS $ logStr (show e :: Text) panic e _ -> pass baseURL <- asks (ServiceInfo.bulkSocketBaseURL . Env.serviceInfo) materialize <- asks (not . Config.nixUserIsTrusted . Env.config) liftIO $ writeChan commandChan $ Just $ Command.Build $ Command.Build.Build { drvPath = BuildTask.derivationPath buildTask, - inputDerivationOutputPaths = toS <$> BuildTask.inputDerivationOutputPaths buildTask, + inputDerivationOutputPaths = encodeUtf8 <$> BuildTask.inputDerivationOutputPaths buildTask, logSettings = LogSettings.LogSettings { token = LogSettings.Sensitive $ BuildTask.logToken buildTask, path = "/api/v1/logs/build/socket", @@ -70,7 +70,7 @@ materializeDerivation = materialize } exitCode <- runWorker procSpec (stderrLineHandler "Builder") commandChan writeEvent - logLocM DebugS $ "Worker exit: " <> show exitCode + logLocM DebugS $ "Worker exit: " <> logStr (show exitCode :: Text) case exitCode of ExitSuccess -> pass _ -> panic $ "Worker failed: " <> show exitCode @@ -87,13 +87,13 @@ convertOutputs :: Text -> [BuildResult.OutputInfo] -> Map Text OutputInfo convertOutputs deriver = foldMap $ \oi -> - M.singleton (toS $ BuildResult.name oi) $ + M.singleton (decodeUtf8With lenientDecode $ BuildResult.name oi) $ OutputInfo.OutputInfo { OutputInfo.deriver = deriver, - name = toSL $ BuildResult.name oi, - path = toSL $ BuildResult.path oi, + name = decodeUtf8With lenientDecode $ BuildResult.name oi, + path = decodeUtf8With lenientDecode $ BuildResult.path oi, size = fromIntegral $ BuildResult.size oi, - hash = toSL $ BuildResult.hash oi + hash = decodeUtf8With lenientDecode $ BuildResult.hash oi } push :: BuildTask -> Map Text OutputInfo -> App () diff --git a/hercules-ci-agent/Hercules/Agent/Cache.hs b/hercules-ci-agent/Hercules/Agent/Cache.hs --- a/hercules-ci-agent/Hercules/Agent/Cache.hs +++ b/hercules-ci-agent/Hercules/Agent/Cache.hs @@ -33,7 +33,7 @@ T.hPutStrLn netrcHandle (T.unlines netrcLns) hClose netrcHandle Nix.withExtraOptions - [ ("netrc-file", toSL netrcPath), + [ ("netrc-file", toS netrcPath), ("substituters", T.intercalate " " (substs <> csubsts)), ("trusted-public-keys", T.intercalate " " (pubkeys <> cpubkeys)) ] @@ -66,7 +66,7 @@ CNix.withStore $ \store -> do (Sum total, Sum signed) <- mconcat <$> for keys \key -> liftIO $ do - pathSet <- paths & map toS & newPathSetWith + pathSet <- paths & newPathSetWith signClosure store key pathSet katipAddContext (sl "num-signatures" signed <> sl "num-paths" total) $ logLocM DebugS "Signed" diff --git a/hercules-ci-agent/Hercules/Agent/Cachix/Init.hs b/hercules-ci-agent/Hercules/Agent/Cachix/Init.hs --- a/hercules-ci-agent/Hercules/Agent/Cachix/Init.hs +++ b/hercules-ci-agent/Hercules/Agent/Cachix/Init.hs @@ -26,7 +26,7 @@ tlsManagerSettings { managerResponseTimeout = responseTimeoutNone, -- managerModifyRequest :: Request -> IO Request - managerModifyRequest = return . setRequestHeader "User-Agent" [toS cachixVersion] + managerModifyRequest = return . setRequestHeader "User-Agent" [encodeUtf8 cachixVersion] } newEnv :: Config.FinalConfig -> Map Text CachixCache.CachixCache -> K.KatipContextT IO Env.Env @@ -61,8 +61,10 @@ sk <- head $ CachixCache.signingKeys keys Just $ escalateAs FatalError $ do k' <- Cachix.Secrets.parseSigningKeyLenient sk - pure $ Cachix.Push.PushCache + pure Cachix.Push.PushCache { pushCacheName = name, - pushCacheSigningKey = k', - pushCacheToken = Servant.Auth.Client.Token $ toSL t + pushCacheSecret = + Cachix.Push.PushSigningKey + (Servant.Auth.Client.Token $ encodeUtf8 t) + k' } diff --git a/hercules-ci-agent/Hercules/Agent/Config.hs b/hercules-ci-agent/Hercules/Agent/Config.hs --- a/hercules-ci-agent/Hercules/Agent/Config.hs +++ b/hercules-ci-agent/Hercules/Agent/Config.hs @@ -84,8 +84,9 @@ determineDefaultApiBaseUrl :: IO Text determineDefaultApiBaseUrl = do - maybeEnv <- System.Environment.lookupEnv "HERCULES_API_BASE_URL" - pure $ maybe defaultApiBaseUrl toS maybeEnv + maybeEnv <- System.Environment.lookupEnv "HERCULES_CI_API_BASE_URL" + maybeEnv' <- System.Environment.lookupEnv "HERCULES_API_BASE_URL" + pure $ maybe defaultApiBaseUrl toS (maybeEnv <|> maybeEnv') defaultApiBaseUrl :: Text defaultApiBaseUrl = "https://hercules-ci.com" @@ -95,7 +96,7 @@ readConfig :: ConfigPath -> IO (Config 'Input) readConfig loc = case loc of - TomlPath fp -> Toml.decodeFile tomlCodec (toSL fp) + TomlPath fp -> Toml.decodeFile tomlCodec (toS fp) finalizeConfig :: ConfigPath -> Config 'Input -> IO (Config 'Final) finalizeConfig loc input = do diff --git a/hercules-ci-agent/Hercules/Agent/Config/BinaryCaches.hs b/hercules-ci-agent/Hercules/Agent/Config/BinaryCaches.hs --- a/hercules-ci-agent/Hercules/Agent/Config/BinaryCaches.hs +++ b/hercules-ci-agent/Hercules/Agent/Config/BinaryCaches.hs @@ -75,8 +75,8 @@ forM_ (M.toList (unknownKinds bcs)) $ \(k, v) -> logLocM WarningS $ "In file " - <> show fname + <> logStr fname <> " in entry " - <> show k + <> logStr k <> ": unknown kind " - <> show (kind v) + <> logStr (kind v) diff --git a/hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs b/hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs --- a/hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs +++ b/hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs @@ -12,6 +12,7 @@ _String, key, ) +import qualified Data.ByteString.Lazy as LBS import Data.Char (isSpace) import qualified Data.Text as T import qualified Hercules.API.Agent.LifeCycle.AgentInfo as AgentInfo @@ -42,7 +43,7 @@ substituters = nixSubstituters nix, -- TODO: Add cachix substituters concurrentTasks = fromIntegral concurrentTasks } - logLocM DebugS $ "Determined environment info: " <> show s + logLocM DebugS $ "Determined environment info: " <> logStr (show s :: Text) pure s data NixInfo @@ -61,11 +62,11 @@ version <- Process.readProcess "nix" ["--version"] stdinEmpty rawJson <- Process.readProcess "nix" ["show-config", "--json"] stdinEmpty cfg <- - case Aeson.eitherDecode (toS rawJson) of + case Aeson.eitherDecode (LBS.fromStrict $ encodeUtf8 $ toS $ rawJson) of Left e -> panic $ "Could not parse nix show-config --json: " <> show e Right r -> pure r pure NixInfo - { nixExeVersion = T.dropAround isSpace (toSL version), + { nixExeVersion = T.dropAround isSpace (toS version), nixPlatforms = ((cfg :: Aeson.Value) ^.. key "system" . key "value" . _String) <> ( cfg diff --git a/hercules-ci-agent/Hercules/Agent/Evaluate.hs b/hercules-ci-agent/Hercules/Agent/Evaluate.hs --- a/hercules-ci-agent/Hercules/Agent/Evaluate.hs +++ b/hercules-ci-agent/Hercules/Agent/Evaluate.hs @@ -135,7 +135,7 @@ <> identifier ) let autoArguments = autoArguments' - <&> \sp -> Eval.ExprArg $ toS $ renderSubPath $ toS <$> sp + <&> \sp -> Eval.ExprArg $ encodeUtf8 $ renderSubPath $ toS <$> sp msgCounter <- liftIO $ newIORef 0 let fixIndex :: MonadIO m => @@ -253,6 +253,7 @@ buildRequiredIndex <- liftIO $ newIORef (0 :: Int) commandChan <- newChan writeChan commandChan $ Just $ Command.Eval eval + let decode = decodeUtf8With lenientDecode withProducer (produceWorkerEvents eval nixPath commandChan) $ \workerEventsP -> fix $ \continue -> joinSTM $ @@ -261,16 +262,16 @@ ( \case Event.Attribute a -> do emit $ EvaluateEvent.Attribute $ AttributeEvent.AttributeEvent - { AttributeEvent.expressionPath = toSL <$> WorkerAttribute.path a, - AttributeEvent.derivationPath = toSL $ WorkerAttribute.drv a + { AttributeEvent.expressionPath = decode <$> WorkerAttribute.path a, + AttributeEvent.derivationPath = decode $ WorkerAttribute.drv a } continue Event.AttributeError e -> do emit $ EvaluateEvent.AttributeError $ AttributeErrorEvent.AttributeErrorEvent - { AttributeErrorEvent.expressionPath = toSL <$> WorkerAttributeError.path e, - AttributeErrorEvent.errorMessage = toSL $ WorkerAttributeError.message e, - AttributeErrorEvent.errorType = toSL <$> WorkerAttributeError.errorType e, - AttributeErrorEvent.errorDerivation = toSL <$> WorkerAttributeError.errorDerivation e + { AttributeErrorEvent.expressionPath = decode <$> WorkerAttributeError.path e, + AttributeErrorEvent.errorMessage = WorkerAttributeError.message e, + AttributeErrorEvent.errorType = WorkerAttributeError.errorType e, + AttributeErrorEvent.errorDerivation = WorkerAttributeError.errorDerivation e } continue Event.EvaluationDone -> @@ -307,7 +308,7 @@ } flush status <- drvPoller notAttempt drv - logLocM DebugS $ "Got derivation status " <> show status + logLocM DebugS $ "Got derivation status " <> logStr (show status :: Text) return status writeChan commandChan $ Just $ Command.BuildResult $ uncurry (BuildResult.BuildResult drv) status continue diff --git a/hercules-ci-agent/Hercules/Agent/Init.hs b/hercules-ci-agent/Hercules/Agent/Init.hs --- a/hercules-ci-agent/Hercules/Agent/Init.hs +++ b/hercules-ci-agent/Hercules/Agent/Init.hs @@ -22,7 +22,7 @@ newEnv config logEnv = do let withLogging :: K.KatipContextT IO a -> IO a withLogging = K.runKatipContextT logEnv () "Init" - withLogging $ K.logLocM K.DebugS $ "Config: " <> show config + withLogging $ K.logLocM K.DebugS $ "Config: " <> K.logStr (show config :: Text) System.Directory.createDirectoryIfMissing True (Config.workDirectory config) SecureDirectory.init config bcs <- withLogging $ BC.parseFile config diff --git a/hercules-ci-agent/Hercules/Agent/Log.hs b/hercules-ci-agent/Hercules/Agent/Log.hs --- a/hercules-ci-agent/Hercules/Agent/Log.hs +++ b/hercules-ci-agent/Hercules/Agent/Log.hs @@ -12,18 +12,13 @@ import Data.Aeson import qualified Data.Aeson as A import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as LBS import qualified Data.Map as M import Katip hiding (logLocM) import Katip.Core import Katip.Monadic (logLocM) import Protolude -instance StringConv [Char] LogStr where - strConv l = logStr . (strConv l :: [Char] -> Text) - -instance StringConv Text LogStr where - strConv _ = logStr - withNamedContext :: (ToJSON a, KatipContext m) => Text -> a -> m b -> m b withNamedContext name = katipAddContext . Katip.sl name @@ -33,17 +28,17 @@ -- TODO: Support context for all exceptions and use plain @panic@ instead. panicWithLog :: KatipContext m => Text -> m a panicWithLog msg = do - logLocM ErrorS $ toSL msg + logLocM ErrorS $ logStr msg panic msg stderrLineHandler :: KatipContext m => Text -> Int -> ByteString -> m () stderrLineHandler _processRole _ ln | "@katip " `BS.isPrefixOf` ln, - Just item <- A.decode (toS $ BS.drop 7 ln) = + Just item <- A.decode (LBS.fromStrict $ BS.drop 7 ln) = -- "This is the lowest level function [...] useful when implementing centralised logging services." Katip.Core.logKatipItem (Katip.Core.SimpleLogPayload . M.toList . fmap (Katip.Core.AnyLogPayload :: A.Value -> Katip.Core.AnyLogPayload) <$> item) stderrLineHandler processRole pid ln = withNamedContext "worker" (pid :: Int) $ logLocM InfoS $ logStr - $ processRole <> ": " <> toSL ln + $ processRole <> ": " <> decodeUtf8With lenientDecode ln diff --git a/hercules-ci-agent/Hercules/Agent/Nix.hs b/hercules-ci-agent/Hercules/Agent/Nix.hs --- a/hercules-ci-agent/Hercules/Agent/Nix.hs +++ b/hercules-ci-agent/Hercules/Agent/Nix.hs @@ -36,9 +36,9 @@ App Text readNixProcess cmd opts paths stdinText = do extraOpts <- getExtraOptionArguments - toSL <$> liftIO (readProcess (toSL cmd) (map toSL (opts <> extraOpts <> ["--"] <> paths)) (toSL stdinText)) + toS <$> liftIO (readProcess (toS cmd) (map toS (opts <> extraOpts <> ["--"] <> paths)) (toS stdinText)) nixProc :: Text -> [Text] -> [Text] -> App System.Process.CreateProcess nixProc exe opts args = do extraOpts <- getExtraOptionArguments - pure $ System.Process.proc (toSL exe) (map toSL (opts <> extraOpts <> ["--"] <> args)) + pure $ System.Process.proc (toS exe) (map toS (opts <> extraOpts <> ["--"] <> args)) diff --git a/hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs b/hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs --- a/hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs +++ b/hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs @@ -17,7 +17,7 @@ DerivationInfo.DerivationPathText -> m DerivationInfo retrieveDerivationInfo store drvPath = liftIO $ do - drv <- getDerivation store (toSL drvPath) + drv <- getDerivation store (encodeUtf8 drvPath) retrieveDerivationInfo' drvPath drv retrieveDerivationInfo' :: Text -> ForeignPtr Derivation -> IO DerivationInfo @@ -28,20 +28,21 @@ env <- getDerivationEnv drv platform <- getDerivationPlatform drv let requiredSystemFeatures = maybe [] splitFeatures $ M.lookup "requiredSystemFeatures" env - splitFeatures = filter (not . T.null) . T.split (== ' ') . toSL + splitFeatures = filter (not . T.null) . T.split (== ' ') . decode + decode = decodeUtf8With lenientDecode pure $ DerivationInfo { derivationPath = drvPath, - platform = toSL platform, + platform = decodeUtf8With lenientDecode platform, requiredSystemFeatures = requiredSystemFeatures, - inputDerivations = inputDrvPaths & map (\(i, os) -> (toSL i, map toSL os)) & M.fromList, - inputSources = map toSL sourcePaths, + inputDerivations = inputDrvPaths & map (\(i, os) -> (decode i, map decode os)) & M.fromList, + inputSources = map decode sourcePaths, outputs = outputs & map ( \output -> - ( toSL $ derivationOutputName output, + ( decode $ derivationOutputName output, DerivationInfo.OutputInfo - { DerivationInfo.path = toSL $ derivationOutputPath output, + { DerivationInfo.path = decode $ derivationOutputPath output, DerivationInfo.isFixed = derivationOutputHashAlgo output /= "" } ) diff --git a/hercules-ci-agent/Hercules/Agent/Options.hs b/hercules-ci-agent/Hercules/Agent/Options.hs --- a/hercules-ci-agent/Hercules/Agent/Options.hs +++ b/hercules-ci-agent/Hercules/Agent/Options.hs @@ -44,7 +44,7 @@ (parseOptions <**> helper) ( fullDesc <> progDesc "Accepts tasks from Hercules CI and runs them." <> header - ("hercules-ci-agent " <> toSL herculesAgentVersion) + ("hercules-ci-agent " <> toS herculesAgentVersion) ) parse :: IO Options diff --git a/hercules-ci-agent/Hercules/Agent/Token.hs b/hercules-ci-agent/Hercules/Agent/Token.hs --- a/hercules-ci-agent/Hercules/Agent/Token.hs +++ b/hercules-ci-agent/Hercules/Agent/Token.hs @@ -39,7 +39,7 @@ readAgentSessionKey :: App (Maybe Text) readAgentSessionKey = do dir <- getDir - logLocM DebugS $ "Data directory: " <> show dir + logLocM DebugS $ "Data directory: " <> logStr (show dir :: Text) let file = dir "session.key" liftIO (System.Directory.doesFileExist file) >>= \case True -> notEmpty <$> readTokenFile file @@ -54,13 +54,13 @@ Just sessKey -> do logLocM DebugS "Found agent session key" let handler e = do - logLocM WarningS $ "Failed to check whether session token matches cluster join token. Using old session. Remove session.key if you need to force a session update. Exception: " <> show e + logLocM WarningS $ "Failed to check whether session token matches cluster join token. Using old session. Remove session.key if you need to force a session update. Exception: " <> logStr (displayException e) pure sessKey safeLiftedHandle handler $ do cjt <- getClusterJoinTokenId - logLocM DebugS $ "Found clusterJoinTokenId " <> show cjt + logLocM DebugS $ "Found clusterJoinTokenId " <> logStr cjt scjt <- getSessionClusterJoinTokenId sessKey - logLocM DebugS $ "Found sessionClusterJoinTokenId " <> show scjt + logLocM DebugS $ "Found sessionClusterJoinTokenId " <> logStr scjt if cjt == scjt then pure sessKey else do @@ -88,7 +88,7 @@ createAgentSession :: App Text createAgentSession = do agentInfo <- EnvironmentInfo.extractAgentInfo - logLocM DebugS $ "Agent info: " <> show agentInfo + logLocM DebugS $ "Agent info: " <> logStr (show agentInfo :: Text) let createAgentBody = CreateAgentSession.CreateAgentSession {agentInfo = agentInfo} token <- asks Env.currentToken @@ -105,7 +105,7 @@ withAgentToken :: App a -> App a withAgentToken m = do agentSessionToken <- ensureAgentSession - local (\env -> env {Env.currentToken = Token $ toS agentSessionToken}) m + local (\env -> env {Env.currentToken = Token $ encodeUtf8 agentSessionToken}) m data TokenError = TokenError Text deriving (Typeable, Show) @@ -117,13 +117,13 @@ t <- asks Env.currentToken >>= \case Token jwt -> pure jwt - v <- decodeToken (toSL t) + v <- decodeToken (BL.fromStrict t) sub <- escalate $ maybeToEither (TokenError "No sub field in cluster join token") (v ^? key "sub" . _String) escalate $ maybeToEither (TokenError "Unrecognized token type") $ T.stripPrefix "t$" sub getSessionClusterJoinTokenId :: Text -> App Text getSessionClusterJoinTokenId t = do - v <- decodeToken (toSL t) + v <- decodeToken (BL.fromStrict $ encodeUtf8 t) escalate $ maybeToEither (TokenError "No parent field in session token") (v ^? key "parent" . _String) decodeToken :: BL.ByteString -> App Aeson.Value diff --git a/src-cnix/CNix.hs b/src-cnix/CNix.hs --- a/src-cnix/CNix.hs +++ b/src-cnix/CNix.hs @@ -58,7 +58,7 @@ C.include "" -C.include "aliases.h" +C.include "hercules-aliases.h" C.include "" diff --git a/src-cnix/CNix/Internal/Raw.hs b/src-cnix/CNix/Internal/Raw.hs --- a/src-cnix/CNix/Internal/Raw.hs +++ b/src-cnix/CNix/Internal/Raw.hs @@ -17,7 +17,7 @@ C.include "" -C.include "aliases.h" +C.include "hercules-aliases.h" C.include "" diff --git a/src-cnix/CNix/Internal/Store.hs b/src-cnix/CNix/Internal/Store.hs --- a/src-cnix/CNix/Internal/Store.hs +++ b/src-cnix/CNix/Internal/Store.hs @@ -41,7 +41,7 @@ C.include "" -C.include "aliases.h" +C.include "hercules-aliases.h" C.using "namespace nix" diff --git a/src/Hercules/Agent/Socket.hs b/src/Hercules/Agent/Socket.hs --- a/src/Hercules/Agent/Socket.hs +++ b/src/Hercules/Agent/Socket.hs @@ -131,7 +131,7 @@ recv conn = do withTimeout ackTimeout (FatalError "Hercules.Agent.Socket.recv timed out") $ (liftIO $ A.eitherDecode <$> WS.receiveData conn) >>= \case - Left e -> liftIO $ throwIO (FatalError $ "Error decoding service message: " <> toSL e) + Left e -> liftIO $ throwIO (FatalError $ "Error decoding service message: " <> toS e) Right r -> pure r handshake conn = katipAddNamespace "Handshake" do siMsg <- recv conn