hercules-ci-agent 0.7.4 → 0.7.5
raw patch · 30 files changed
+521/−491 lines, 30 filesdep ~cachixdep ~protolude
Dependency ranges changed: cachix, protolude
Files
- CHANGELOG.md +16/−0
- cbits/aliases.h +0/−24
- cbits/hercules-aliases.h +24/−0
- cbits/hercules-logger.cc +0/−90
- cbits/hercules-logger.cxx +90/−0
- cbits/hercules-store.cc +0/−271
- cbits/hercules-store.cxx +271/−0
- hercules-ci-agent-worker/Hercules/Agent/Worker.hs +13/−12
- hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs +2/−2
- hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs +9/−6
- hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +1/−1
- hercules-ci-agent.cabal +31/−22
- hercules-ci-agent/Hercules/Agent.hs +1/−1
- hercules-ci-agent/Hercules/Agent/Build.hs +7/−7
- hercules-ci-agent/Hercules/Agent/Cache.hs +2/−2
- hercules-ci-agent/Hercules/Agent/Cachix/Init.hs +6/−4
- hercules-ci-agent/Hercules/Agent/Config.hs +4/−3
- hercules-ci-agent/Hercules/Agent/Config/BinaryCaches.hs +3/−3
- hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs +4/−3
- hercules-ci-agent/Hercules/Agent/Evaluate.hs +9/−8
- hercules-ci-agent/Hercules/Agent/Init.hs +1/−1
- hercules-ci-agent/Hercules/Agent/Log.hs +4/−9
- hercules-ci-agent/Hercules/Agent/Nix.hs +2/−2
- hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs +8/−7
- hercules-ci-agent/Hercules/Agent/Options.hs +1/−1
- hercules-ci-agent/Hercules/Agent/Token.hs +8/−8
- src-cnix/CNix.hs +1/−1
- src-cnix/CNix/Internal/Raw.hs +1/−1
- src-cnix/CNix/Internal/Store.hs +1/−1
- src/Hercules/Agent/Socket.hs +1/−1
CHANGELOG.md view
@@ -5,6 +5,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.7.5] - 2020-11-27++### Fixed++ - GHC 8.10.2 compatibility for NixOS unstable / NixOS 21.03++ - Build with cachix 0.5.1. Write token support is not included due to a break configuration semantics. It will be available in >= 0.8.++### Changed++ - The in-repo expressions have upgraded their dependency to NixOS 20.09++ - Agent will now use `HERCULES_CI_API_BASE_URL` over `HERCULES_API_BASE_URL` if set.++ - Temporarily switch to cabal tooling due to breakage. `master` continues to be stack-based.+ ## [0.7.4] - 2020-08-25 ### Fixed
− cbits/aliases.h
@@ -1,24 +0,0 @@-#pragma once--#include "hercules-store.hh"-#include "hercules-logger.hh"-#include "derivations.hh"--// inline-c-cpp doesn't seem to handle namespace operator or template-// syntax so we help it a bit for now. This definition can be inlined-// when it is supported by inline-c-cpp.-typedef nix::ref<nix::Store> refStore;--typedef nix::ref<HerculesStore> refHerculesStore;--typedef nix::Logger::Fields LoggerFields;--typedef HerculesLogger::LogEntry HerculesLoggerEntry;-typedef std::queue<std::unique_ptr<HerculesLogger::LogEntry>> LogEntryQueue;--typedef nix::Strings::iterator StringsIterator;-typedef nix::DerivationOutputs::iterator DerivationOutputsIterator;-typedef nix::DerivationInputs::iterator DerivationInputsIterator;-typedef nix::StringPairs::iterator StringPairsIterator;--using namespace std;
+ cbits/hercules-aliases.h view
@@ -0,0 +1,24 @@+#pragma once++#include "hercules-store.hh"+#include "hercules-logger.hh"+#include "derivations.hh"++// inline-c-cpp doesn't seem to handle namespace operator or template+// syntax so we help it a bit for now. This definition can be inlined+// when it is supported by inline-c-cpp.+typedef nix::ref<nix::Store> refStore;++typedef nix::ref<HerculesStore> refHerculesStore;++typedef nix::Logger::Fields LoggerFields;++typedef HerculesLogger::LogEntry HerculesLoggerEntry;+typedef std::queue<std::unique_ptr<HerculesLogger::LogEntry>> LogEntryQueue;++typedef nix::Strings::iterator StringsIterator;+typedef nix::DerivationOutputs::iterator DerivationOutputsIterator;+typedef nix::DerivationInputs::iterator DerivationInputsIterator;+typedef nix::StringPairs::iterator StringPairsIterator;++using namespace std;
− cbits/hercules-logger.cc
@@ -1,90 +0,0 @@-#include "hercules-logger.hh"--HerculesLogger::HerculesLogger()-{--}--void HerculesLogger::push(std::unique_ptr<LogEntry> entry) {- auto state(state_.lock());- state->queue.push(std::move(entry));- wakeup.notify_one();-}--uint64_t HerculesLogger::getMs() {- auto t = std::chrono::steady_clock::now();- auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(t - t_zero).count();- return millis;-}--void HerculesLogger::log(nix::Verbosity lvl, const nix::FormatOrString & fs) {- push(std::make_unique<LogEntry>(LogEntry {- .entryType = 1,- .level = lvl,- .ms = getMs(),- .text = fs.s- }));-}--void HerculesLogger::startActivity(nix::ActivityId act, nix::Verbosity lvl, nix::ActivityType type,- const std::string & s, const Fields & fields, nix::ActivityId parent) {- push(std::make_unique<LogEntry>(LogEntry {- .entryType = 2,- .level = lvl,- .ms = getMs(),- .text = s,- .activityId = act,- .type = type,- .parent = parent,- .fields = fields- }));-}--void HerculesLogger::stopActivity(nix::ActivityId act) {- push(std::make_unique<LogEntry>(LogEntry {- .entryType = 3,- .ms = getMs(),- .activityId = act- }));-}--void HerculesLogger::result(nix::ActivityId act, nix::ResultType type, const Fields & fields) {- push(std::make_unique<LogEntry>(LogEntry {- .entryType = 4,- .ms = getMs(),- .activityId = act,- .type = type,- .fields = fields- }));-}--void HerculesLogger::close() {- auto state(state_.lock());- state->queue.push(nullptr);- wakeup.notify_one();-}--std::unique_ptr<HerculesLogger::LogEntry> HerculesLogger::pop () {- auto state(state_.lock());-- while (state->queue.empty())- state.wait(wakeup);-- auto r = std::move(state->queue.front());- state->queue.pop();- return r;-}--void HerculesLogger::popMany (int max, std::queue<std::unique_ptr<LogEntry>> &out) {- auto state(state_.lock());-- while (state->queue.empty())- state.wait(wakeup);-- for (int i = 0; i < max && !state->queue.empty(); i++) {- out.push(std::move(state->queue.front()));- state->queue.pop();- }-}--HerculesLogger *herculesLogger = nullptr;
+ cbits/hercules-logger.cxx view
@@ -0,0 +1,90 @@+#include "hercules-logger.hh"++HerculesLogger::HerculesLogger()+{++}++void HerculesLogger::push(std::unique_ptr<LogEntry> entry) {+ auto state(state_.lock());+ state->queue.push(std::move(entry));+ wakeup.notify_one();+}++uint64_t HerculesLogger::getMs() {+ auto t = std::chrono::steady_clock::now();+ auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(t - t_zero).count();+ return millis;+}++void HerculesLogger::log(nix::Verbosity lvl, const nix::FormatOrString & fs) {+ push(std::make_unique<LogEntry>(LogEntry {+ .entryType = 1,+ .level = lvl,+ .ms = getMs(),+ .text = fs.s+ }));+}++void HerculesLogger::startActivity(nix::ActivityId act, nix::Verbosity lvl, nix::ActivityType type,+ const std::string & s, const Fields & fields, nix::ActivityId parent) {+ push(std::make_unique<LogEntry>(LogEntry {+ .entryType = 2,+ .level = lvl,+ .ms = getMs(),+ .text = s,+ .activityId = act,+ .type = type,+ .parent = parent,+ .fields = fields+ }));+}++void HerculesLogger::stopActivity(nix::ActivityId act) {+ push(std::make_unique<LogEntry>(LogEntry {+ .entryType = 3,+ .ms = getMs(),+ .activityId = act+ }));+}++void HerculesLogger::result(nix::ActivityId act, nix::ResultType type, const Fields & fields) {+ push(std::make_unique<LogEntry>(LogEntry {+ .entryType = 4,+ .ms = getMs(),+ .activityId = act,+ .type = type,+ .fields = fields+ }));+}++void HerculesLogger::close() {+ auto state(state_.lock());+ state->queue.push(nullptr);+ wakeup.notify_one();+}++std::unique_ptr<HerculesLogger::LogEntry> HerculesLogger::pop () {+ auto state(state_.lock());++ while (state->queue.empty())+ state.wait(wakeup);++ auto r = std::move(state->queue.front());+ state->queue.pop();+ return r;+}++void HerculesLogger::popMany (int max, std::queue<std::unique_ptr<LogEntry>> &out) {+ auto state(state_.lock());++ while (state->queue.empty())+ state.wait(wakeup);++ for (int i = 0; i < max && !state->queue.empty(); i++) {+ out.push(std::move(state->queue.front()));+ state->queue.pop();+ }+}++HerculesLogger *herculesLogger = nullptr;
− cbits/hercules-store.cc
@@ -1,271 +0,0 @@-#include <cstdio>-#include <cstring>-#include <math.h>-#include <nix/config.h>-#include <nix/shared.hh>-#include <nix/store-api.hh>-#include <nix/common-eval-args.hh>-#include <nix/get-drvs.hh>-#include <nix/derivations.hh>-#include <nix/affinity.hh>-#include <nix/globals.hh>--#include "hercules-store.hh"--using namespace nix;--WrappingStore::WrappingStore(const Params& params, ref<Store> storeToWrap)- : Store(params), wrappedStore(storeToWrap) {}--WrappingStore::~WrappingStore() {}--std::string WrappingStore::getUri() {- return "wrapped:" + wrappedStore->getUri();-};--bool WrappingStore::isValidPathUncached(const Path& path) {- return wrappedStore->isValidPath(path); // not ideal-}--PathSet WrappingStore::queryValidPaths(const PathSet& paths,- SubstituteFlag maybeSubstitute) {- return wrappedStore->queryValidPaths(paths, maybeSubstitute);-}-PathSet WrappingStore::queryAllValidPaths() {- return wrappedStore->queryAllValidPaths();-}--// protected:-void WrappingStore::queryPathInfoUncached(- const Path& path,- Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {- unsupported("queryPathInfoUncached");- /*- Callback<ref<ValidPathInfo>>- callback2([&callback](std::future<ref<ValidPathInfo>> vpi){- std::shared_ptr<ValidPathInfo> !@#$%^&*- callback(vpi);- });- return wrappedStore->queryPathInfo(path, callback2); // not ideal- */-}--// public:--void WrappingStore::queryReferrers(const Path& path, PathSet& referrers) {- wrappedStore->queryReferrers(path, referrers);-}--PathSet WrappingStore::queryValidDerivers(const Path& path) {- return wrappedStore->queryValidDerivers(path);-}--PathSet WrappingStore::queryDerivationOutputs(const Path& path) {- return wrappedStore->queryDerivationOutputs(path);-}--StringSet WrappingStore::queryDerivationOutputNames(const Path& path) {- return wrappedStore->queryDerivationOutputNames(path);-}--Path WrappingStore::queryPathFromHashPart(const string& hashPart) {- return wrappedStore->queryPathFromHashPart(hashPart);-}--PathSet WrappingStore::querySubstitutablePaths(const PathSet& paths) {- return wrappedStore->querySubstitutablePaths(paths);-}--void WrappingStore::querySubstitutablePathInfos(const PathSet& paths,- SubstitutablePathInfos& infos) {- wrappedStore->querySubstitutablePathInfos(paths, infos);-}--bool WrappingStore::wantMassQuery() {- return wrappedStore->wantMassQuery();-}--void WrappingStore::addToStore(const ValidPathInfo& info,- Source& narSource,- RepairFlag repair,- CheckSigsFlag checkSigs,- std::shared_ptr<FSAccessor> accessor) {- wrappedStore->addToStore(info, narSource, repair, checkSigs, accessor);-}--void WrappingStore::addToStore(const ValidPathInfo& info,- const ref<std::string>& nar,- RepairFlag repair,- CheckSigsFlag checkSigs,- std::shared_ptr<FSAccessor> accessor) {- wrappedStore->addToStore(info, nar, repair, checkSigs, accessor);-}--Path WrappingStore::addToStore(const string& name,- const Path& srcPath,- bool recursive,- HashType hashAlgo,- PathFilter& filter,- RepairFlag repair) {- return wrappedStore->addToStore(name, srcPath, recursive, hashAlgo, filter,- repair);-}--Path WrappingStore::addTextToStore(const string& name,- const string& s,- const PathSet& references,- RepairFlag repair) {- return wrappedStore->addTextToStore(name, s, references, repair);-}--void WrappingStore::narFromPath(const Path& path, Sink& sink) {- wrappedStore->narFromPath(path, sink);-}--void WrappingStore::buildPaths(const PathSet& paths, BuildMode buildMode) {- wrappedStore->buildPaths(paths, buildMode);-}--BuildResult WrappingStore::buildDerivation(const Path& drvPath,- const BasicDerivation& drv,- BuildMode buildMode) {- return wrappedStore->buildDerivation(drvPath, drv, buildMode);-}--void WrappingStore::ensurePath(const Path& path) {- wrappedStore->ensurePath(path);-}--void WrappingStore::addTempRoot(const Path& path) {- wrappedStore->addTempRoot(path);-}--void WrappingStore::addIndirectRoot(const Path& path) {- wrappedStore->addIndirectRoot(path);-}--void WrappingStore::syncWithGC() {- wrappedStore->syncWithGC();-}--void WrappingStore::collectGarbage(const GCOptions& options,- GCResults& results) {- wrappedStore->collectGarbage(options, results);-}--void WrappingStore::optimiseStore() {- wrappedStore->optimiseStore();-};--bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) {- return wrappedStore->verifyStore(checkContents, repair);-};--ref<FSAccessor> WrappingStore::getFSAccessor() {- return wrappedStore->getFSAccessor();-}--void WrappingStore::addSignatures(const Path& storePath,- const StringSet& sigs) {- wrappedStore->addSignatures(storePath, sigs);-};--void WrappingStore::computeFSClosure(const PathSet& paths,- PathSet& out,- bool flipDirection,- bool includeOutputs,- bool includeDerivers) {- wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs,- includeDerivers);-}--void WrappingStore::queryMissing(const PathSet& targets,- PathSet& willBuild,- PathSet& willSubstitute,- PathSet& unknown,- unsigned long long& downloadSize,- unsigned long long& narSize) {- wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown,- downloadSize, narSize);-}--std::shared_ptr<std::string> WrappingStore::getBuildLog(const Path& path) {- return wrappedStore->getBuildLog(path);-}--void WrappingStore::connect() {- wrappedStore->connect();-};--int WrappingStore::getPriority() {- return wrappedStore->getPriority();-}--Path WrappingStore::toRealPath(const Path& storePath) {- return wrappedStore->toRealPath(storePath);-};--/////--HerculesStore::HerculesStore(const Params& params, ref<Store> storeToWrap)- : WrappingStore(params, storeToWrap) {}--void HerculesStore::ensurePath(const Path& path) {- /* We avoid asking substituters for paths, since- those would yield negative pathInfo caches on remote store.-- Instead, we only assert if path exists in the store.-- Once IFD build is performed, we ask for substitution- via ensurePath.- */- if (!wrappedStore->isValidPath(path)) {- std::exception_ptr exceptionToThrow(nullptr);- builderCallback(strdup(path.c_str()), &exceptionToThrow);- if (exceptionToThrow != nullptr) {- std::rethrow_exception(exceptionToThrow);- }- wrappedStore->ensurePath(path);- }- ensuredPaths.insert(path);-};--// Avoid substituting in evaluator, see `ensurePath` for more details-void HerculesStore::queryMissing(const PathSet& targets,- PathSet& willBuild,- PathSet& willSubstitute,- PathSet& unknown,- unsigned long long& downloadSize,- unsigned long long& narSize) {-};--void HerculesStore::buildPaths(const PathSet& paths, BuildMode buildMode) {- for (Path path : paths) {- std::exception_ptr exceptionToThrow(nullptr);- builderCallback(strdup(path.c_str()), &exceptionToThrow);- if (exceptionToThrow != nullptr) {- std::rethrow_exception(exceptionToThrow);- }- }-}--BuildResult HerculesStore::buildDerivation(const Path& drvPath,- const BasicDerivation& drv,- BuildMode buildMode) {- unsupported("buildDerivation");-- std::cerr << "building derivation " << drvPath << std::endl;- auto r = wrappedStore->buildDerivation(drvPath, drv, buildMode);- std::cerr << "built derivation " << drvPath << std::endl;- return r;-}--void HerculesStore::printDiagnostics() {- for (std::string path : ensuredPaths) {- std::cerr << path << std::endl;- }-}--void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(const char *, std::exception_ptr *exceptionToThrow)) {- builderCallback = newBuilderCallback;-}
+ cbits/hercules-store.cxx view
@@ -0,0 +1,271 @@+#include <cstdio>+#include <cstring>+#include <math.h>+#include <nix/config.h>+#include <nix/shared.hh>+#include <nix/store-api.hh>+#include <nix/common-eval-args.hh>+#include <nix/get-drvs.hh>+#include <nix/derivations.hh>+#include <nix/affinity.hh>+#include <nix/globals.hh>++#include "hercules-store.hh"++using namespace nix;++WrappingStore::WrappingStore(const Params& params, ref<Store> storeToWrap)+ : Store(params), wrappedStore(storeToWrap) {}++WrappingStore::~WrappingStore() {}++std::string WrappingStore::getUri() {+ return "wrapped:" + wrappedStore->getUri();+};++bool WrappingStore::isValidPathUncached(const Path& path) {+ return wrappedStore->isValidPath(path); // not ideal+}++PathSet WrappingStore::queryValidPaths(const PathSet& paths,+ SubstituteFlag maybeSubstitute) {+ return wrappedStore->queryValidPaths(paths, maybeSubstitute);+}+PathSet WrappingStore::queryAllValidPaths() {+ return wrappedStore->queryAllValidPaths();+}++// protected:+void WrappingStore::queryPathInfoUncached(+ const Path& path,+ Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {+ unsupported("queryPathInfoUncached");+ /*+ Callback<ref<ValidPathInfo>>+ callback2([&callback](std::future<ref<ValidPathInfo>> vpi){+ std::shared_ptr<ValidPathInfo> !@#$%^&*+ callback(vpi);+ });+ return wrappedStore->queryPathInfo(path, callback2); // not ideal+ */+}++// public:++void WrappingStore::queryReferrers(const Path& path, PathSet& referrers) {+ wrappedStore->queryReferrers(path, referrers);+}++PathSet WrappingStore::queryValidDerivers(const Path& path) {+ return wrappedStore->queryValidDerivers(path);+}++PathSet WrappingStore::queryDerivationOutputs(const Path& path) {+ return wrappedStore->queryDerivationOutputs(path);+}++StringSet WrappingStore::queryDerivationOutputNames(const Path& path) {+ return wrappedStore->queryDerivationOutputNames(path);+}++Path WrappingStore::queryPathFromHashPart(const string& hashPart) {+ return wrappedStore->queryPathFromHashPart(hashPart);+}++PathSet WrappingStore::querySubstitutablePaths(const PathSet& paths) {+ return wrappedStore->querySubstitutablePaths(paths);+}++void WrappingStore::querySubstitutablePathInfos(const PathSet& paths,+ SubstitutablePathInfos& infos) {+ wrappedStore->querySubstitutablePathInfos(paths, infos);+}++bool WrappingStore::wantMassQuery() {+ return wrappedStore->wantMassQuery();+}++void WrappingStore::addToStore(const ValidPathInfo& info,+ Source& narSource,+ RepairFlag repair,+ CheckSigsFlag checkSigs,+ std::shared_ptr<FSAccessor> accessor) {+ wrappedStore->addToStore(info, narSource, repair, checkSigs, accessor);+}++void WrappingStore::addToStore(const ValidPathInfo& info,+ const ref<std::string>& nar,+ RepairFlag repair,+ CheckSigsFlag checkSigs,+ std::shared_ptr<FSAccessor> accessor) {+ wrappedStore->addToStore(info, nar, repair, checkSigs, accessor);+}++Path WrappingStore::addToStore(const string& name,+ const Path& srcPath,+ bool recursive,+ HashType hashAlgo,+ PathFilter& filter,+ RepairFlag repair) {+ return wrappedStore->addToStore(name, srcPath, recursive, hashAlgo, filter,+ repair);+}++Path WrappingStore::addTextToStore(const string& name,+ const string& s,+ const PathSet& references,+ RepairFlag repair) {+ return wrappedStore->addTextToStore(name, s, references, repair);+}++void WrappingStore::narFromPath(const Path& path, Sink& sink) {+ wrappedStore->narFromPath(path, sink);+}++void WrappingStore::buildPaths(const PathSet& paths, BuildMode buildMode) {+ wrappedStore->buildPaths(paths, buildMode);+}++BuildResult WrappingStore::buildDerivation(const Path& drvPath,+ const BasicDerivation& drv,+ BuildMode buildMode) {+ return wrappedStore->buildDerivation(drvPath, drv, buildMode);+}++void WrappingStore::ensurePath(const Path& path) {+ wrappedStore->ensurePath(path);+}++void WrappingStore::addTempRoot(const Path& path) {+ wrappedStore->addTempRoot(path);+}++void WrappingStore::addIndirectRoot(const Path& path) {+ wrappedStore->addIndirectRoot(path);+}++void WrappingStore::syncWithGC() {+ wrappedStore->syncWithGC();+}++void WrappingStore::collectGarbage(const GCOptions& options,+ GCResults& results) {+ wrappedStore->collectGarbage(options, results);+}++void WrappingStore::optimiseStore() {+ wrappedStore->optimiseStore();+};++bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) {+ return wrappedStore->verifyStore(checkContents, repair);+};++ref<FSAccessor> WrappingStore::getFSAccessor() {+ return wrappedStore->getFSAccessor();+}++void WrappingStore::addSignatures(const Path& storePath,+ const StringSet& sigs) {+ wrappedStore->addSignatures(storePath, sigs);+};++void WrappingStore::computeFSClosure(const PathSet& paths,+ PathSet& out,+ bool flipDirection,+ bool includeOutputs,+ bool includeDerivers) {+ wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs,+ includeDerivers);+}++void WrappingStore::queryMissing(const PathSet& targets,+ PathSet& willBuild,+ PathSet& willSubstitute,+ PathSet& unknown,+ unsigned long long& downloadSize,+ unsigned long long& narSize) {+ wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown,+ downloadSize, narSize);+}++std::shared_ptr<std::string> WrappingStore::getBuildLog(const Path& path) {+ return wrappedStore->getBuildLog(path);+}++void WrappingStore::connect() {+ wrappedStore->connect();+};++int WrappingStore::getPriority() {+ return wrappedStore->getPriority();+}++Path WrappingStore::toRealPath(const Path& storePath) {+ return wrappedStore->toRealPath(storePath);+};++/////++HerculesStore::HerculesStore(const Params& params, ref<Store> storeToWrap)+ : WrappingStore(params, storeToWrap) {}++void HerculesStore::ensurePath(const Path& path) {+ /* We avoid asking substituters for paths, since+ those would yield negative pathInfo caches on remote store.++ Instead, we only assert if path exists in the store.++ Once IFD build is performed, we ask for substitution+ via ensurePath.+ */+ if (!wrappedStore->isValidPath(path)) {+ std::exception_ptr exceptionToThrow(nullptr);+ builderCallback(strdup(path.c_str()), &exceptionToThrow);+ if (exceptionToThrow != nullptr) {+ std::rethrow_exception(exceptionToThrow);+ }+ wrappedStore->ensurePath(path);+ }+ ensuredPaths.insert(path);+};++// Avoid substituting in evaluator, see `ensurePath` for more details+void HerculesStore::queryMissing(const PathSet& targets,+ PathSet& willBuild,+ PathSet& willSubstitute,+ PathSet& unknown,+ unsigned long long& downloadSize,+ unsigned long long& narSize) {+};++void HerculesStore::buildPaths(const PathSet& paths, BuildMode buildMode) {+ for (Path path : paths) {+ std::exception_ptr exceptionToThrow(nullptr);+ builderCallback(strdup(path.c_str()), &exceptionToThrow);+ if (exceptionToThrow != nullptr) {+ std::rethrow_exception(exceptionToThrow);+ }+ }+}++BuildResult HerculesStore::buildDerivation(const Path& drvPath,+ const BasicDerivation& drv,+ BuildMode buildMode) {+ unsupported("buildDerivation");++ std::cerr << "building derivation " << drvPath << std::endl;+ auto r = wrappedStore->buildDerivation(drvPath, drv, buildMode);+ std::cerr << "built derivation " << drvPath << std::endl;+ return r;+}++void HerculesStore::printDiagnostics() {+ for (std::string path : ensuredPaths) {+ std::cerr << path << std::endl;+ }+}++void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(const char *, std::exception_ptr *exceptionToThrow)) {+ builderCallback = newBuilderCallback;+}
hercules-ci-agent-worker/Hercules/Agent/Worker.hs view
@@ -58,7 +58,7 @@ import Katip import qualified Language.C.Inline.Cpp.Exceptions as C import qualified Network.URI-import Protolude hiding (bracket, catch, evalState, wait, withAsync)+import Protolude hiding (bracket, catch, evalState, wait, withAsync, yield) import qualified System.Environment as Environment import System.IO (BufferMode (LineBuffering), hSetBuffering) import System.Posix.IO (dup, fdToHandle, stdError)@@ -162,10 +162,10 @@ ) renderException :: SomeException -> Text-renderException e | Just (C.CppStdException msg) <- fromException e = toSL msg+renderException e | Just (C.CppStdException msg) <- fromException e = toS msg renderException e | Just (C.CppOtherException maybeType) <- fromException e =- "Unexpected C++ exception" <> foldMap (\t -> " of type " <> toSL t) maybeType+ "Unexpected C++ exception" <> foldMap (\t -> " of type " <> toS t) maybeType renderException e | Just (FatalError msg) <- fromException e = msg renderException e = toS $ displayException e @@ -305,7 +305,7 @@ checkVersion = Socket.checkVersion', baseURL = baseURL, path = LogSettings.path l,- token = toSL $ LogSettings.reveal $ LogSettings.token l+ token = encodeUtf8 $ LogSettings.reveal $ LogSettings.token l } -- TODO: test@@ -313,8 +313,8 @@ autoArgArgs kvs = do (k, v) <- M.toList kvs case v of- Eval.LiteralArg s -> ["--argstr", toS k, s]- Eval.ExprArg s -> ["--arg", toS k, s]+ Eval.LiteralArg s -> ["--argstr", encodeUtf8 k, s]+ Eval.ExprArg s -> ["--arg", encodeUtf8 k, s] withDrvInProgress :: MonadUnliftIO m => HerculesState -> Text -> m a -> m a withDrvInProgress HerculesState {drvsInProgress = ref} drvPath =@@ -371,16 +371,17 @@ let store = nixStore hStore s <- storeUri store UnliftIO unlift <- lift askUnliftIO- liftIO $ setBuilderCallback hStore $ \path -> unlift $ katipAddContext (sl "fullpath" (toSL path :: Text)) $ do+ let decode = decodeUtf8With lenientDecode+ liftIO $ setBuilderCallback hStore $ \path -> unlift $ katipAddContext (sl "fullpath" (decode path)) $ do logLocM DebugS "Building" let (plainDrv, bangOut) = BS.span (/= fromIntegral (ord '!')) path outputName = BS.dropWhile (== fromIntegral (ord '!')) bangOut- plainDrvText = toS plainDrv+ plainDrvText = decode plainDrv withDrvInProgress st plainDrvText $ do- liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (toSL outputName) Nothing+ liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (decode outputName) Nothing derivation <- liftIO $ getDerivation store plainDrv outputPath <- liftIO $ getDerivationOutputPath derivation outputName- katipAddContext (sl "outputPath" (toSL outputPath :: Text)) $ do+ katipAddContext (sl "outputPath" (decode outputPath)) $ do logLocM DebugS "Naively calling ensurePath" liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e0 -> do katipAddContext (sl "message" (show (e0 :: SomeException) :: Text)) $@@ -395,7 +396,7 @@ liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e1 -> do katipAddContext (sl "message" (show (e1 :: SomeException) :: Text)) $ logLocM DebugS "Recovering from fresh ensurePath"- liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (toSL outputName) (Just attempt0)+ liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (decode outputName) (Just attempt0) -- TODO sync result' <- liftIO $ atomically $ do@@ -416,7 +417,7 @@ ) logLocM DebugS ("Built") withEvalState store $ \evalState -> do- katipAddContext (sl "storeURI" (toSL s :: Text)) $+ katipAddContext (sl "storeURI" (decode s)) $ logLocM DebugS "EvalState loaded." args <- liftIO $
hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs view
@@ -16,13 +16,13 @@ import qualified Hercules.Agent.WorkerProtocol.Event as Event import qualified Hercules.Agent.WorkerProtocol.Event.BuildResult as Event.BuildResult import Katip-import Protolude+import Protolude hiding (yield) import Unsafe.Coerce runBuild :: (MonadIO m, KatipContext m) => Ptr (Ref NixStore) -> Command.Build.Build -> ConduitT i Event m () runBuild store build = do let extraPaths = Command.Build.inputDerivationOutputPaths build- drvPath = toS $ Command.Build.drvPath build+ drvPath = encodeUtf8 $ Command.Build.drvPath build x <- for extraPaths $ \input -> do liftIO $ try $ CNix.ensurePath store input materialize <- case sequenceA x of
hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Logger.hs view
@@ -18,7 +18,7 @@ import Katip import qualified Language.C.Inline.Cpp as C import qualified Language.C.Inline.Cpp.Exceptions as C-import Protolude hiding (bracket, finally, mask_, onException, tryJust, wait, withAsync)+import Protolude hiding (bracket, finally, mask_, onException, tryJust, wait, withAsync, yield) import System.IO (BufferMode (LineBuffering), hSetBuffering) import System.IO.Error (isEOFError) import System.Posix.IO (closeFd, createPipe, dup, dupTo, fdToHandle, stdError)@@ -38,7 +38,7 @@ C.include "<nix/globals.hh>" -C.include "aliases.h"+C.include "hercules-aliases.h" C.include "hercules-logger.hh" @@ -166,7 +166,7 @@ { i = i_, ms = ms_, level = fromIntegral level_,- msg = toSL text_+ msg = decode text_ } 2 -> do text_ <- unsafePackMallocCString =<< peek textStrPtr@@ -181,7 +181,7 @@ act = LogEntry.ActivityId act_, level = fromIntegral level_, typ = LogEntry.ActivityType typ_,- text = toSL text_,+ text = decode text_, parent = LogEntry.ActivityId parent_, fields = fields_ }@@ -231,9 +231,12 @@ }|] >>= \case 0 -> LogEntry.Int <$> peek uintPtr- 1 -> LogEntry.String . toSL <$> unsafeMallocBS (peek stringPtr)+ 1 -> LogEntry.String . decode <$> unsafeMallocBS (peek stringPtr) _ -> panic "convertAndDeleteFields invalid internal type" +decode :: ByteString -> Text+decode = decodeUtf8With lenientDecode+ close :: IO () close = [C.throwBlock| void {@@ -314,7 +317,7 @@ Left _ -> pass Right "__%%hercules terminate log%%__" -> pass Right ln -> do- katipAddContext (sl "line" (toSL ln :: Text))+ katipAddContext (sl "line" (decode ln)) $ logLocM DebugS $ "Intercepted stderr" liftIO
hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs view
@@ -38,7 +38,7 @@ C.include "<nix/fs-accessor.hh>" -C.include "aliases.h"+C.include "hercules-aliases.h" C.using "namespace nix"
hercules-ci-agent.cabal view
@@ -1,7 +1,7 @@ cabal-version: 2.4 name: hercules-ci-agent-version: 0.7.4+version: 0.7.5 synopsis: Runs Continuous Integration tasks on your machines category: Nix, CI, Testing, DevOps homepage: https://docs.hercules-ci.com@@ -13,7 +13,7 @@ build-type: Simple extra-source-files: CHANGELOG.md- cbits/aliases.h+ cbits/hercules-aliases.h cbits/hercules-store.hh cbits/hercules-logger.hh testdata/vm-test-run-agent-test.drv@@ -22,6 +22,29 @@ type: git location: https://github.com/hercules-ci/hercules-ci-agent +-- match the C++ language standard Nix is using+common cxx-opts+ cxx-options:+ -std=c++17+ -Wall+ extra-libraries: stdc++++ if os(darwin)+ -- avoid https://gitlab.haskell.org/ghc/ghc/issues/11829+ ld-options: -Wl,-keep_dwarf_unwind++ if impl(ghc >= 8.10)+ ghc-options:+ -optcxx-std=c++17+ -optcxx-Wall+ else+ ghc-options:+ -optc-std=c++17+ -optc-Wall+ if os(darwin)+ ghc-options: -pgmc=clang++++ library exposed-modules: Data.Fixed.Extras@@ -68,7 +91,7 @@ , mtl , network-uri , optparse-applicative- , protolude+ , protolude >= 0.3 , process , safe-exceptions , stm@@ -92,6 +115,7 @@ default-language: Haskell2010 library cnix+ import: cxx-opts exposed-modules: CNix CNix.Internal.Context@@ -102,15 +126,15 @@ include-dirs: cbits cxx-sources:- cbits/hercules-store.cc- cbits/hercules-logger.cc+ cbits/hercules-store.cxx+ cbits/hercules-logger.cxx build-depends: base , inline-c , inline-c-cpp , internal-ffi , bytestring- , cachix >= 0.3.8+ , cachix >= 0.5.1 , conduit , containers , protolude@@ -121,7 +145,6 @@ , nix-main >= 2.0 , bdw-gc extra-libraries:- stdc++ boost_context default-language: Haskell2010 default-extensions: DeriveGeneric DeriveTraversable DisambiguateRecordFields FlexibleContexts InstanceSigs LambdaCase MultiParamTypeClasses NoImplicitPrelude OverloadedStrings RankNTypes TupleSections TypeApplications TypeOperators@@ -133,13 +156,6 @@ -fwarn-missing-signatures -fwarn-name-shadowing -fwarn-incomplete-patterns- -- match what Nix is using- -optc-std=c++17- cxx-options: -Wall -std=c++17- if os(darwin)- -- avoid https://gitlab.haskell.org/ghc/ghc/issues/11829- ld-options: -Wl,-keep_dwarf_unwind- ghc-options: -pgmc=clang++ executable hercules-ci-agent main-is: Main.hs@@ -243,6 +259,7 @@ default-language: Haskell2010 executable hercules-ci-agent-worker+ import: cxx-opts main-is: Main.hs other-modules: Hercules.Agent.Worker@@ -266,17 +283,9 @@ -threaded -rtsopts -with-rtsopts=-maxN8- -- match what Nix is using- -optc-std=c++17- cxx-options: -Wall -std=c++17- if os(darwin)- -- avoid https://gitlab.haskell.org/ghc/ghc/issues/11829- ld-options: -Wl,-keep_dwarf_unwind- ghc-options: -pgmc=clang++ include-dirs: cbits extra-libraries:- stdc++ boost_context build-depends: aeson
hercules-ci-agent/Hercules/Agent.hs view
@@ -173,7 +173,7 @@ report' (Left e) = withNamedContext "message" (displayException e) $ withNamedContext "exception" (show e :: Text) do logLocM ErrorS "Exception in task"- report $ TaskStatus.Exceptional $ toSL $ displayException e+ report $ TaskStatus.Exceptional $ toS $ displayException e -- TODO use socket report status = retry (cap 60 exponential)
hercules-ci-agent/Hercules/Agent/Build.hs view
@@ -54,14 +54,14 @@ writeEvent event = case event of Event.BuildResult r -> writeIORef statusRef $ Just r Event.Exception e -> do- logLocM DebugS $ show e+ logLocM DebugS $ logStr (show e :: Text) panic e _ -> pass baseURL <- asks (ServiceInfo.bulkSocketBaseURL . Env.serviceInfo) materialize <- asks (not . Config.nixUserIsTrusted . Env.config) liftIO $ writeChan commandChan $ Just $ Command.Build $ Command.Build.Build { drvPath = BuildTask.derivationPath buildTask,- inputDerivationOutputPaths = toS <$> BuildTask.inputDerivationOutputPaths buildTask,+ inputDerivationOutputPaths = encodeUtf8 <$> BuildTask.inputDerivationOutputPaths buildTask, logSettings = LogSettings.LogSettings { token = LogSettings.Sensitive $ BuildTask.logToken buildTask, path = "/api/v1/logs/build/socket",@@ -70,7 +70,7 @@ materializeDerivation = materialize } exitCode <- runWorker procSpec (stderrLineHandler "Builder") commandChan writeEvent- logLocM DebugS $ "Worker exit: " <> show exitCode+ logLocM DebugS $ "Worker exit: " <> logStr (show exitCode :: Text) case exitCode of ExitSuccess -> pass _ -> panic $ "Worker failed: " <> show exitCode@@ -87,13 +87,13 @@ convertOutputs :: Text -> [BuildResult.OutputInfo] -> Map Text OutputInfo convertOutputs deriver = foldMap $ \oi ->- M.singleton (toS $ BuildResult.name oi) $+ M.singleton (decodeUtf8With lenientDecode $ BuildResult.name oi) $ OutputInfo.OutputInfo { OutputInfo.deriver = deriver,- name = toSL $ BuildResult.name oi,- path = toSL $ BuildResult.path oi,+ name = decodeUtf8With lenientDecode $ BuildResult.name oi,+ path = decodeUtf8With lenientDecode $ BuildResult.path oi, size = fromIntegral $ BuildResult.size oi,- hash = toSL $ BuildResult.hash oi+ hash = decodeUtf8With lenientDecode $ BuildResult.hash oi } push :: BuildTask -> Map Text OutputInfo -> App ()
hercules-ci-agent/Hercules/Agent/Cache.hs view
@@ -33,7 +33,7 @@ T.hPutStrLn netrcHandle (T.unlines netrcLns) hClose netrcHandle Nix.withExtraOptions- [ ("netrc-file", toSL netrcPath),+ [ ("netrc-file", toS netrcPath), ("substituters", T.intercalate " " (substs <> csubsts)), ("trusted-public-keys", T.intercalate " " (pubkeys <> cpubkeys)) ]@@ -66,7 +66,7 @@ CNix.withStore $ \store -> do (Sum total, Sum signed) <- mconcat <$> for keys \key -> liftIO $ do- pathSet <- paths & map toS & newPathSetWith+ pathSet <- paths & newPathSetWith signClosure store key pathSet katipAddContext (sl "num-signatures" signed <> sl "num-paths" total) $ logLocM DebugS "Signed"
hercules-ci-agent/Hercules/Agent/Cachix/Init.hs view
@@ -26,7 +26,7 @@ tlsManagerSettings { managerResponseTimeout = responseTimeoutNone, -- managerModifyRequest :: Request -> IO Request- managerModifyRequest = return . setRequestHeader "User-Agent" [toS cachixVersion]+ managerModifyRequest = return . setRequestHeader "User-Agent" [encodeUtf8 cachixVersion] } newEnv :: Config.FinalConfig -> Map Text CachixCache.CachixCache -> K.KatipContextT IO Env.Env@@ -61,8 +61,10 @@ sk <- head $ CachixCache.signingKeys keys Just $ escalateAs FatalError $ do k' <- Cachix.Secrets.parseSigningKeyLenient sk- pure $ Cachix.Push.PushCache+ pure Cachix.Push.PushCache { pushCacheName = name,- pushCacheSigningKey = k',- pushCacheToken = Servant.Auth.Client.Token $ toSL t+ pushCacheSecret =+ Cachix.Push.PushSigningKey+ (Servant.Auth.Client.Token $ encodeUtf8 t)+ k' }
hercules-ci-agent/Hercules/Agent/Config.hs view
@@ -84,8 +84,9 @@ determineDefaultApiBaseUrl :: IO Text determineDefaultApiBaseUrl = do- maybeEnv <- System.Environment.lookupEnv "HERCULES_API_BASE_URL"- pure $ maybe defaultApiBaseUrl toS maybeEnv+ maybeEnv <- System.Environment.lookupEnv "HERCULES_CI_API_BASE_URL"+ maybeEnv' <- System.Environment.lookupEnv "HERCULES_API_BASE_URL"+ pure $ maybe defaultApiBaseUrl toS (maybeEnv <|> maybeEnv') defaultApiBaseUrl :: Text defaultApiBaseUrl = "https://hercules-ci.com"@@ -95,7 +96,7 @@ readConfig :: ConfigPath -> IO (Config 'Input) readConfig loc = case loc of- TomlPath fp -> Toml.decodeFile tomlCodec (toSL fp)+ TomlPath fp -> Toml.decodeFile tomlCodec (toS fp) finalizeConfig :: ConfigPath -> Config 'Input -> IO (Config 'Final) finalizeConfig loc input = do
hercules-ci-agent/Hercules/Agent/Config/BinaryCaches.hs view
@@ -75,8 +75,8 @@ forM_ (M.toList (unknownKinds bcs)) $ \(k, v) -> logLocM WarningS $ "In file "- <> show fname+ <> logStr fname <> " in entry "- <> show k+ <> logStr k <> ": unknown kind "- <> show (kind v)+ <> logStr (kind v)
hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs view
@@ -12,6 +12,7 @@ _String, key, )+import qualified Data.ByteString.Lazy as LBS import Data.Char (isSpace) import qualified Data.Text as T import qualified Hercules.API.Agent.LifeCycle.AgentInfo as AgentInfo@@ -42,7 +43,7 @@ substituters = nixSubstituters nix, -- TODO: Add cachix substituters concurrentTasks = fromIntegral concurrentTasks }- logLocM DebugS $ "Determined environment info: " <> show s+ logLocM DebugS $ "Determined environment info: " <> logStr (show s :: Text) pure s data NixInfo@@ -61,11 +62,11 @@ version <- Process.readProcess "nix" ["--version"] stdinEmpty rawJson <- Process.readProcess "nix" ["show-config", "--json"] stdinEmpty cfg <-- case Aeson.eitherDecode (toS rawJson) of+ case Aeson.eitherDecode (LBS.fromStrict $ encodeUtf8 $ toS $ rawJson) of Left e -> panic $ "Could not parse nix show-config --json: " <> show e Right r -> pure r pure NixInfo- { nixExeVersion = T.dropAround isSpace (toSL version),+ { nixExeVersion = T.dropAround isSpace (toS version), nixPlatforms = ((cfg :: Aeson.Value) ^.. key "system" . key "value" . _String) <> ( cfg
hercules-ci-agent/Hercules/Agent/Evaluate.hs view
@@ -135,7 +135,7 @@ <> identifier ) let autoArguments = autoArguments'- <&> \sp -> Eval.ExprArg $ toS $ renderSubPath $ toS <$> sp+ <&> \sp -> Eval.ExprArg $ encodeUtf8 $ renderSubPath $ toS <$> sp msgCounter <- liftIO $ newIORef 0 let fixIndex :: MonadIO m =>@@ -253,6 +253,7 @@ buildRequiredIndex <- liftIO $ newIORef (0 :: Int) commandChan <- newChan writeChan commandChan $ Just $ Command.Eval eval+ let decode = decodeUtf8With lenientDecode withProducer (produceWorkerEvents eval nixPath commandChan) $ \workerEventsP -> fix $ \continue -> joinSTM $@@ -261,16 +262,16 @@ ( \case Event.Attribute a -> do emit $ EvaluateEvent.Attribute $ AttributeEvent.AttributeEvent- { AttributeEvent.expressionPath = toSL <$> WorkerAttribute.path a,- AttributeEvent.derivationPath = toSL $ WorkerAttribute.drv a+ { AttributeEvent.expressionPath = decode <$> WorkerAttribute.path a,+ AttributeEvent.derivationPath = decode $ WorkerAttribute.drv a } continue Event.AttributeError e -> do emit $ EvaluateEvent.AttributeError $ AttributeErrorEvent.AttributeErrorEvent- { AttributeErrorEvent.expressionPath = toSL <$> WorkerAttributeError.path e,- AttributeErrorEvent.errorMessage = toSL $ WorkerAttributeError.message e,- AttributeErrorEvent.errorType = toSL <$> WorkerAttributeError.errorType e,- AttributeErrorEvent.errorDerivation = toSL <$> WorkerAttributeError.errorDerivation e+ { AttributeErrorEvent.expressionPath = decode <$> WorkerAttributeError.path e,+ AttributeErrorEvent.errorMessage = WorkerAttributeError.message e,+ AttributeErrorEvent.errorType = WorkerAttributeError.errorType e,+ AttributeErrorEvent.errorDerivation = WorkerAttributeError.errorDerivation e } continue Event.EvaluationDone ->@@ -307,7 +308,7 @@ } flush status <- drvPoller notAttempt drv- logLocM DebugS $ "Got derivation status " <> show status+ logLocM DebugS $ "Got derivation status " <> logStr (show status :: Text) return status writeChan commandChan $ Just $ Command.BuildResult $ uncurry (BuildResult.BuildResult drv) status continue
hercules-ci-agent/Hercules/Agent/Init.hs view
@@ -22,7 +22,7 @@ newEnv config logEnv = do let withLogging :: K.KatipContextT IO a -> IO a withLogging = K.runKatipContextT logEnv () "Init"- withLogging $ K.logLocM K.DebugS $ "Config: " <> show config+ withLogging $ K.logLocM K.DebugS $ "Config: " <> K.logStr (show config :: Text) System.Directory.createDirectoryIfMissing True (Config.workDirectory config) SecureDirectory.init config bcs <- withLogging $ BC.parseFile config
hercules-ci-agent/Hercules/Agent/Log.hs view
@@ -12,18 +12,13 @@ import Data.Aeson import qualified Data.Aeson as A import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as LBS import qualified Data.Map as M import Katip hiding (logLocM) import Katip.Core import Katip.Monadic (logLocM) import Protolude -instance StringConv [Char] LogStr where- strConv l = logStr . (strConv l :: [Char] -> Text)--instance StringConv Text LogStr where- strConv _ = logStr- withNamedContext :: (ToJSON a, KatipContext m) => Text -> a -> m b -> m b withNamedContext name = katipAddContext . Katip.sl name @@ -33,17 +28,17 @@ -- TODO: Support context for all exceptions and use plain @panic@ instead. panicWithLog :: KatipContext m => Text -> m a panicWithLog msg = do- logLocM ErrorS $ toSL msg+ logLocM ErrorS $ logStr msg panic msg stderrLineHandler :: KatipContext m => Text -> Int -> ByteString -> m () stderrLineHandler _processRole _ ln | "@katip " `BS.isPrefixOf` ln,- Just item <- A.decode (toS $ BS.drop 7 ln) =+ Just item <- A.decode (LBS.fromStrict $ BS.drop 7 ln) = -- "This is the lowest level function [...] useful when implementing centralised logging services." Katip.Core.logKatipItem (Katip.Core.SimpleLogPayload . M.toList . fmap (Katip.Core.AnyLogPayload :: A.Value -> Katip.Core.AnyLogPayload) <$> item) stderrLineHandler processRole pid ln = withNamedContext "worker" (pid :: Int) $ logLocM InfoS $ logStr- $ processRole <> ": " <> toSL ln+ $ processRole <> ": " <> decodeUtf8With lenientDecode ln
hercules-ci-agent/Hercules/Agent/Nix.hs view
@@ -36,9 +36,9 @@ App Text readNixProcess cmd opts paths stdinText = do extraOpts <- getExtraOptionArguments- toSL <$> liftIO (readProcess (toSL cmd) (map toSL (opts <> extraOpts <> ["--"] <> paths)) (toSL stdinText))+ toS <$> liftIO (readProcess (toS cmd) (map toS (opts <> extraOpts <> ["--"] <> paths)) (toS stdinText)) nixProc :: Text -> [Text] -> [Text] -> App System.Process.CreateProcess nixProc exe opts args = do extraOpts <- getExtraOptionArguments- pure $ System.Process.proc (toSL exe) (map toSL (opts <> extraOpts <> ["--"] <> args))+ pure $ System.Process.proc (toS exe) (map toS (opts <> extraOpts <> ["--"] <> args))
hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs view
@@ -17,7 +17,7 @@ DerivationInfo.DerivationPathText -> m DerivationInfo retrieveDerivationInfo store drvPath = liftIO $ do- drv <- getDerivation store (toSL drvPath)+ drv <- getDerivation store (encodeUtf8 drvPath) retrieveDerivationInfo' drvPath drv retrieveDerivationInfo' :: Text -> ForeignPtr Derivation -> IO DerivationInfo@@ -28,20 +28,21 @@ env <- getDerivationEnv drv platform <- getDerivationPlatform drv let requiredSystemFeatures = maybe [] splitFeatures $ M.lookup "requiredSystemFeatures" env- splitFeatures = filter (not . T.null) . T.split (== ' ') . toSL+ splitFeatures = filter (not . T.null) . T.split (== ' ') . decode+ decode = decodeUtf8With lenientDecode pure $ DerivationInfo { derivationPath = drvPath,- platform = toSL platform,+ platform = decodeUtf8With lenientDecode platform, requiredSystemFeatures = requiredSystemFeatures,- inputDerivations = inputDrvPaths & map (\(i, os) -> (toSL i, map toSL os)) & M.fromList,- inputSources = map toSL sourcePaths,+ inputDerivations = inputDrvPaths & map (\(i, os) -> (decode i, map decode os)) & M.fromList,+ inputSources = map decode sourcePaths, outputs = outputs & map ( \output ->- ( toSL $ derivationOutputName output,+ ( decode $ derivationOutputName output, DerivationInfo.OutputInfo- { DerivationInfo.path = toSL $ derivationOutputPath output,+ { DerivationInfo.path = decode $ derivationOutputPath output, DerivationInfo.isFixed = derivationOutputHashAlgo output /= "" } )
hercules-ci-agent/Hercules/Agent/Options.hs view
@@ -44,7 +44,7 @@ (parseOptions <**> helper) ( fullDesc <> progDesc "Accepts tasks from Hercules CI and runs them." <> header- ("hercules-ci-agent " <> toSL herculesAgentVersion)+ ("hercules-ci-agent " <> toS herculesAgentVersion) ) parse :: IO Options
hercules-ci-agent/Hercules/Agent/Token.hs view
@@ -39,7 +39,7 @@ readAgentSessionKey :: App (Maybe Text) readAgentSessionKey = do dir <- getDir- logLocM DebugS $ "Data directory: " <> show dir+ logLocM DebugS $ "Data directory: " <> logStr (show dir :: Text) let file = dir </> "session.key" liftIO (System.Directory.doesFileExist file) >>= \case True -> notEmpty <$> readTokenFile file@@ -54,13 +54,13 @@ Just sessKey -> do logLocM DebugS "Found agent session key" let handler e = do- logLocM WarningS $ "Failed to check whether session token matches cluster join token. Using old session. Remove session.key if you need to force a session update. Exception: " <> show e+ logLocM WarningS $ "Failed to check whether session token matches cluster join token. Using old session. Remove session.key if you need to force a session update. Exception: " <> logStr (displayException e) pure sessKey safeLiftedHandle handler $ do cjt <- getClusterJoinTokenId- logLocM DebugS $ "Found clusterJoinTokenId " <> show cjt+ logLocM DebugS $ "Found clusterJoinTokenId " <> logStr cjt scjt <- getSessionClusterJoinTokenId sessKey- logLocM DebugS $ "Found sessionClusterJoinTokenId " <> show scjt+ logLocM DebugS $ "Found sessionClusterJoinTokenId " <> logStr scjt if cjt == scjt then pure sessKey else do@@ -88,7 +88,7 @@ createAgentSession :: App Text createAgentSession = do agentInfo <- EnvironmentInfo.extractAgentInfo- logLocM DebugS $ "Agent info: " <> show agentInfo+ logLocM DebugS $ "Agent info: " <> logStr (show agentInfo :: Text) let createAgentBody = CreateAgentSession.CreateAgentSession {agentInfo = agentInfo} token <- asks Env.currentToken@@ -105,7 +105,7 @@ withAgentToken :: App a -> App a withAgentToken m = do agentSessionToken <- ensureAgentSession- local (\env -> env {Env.currentToken = Token $ toS agentSessionToken}) m+ local (\env -> env {Env.currentToken = Token $ encodeUtf8 agentSessionToken}) m data TokenError = TokenError Text deriving (Typeable, Show)@@ -117,13 +117,13 @@ t <- asks Env.currentToken >>= \case Token jwt -> pure jwt- v <- decodeToken (toSL t)+ v <- decodeToken (BL.fromStrict t) sub <- escalate $ maybeToEither (TokenError "No sub field in cluster join token") (v ^? key "sub" . _String) escalate $ maybeToEither (TokenError "Unrecognized token type") $ T.stripPrefix "t$" sub getSessionClusterJoinTokenId :: Text -> App Text getSessionClusterJoinTokenId t = do- v <- decodeToken (toSL t)+ v <- decodeToken (BL.fromStrict $ encodeUtf8 t) escalate $ maybeToEither (TokenError "No parent field in session token") (v ^? key "parent" . _String) decodeToken :: BL.ByteString -> App Aeson.Value
src-cnix/CNix.hs view
@@ -58,7 +58,7 @@ C.include "<nix/globals.hh>" -C.include "aliases.h"+C.include "hercules-aliases.h" C.include "<gc/gc.h>"
src-cnix/CNix/Internal/Raw.hs view
@@ -17,7 +17,7 @@ C.include "<nix/eval-inline.hh>" -C.include "aliases.h"+C.include "hercules-aliases.h" C.include "<gc/gc.h>"
src-cnix/CNix/Internal/Store.hs view
@@ -41,7 +41,7 @@ C.include "<nix/globals.hh>" -C.include "aliases.h"+C.include "hercules-aliases.h" C.using "namespace nix"
src/Hercules/Agent/Socket.hs view
@@ -131,7 +131,7 @@ recv conn = do withTimeout ackTimeout (FatalError "Hercules.Agent.Socket.recv timed out") $ (liftIO $ A.eitherDecode <$> WS.receiveData conn) >>= \case- Left e -> liftIO $ throwIO (FatalError $ "Error decoding service message: " <> toSL e)+ Left e -> liftIO $ throwIO (FatalError $ "Error decoding service message: " <> toS e) Right r -> pure r handshake conn = katipAddNamespace "Handshake" do siMsg <- recv conn