diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.2] + +### Added + + - Preparations for the next Nix version + +### Fixed + + - #304, `message:epollControl: invalid argument (Bad file descriptor)` in effect task + ## [0.8.1] ### Added diff --git a/cbits/hercules-logger.cxx b/cbits/hercules-logger.cxx --- a/cbits/hercules-logger.cxx +++ b/cbits/hercules-logger.cxx @@ -1,10 +1,5 @@ #include "hercules-logger.hh" -HerculesLogger::HerculesLogger() -{ - -} - void HerculesLogger::push(std::unique_ptr entry) { auto state(state_.lock()); state->queue.push(std::move(entry)); @@ -25,6 +20,15 @@ .text = fs.s })); } + +#ifdef NIX_2_4 +// TODO structured +void HerculesLogger::logEI(const nix::ErrorInfo & ei) { + std::stringstream oss; + showErrorInfo(oss, ei, false); + log(ei.level, oss.str()); +} +#endif void HerculesLogger::startActivity(nix::ActivityId act, nix::Verbosity lvl, nix::ActivityType type, const std::string & s, const Fields & fields, nix::ActivityId parent) { diff --git a/cbits/hercules-logger.hh b/cbits/hercules-logger.hh --- a/cbits/hercules-logger.hh +++ b/cbits/hercules-logger.hh @@ -1,6 +1,11 @@ #pragma once #include + +#ifdef NIX_2_4 +#include +#endif + #include #include #include @@ -27,7 +32,7 @@ uint64_t getMs(); public: - HerculesLogger(); + inline HerculesLogger() {}; struct LogEntry { int entryType; @@ -45,6 +50,10 @@ const std::string & s, const Fields & fields, nix::ActivityId parent) override; void stopActivity(nix::ActivityId act) override; void result(nix::ActivityId act, nix::ResultType type, const Fields & fields) override; + +#ifdef NIX_2_4 + void logEI(const nix::ErrorInfo &ei) override; +#endif std::unique_ptr pop(); void popMany(int max, std::queue> &out); diff --git a/cbits/hercules-store.cxx b/cbits/hercules-store.cxx deleted file mode 100644 --- a/cbits/hercules-store.cxx +++ /dev/null @@ -1,271 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hercules-store.hh" - -using namespace nix; - -WrappingStore::WrappingStore(const Params& params, ref storeToWrap) - : Store(params), wrappedStore(storeToWrap) {} - -WrappingStore::~WrappingStore() {} - -std::string WrappingStore::getUri() { - return "wrapped:" + wrappedStore->getUri(); -}; - -bool WrappingStore::isValidPathUncached(const Path& path) { - return wrappedStore->isValidPath(path); // not ideal -} - -PathSet WrappingStore::queryValidPaths(const PathSet& paths, - SubstituteFlag maybeSubstitute) { - return wrappedStore->queryValidPaths(paths, maybeSubstitute); -} -PathSet WrappingStore::queryAllValidPaths() { - return wrappedStore->queryAllValidPaths(); -} - -// protected: -void WrappingStore::queryPathInfoUncached( - const Path& path, - Callback> callback) noexcept { - unsupported("queryPathInfoUncached"); - /* - Callback> - callback2([&callback](std::future> vpi){ - std::shared_ptr !@#$%^&* - callback(vpi); - }); - return wrappedStore->queryPathInfo(path, callback2); // not ideal - */ -} - -// public: - -void WrappingStore::queryReferrers(const Path& path, PathSet& referrers) { - wrappedStore->queryReferrers(path, referrers); -} - -PathSet WrappingStore::queryValidDerivers(const Path& path) { - return wrappedStore->queryValidDerivers(path); -} - -PathSet WrappingStore::queryDerivationOutputs(const Path& path) { - return wrappedStore->queryDerivationOutputs(path); -} - -StringSet WrappingStore::queryDerivationOutputNames(const Path& path) { - return wrappedStore->queryDerivationOutputNames(path); -} - -Path WrappingStore::queryPathFromHashPart(const string& hashPart) { - return wrappedStore->queryPathFromHashPart(hashPart); -} - -PathSet WrappingStore::querySubstitutablePaths(const PathSet& paths) { - return wrappedStore->querySubstitutablePaths(paths); -} - -void WrappingStore::querySubstitutablePathInfos(const PathSet& paths, - SubstitutablePathInfos& infos) { - wrappedStore->querySubstitutablePathInfos(paths, infos); -} - -bool WrappingStore::wantMassQuery() { - return wrappedStore->wantMassQuery(); -} - -void WrappingStore::addToStore(const ValidPathInfo& info, - Source& narSource, - RepairFlag repair, - CheckSigsFlag checkSigs, - std::shared_ptr accessor) { - wrappedStore->addToStore(info, narSource, repair, checkSigs, accessor); -} - -void WrappingStore::addToStore(const ValidPathInfo& info, - const ref& nar, - RepairFlag repair, - CheckSigsFlag checkSigs, - std::shared_ptr accessor) { - wrappedStore->addToStore(info, nar, repair, checkSigs, accessor); -} - -Path WrappingStore::addToStore(const string& name, - const Path& srcPath, - bool recursive, - HashType hashAlgo, - PathFilter& filter, - RepairFlag repair) { - return wrappedStore->addToStore(name, srcPath, recursive, hashAlgo, filter, - repair); -} - -Path WrappingStore::addTextToStore(const string& name, - const string& s, - const PathSet& references, - RepairFlag repair) { - return wrappedStore->addTextToStore(name, s, references, repair); -} - -void WrappingStore::narFromPath(const Path& path, Sink& sink) { - wrappedStore->narFromPath(path, sink); -} - -void WrappingStore::buildPaths(const PathSet& paths, BuildMode buildMode) { - wrappedStore->buildPaths(paths, buildMode); -} - -BuildResult WrappingStore::buildDerivation(const Path& drvPath, - const BasicDerivation& drv, - BuildMode buildMode) { - return wrappedStore->buildDerivation(drvPath, drv, buildMode); -} - -void WrappingStore::ensurePath(const Path& path) { - wrappedStore->ensurePath(path); -} - -void WrappingStore::addTempRoot(const Path& path) { - wrappedStore->addTempRoot(path); -} - -void WrappingStore::addIndirectRoot(const Path& path) { - wrappedStore->addIndirectRoot(path); -} - -void WrappingStore::syncWithGC() { - wrappedStore->syncWithGC(); -} - -void WrappingStore::collectGarbage(const GCOptions& options, - GCResults& results) { - wrappedStore->collectGarbage(options, results); -} - -void WrappingStore::optimiseStore() { - wrappedStore->optimiseStore(); -}; - -bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) { - return wrappedStore->verifyStore(checkContents, repair); -}; - -ref WrappingStore::getFSAccessor() { - return wrappedStore->getFSAccessor(); -} - -void WrappingStore::addSignatures(const Path& storePath, - const StringSet& sigs) { - wrappedStore->addSignatures(storePath, sigs); -}; - -void WrappingStore::computeFSClosure(const PathSet& paths, - PathSet& out, - bool flipDirection, - bool includeOutputs, - bool includeDerivers) { - wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs, - includeDerivers); -} - -void WrappingStore::queryMissing(const PathSet& targets, - PathSet& willBuild, - PathSet& willSubstitute, - PathSet& unknown, - unsigned long long& downloadSize, - unsigned long long& narSize) { - wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown, - downloadSize, narSize); -} - -std::shared_ptr WrappingStore::getBuildLog(const Path& path) { - return wrappedStore->getBuildLog(path); -} - -void WrappingStore::connect() { - wrappedStore->connect(); -}; - -int WrappingStore::getPriority() { - return wrappedStore->getPriority(); -} - -Path WrappingStore::toRealPath(const Path& storePath) { - return wrappedStore->toRealPath(storePath); -}; - -///// - -HerculesStore::HerculesStore(const Params& params, ref storeToWrap) - : WrappingStore(params, storeToWrap) {} - -void HerculesStore::ensurePath(const Path& path) { - /* We avoid asking substituters for paths, since - those would yield negative pathInfo caches on remote store. - - Instead, we only assert if path exists in the store. - - Once IFD build is performed, we ask for substitution - via ensurePath. - */ - if (!wrappedStore->isValidPath(path)) { - std::exception_ptr exceptionToThrow(nullptr); - builderCallback(strdup(path.c_str()), &exceptionToThrow); - if (exceptionToThrow != nullptr) { - std::rethrow_exception(exceptionToThrow); - } - wrappedStore->ensurePath(path); - } - ensuredPaths.insert(path); -}; - -// Avoid substituting in evaluator, see `ensurePath` for more details -void HerculesStore::queryMissing(const PathSet& targets, - PathSet& willBuild, - PathSet& willSubstitute, - PathSet& unknown, - unsigned long long& downloadSize, - unsigned long long& narSize) { -}; - -void HerculesStore::buildPaths(const PathSet& paths, BuildMode buildMode) { - for (Path path : paths) { - std::exception_ptr exceptionToThrow(nullptr); - builderCallback(strdup(path.c_str()), &exceptionToThrow); - if (exceptionToThrow != nullptr) { - std::rethrow_exception(exceptionToThrow); - } - } -} - -BuildResult HerculesStore::buildDerivation(const Path& drvPath, - const BasicDerivation& drv, - BuildMode buildMode) { - unsupported("buildDerivation"); - - std::cerr << "building derivation " << drvPath << std::endl; - auto r = wrappedStore->buildDerivation(drvPath, drv, buildMode); - std::cerr << "built derivation " << drvPath << std::endl; - return r; -} - -void HerculesStore::printDiagnostics() { - for (std::string path : ensuredPaths) { - std::cerr << path << std::endl; - } -} - -void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(const char *, std::exception_ptr *exceptionToThrow)) { - builderCallback = newBuilderCallback; -} diff --git a/cbits/hercules-store.hh b/cbits/hercules-store.hh deleted file mode 100644 --- a/cbits/hercules-store.hh +++ /dev/null @@ -1,150 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "HsFFI.h" - -using namespace nix; - -class WrappingStore : public Store { - public: - ref wrappedStore; - - WrappingStore(const Params & params, ref storeToWrap); - - - virtual ~WrappingStore(); - - virtual std::string getUri() override; - -protected: - - virtual bool isValidPathUncached(const Path & path) override; - -public: - - virtual PathSet queryValidPaths(const PathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override; - virtual PathSet queryAllValidPaths() override; - -protected: - virtual void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override; - -public: - - virtual void queryReferrers(const Path & path, - PathSet & referrers) override; - - virtual PathSet queryValidDerivers(const Path & path) override; - - virtual PathSet queryDerivationOutputs(const Path & path) override; - - virtual StringSet queryDerivationOutputNames(const Path & path) override; - - virtual Path queryPathFromHashPart(const string & hashPart) override; - - virtual PathSet querySubstitutablePaths(const PathSet & paths) override; - - virtual void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) override; - - virtual bool wantMassQuery() override; - - virtual void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr accessor = 0) override; - - virtual void addToStore(const ValidPathInfo & info, const ref & nar, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr accessor = 0) override; - - virtual Path addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; - - virtual Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair = NoRepair) override; - - virtual void narFromPath(const Path & path, Sink & sink) override; - - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; - - virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override; - - virtual void ensurePath(const Path & path) override; - - virtual void addTempRoot(const Path & path) override; - - virtual void addIndirectRoot(const Path & path) override; - - virtual void syncWithGC() override; - - virtual void collectGarbage(const GCOptions & options, GCResults & results) override; - - virtual void optimiseStore() override; - - virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) override; - - virtual ref getFSAccessor() override; - - virtual void addSignatures(const Path & storePath, const StringSet & sigs) override; - - - virtual void computeFSClosure(const PathSet & paths, - PathSet & out, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false) override; - - virtual void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override; - - virtual std::shared_ptr getBuildLog(const Path & path) override; - - virtual void connect() override; - - virtual int getPriority() override; - - virtual Path toRealPath(const Path & storePath) override; -}; - -class HerculesStore : public WrappingStore { -public: - PathSet ensuredPaths; - void (* builderCallback)(const char *, std::exception_ptr *exceptionToThrow); - - HerculesStore(const Params & params, ref storeToWrap); - - // Overrides - - virtual void ensurePath(const Path & path) override; - - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; - - virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override; - - virtual void queryMissing(const PathSet& targets, - PathSet& willBuild, - PathSet& willSubstitute, - PathSet& unknown, - unsigned long long& downloadSize, - unsigned long long& narSize) override; - - // Additions - - void printDiagnostics(); - - void setBuilderCallback(void (* newBuilderCallback)(const char *, std::exception_ptr *exceptionToThrow)); - - void inhibitBuilds(); - void uninhibitBuilds(); -}; diff --git a/cbits/nix-2.3/hercules-store.cxx b/cbits/nix-2.3/hercules-store.cxx new file mode 100644 --- /dev/null +++ b/cbits/nix-2.3/hercules-store.cxx @@ -0,0 +1,277 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hercules-store.hh" + +using namespace nix; + +WrappingStore::WrappingStore(const Params& params, ref storeToWrap) + : Store(params), wrappedStore(storeToWrap) {} + +WrappingStore::~WrappingStore() {} + +std::string WrappingStore::getUri() { + return "wrapped:" + wrappedStore->getUri(); +}; + +bool WrappingStore::isValidPathUncached(const Path& path) { + return wrappedStore->isValidPath(path); // not ideal +} + +PathSet WrappingStore::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + return wrappedStore->queryValidPaths(paths, maybeSubstitute); +} +PathSet WrappingStore::queryAllValidPaths() { + return wrappedStore->queryAllValidPaths(); +} + +// protected: +void WrappingStore::queryPathInfoUncached( + const Path& path, + Callback> callback) noexcept { + unsupported("queryPathInfoUncached"); + /* + Callback> + callback2([&callback](std::future> vpi){ + std::shared_ptr !@#$%^&* + callback(vpi); + }); + return wrappedStore->queryPathInfo(path, callback2); // not ideal + */ +} + +// public: + +void WrappingStore::queryReferrers(const Path& path, PathSet& referrers) { + wrappedStore->queryReferrers(path, referrers); +} + +PathSet WrappingStore::queryValidDerivers(const Path& path) { + return wrappedStore->queryValidDerivers(path); +} + +PathSet WrappingStore::queryDerivationOutputs(const Path& path) { + return wrappedStore->queryDerivationOutputs(path); +} + +StringSet WrappingStore::queryDerivationOutputNames(const Path& path) { + return wrappedStore->queryDerivationOutputNames(path); +} + +Path WrappingStore::queryPathFromHashPart(const string& hashPart) { + return wrappedStore->queryPathFromHashPart(hashPart); +} + +PathSet WrappingStore::querySubstitutablePaths(const PathSet& paths) { + return wrappedStore->querySubstitutablePaths(paths); +} + +void WrappingStore::querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + wrappedStore->querySubstitutablePathInfos(paths, infos); +} + +bool WrappingStore::wantMassQuery() { + return wrappedStore->wantMassQuery(); +} + +void WrappingStore::addToStore(const ValidPathInfo& info, + Source& narSource, + RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr accessor) { + wrappedStore->addToStore(info, narSource, repair, checkSigs, accessor); +} + +void WrappingStore::addToStore(const ValidPathInfo& info, + const ref& nar, + RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr accessor) { + wrappedStore->addToStore(info, nar, repair, checkSigs, accessor); +} + +Path WrappingStore::addToStore(const string& name, + const Path& srcPath, + bool recursive, + HashType hashAlgo, + PathFilter& filter, + RepairFlag repair) { + return wrappedStore->addToStore(name, srcPath, recursive, hashAlgo, filter, + repair); +} + +Path WrappingStore::addTextToStore(const string& name, + const string& s, + const PathSet& references, + RepairFlag repair) { + return wrappedStore->addTextToStore(name, s, references, repair); +} + +void WrappingStore::narFromPath(const Path& path, Sink& sink) { + wrappedStore->narFromPath(path, sink); +} + +void WrappingStore::buildPaths(const PathSet& paths, BuildMode buildMode) { + wrappedStore->buildPaths(paths, buildMode); +} + +BuildResult WrappingStore::buildDerivation(const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + return wrappedStore->buildDerivation(drvPath, drv, buildMode); +} + +void WrappingStore::ensurePath(const Path& path) { + wrappedStore->ensurePath(path); +} + +void WrappingStore::addTempRoot(const Path& path) { + wrappedStore->addTempRoot(path); +} + +void WrappingStore::addIndirectRoot(const Path& path) { + wrappedStore->addIndirectRoot(path); +} + +void WrappingStore::syncWithGC() { + wrappedStore->syncWithGC(); +} + +void WrappingStore::collectGarbage(const GCOptions& options, + GCResults& results) { + wrappedStore->collectGarbage(options, results); +} + +void WrappingStore::optimiseStore() { + wrappedStore->optimiseStore(); +}; + +bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) { + return wrappedStore->verifyStore(checkContents, repair); +}; + +ref WrappingStore::getFSAccessor() { + return wrappedStore->getFSAccessor(); +} + +void WrappingStore::addSignatures(const Path& storePath, + const StringSet& sigs) { + wrappedStore->addSignatures(storePath, sigs); +}; + +void WrappingStore::computeFSClosure(const PathSet& paths, + PathSet& out, + bool flipDirection, + bool includeOutputs, + bool includeDerivers) { + wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs, + includeDerivers); +} + +void WrappingStore::queryMissing(const PathSet& targets, + PathSet& willBuild, + PathSet& willSubstitute, + PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) { + wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown, + downloadSize, narSize); +} + +std::shared_ptr WrappingStore::getBuildLog(const Path& path) { + return wrappedStore->getBuildLog(path); +} + +void WrappingStore::connect() { + wrappedStore->connect(); +}; + +int WrappingStore::getPriority() { + return wrappedStore->getPriority(); +} + +Path WrappingStore::toRealPath(const Path& storePath) { + return wrappedStore->toRealPath(storePath); +}; + +///// + +HerculesStore::HerculesStore(const Params& params, ref storeToWrap) + : WrappingStore(params, storeToWrap) {} + +void HerculesStore::ensurePath(const Path& path) { + /* We avoid asking substituters for paths, since + those would yield negative pathInfo caches on remote store. + + Instead, we only assert if path exists in the store. + + Once IFD build is performed, we ask for substitution + via ensurePath. + */ + if (!wrappedStore->isValidPath(path)) { + std::exception_ptr exceptionToThrow(nullptr); + auto pathWO = nix::parsePathWithOutputs(*this, path); + auto pathsPtr = new std::vector{pathWO}; + // TODO place outside the loop + builderCallback(pathsPtr, &exceptionToThrow); + if (exceptionToThrow != nullptr) { + std::rethrow_exception(exceptionToThrow); + } + wrappedStore->ensurePath(path); + } + ensuredPaths.insert(path); +}; + +// Avoid substituting in evaluator, see `ensurePath` for more details +void HerculesStore::queryMissing(const PathSet& targets, + PathSet& willBuild, + PathSet& willSubstitute, + PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) { +}; + +void HerculesStore::buildPaths(const PathSet& paths, BuildMode buildMode) { + for (Path path : paths) { + std::exception_ptr exceptionToThrow(nullptr); + auto pathWO = nix::parsePathWithOutputs(*this, path); + auto pathsPtr = new std::vector{pathWO}; + // TODO place outside the loop + builderCallback(pathsPtr, &exceptionToThrow); + if (exceptionToThrow != nullptr) { + std::rethrow_exception(exceptionToThrow); + } + } +} + +BuildResult HerculesStore::buildDerivation(const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + unsupported("buildDerivation"); + + std::cerr << "building derivation " << drvPath << std::endl; + auto r = wrappedStore->buildDerivation(drvPath, drv, buildMode); + std::cerr << "built derivation " << drvPath << std::endl; + return r; +} + +void HerculesStore::printDiagnostics() { + for (std::string path : ensuredPaths) { + std::cerr << path << std::endl; + } +} + +void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(std::vector*, std::exception_ptr *exceptionToThrow)) { + builderCallback = newBuilderCallback; +} diff --git a/cbits/nix-2.3/hercules-store.hh b/cbits/nix-2.3/hercules-store.hh new file mode 100644 --- /dev/null +++ b/cbits/nix-2.3/hercules-store.hh @@ -0,0 +1,151 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "HsFFI.h" + +using namespace nix; + +class WrappingStore : public Store { + public: + ref wrappedStore; + + WrappingStore(const Params & params, ref storeToWrap); + + + virtual ~WrappingStore(); + + virtual std::string getUri() override; + +protected: + + virtual bool isValidPathUncached(const Path & path) override; + +public: + + virtual PathSet queryValidPaths(const PathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + virtual PathSet queryAllValidPaths() override; + +protected: + virtual void queryPathInfoUncached(const Path & path, + Callback> callback) noexcept override; + +public: + + virtual void queryReferrers(const Path & path, + PathSet & referrers) override; + + virtual PathSet queryValidDerivers(const Path & path) override; + + virtual PathSet queryDerivationOutputs(const Path & path) override; + + virtual StringSet queryDerivationOutputNames(const Path & path) override; + + virtual Path queryPathFromHashPart(const string & hashPart) override; + + virtual PathSet querySubstitutablePaths(const PathSet & paths) override; + + virtual void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) override; + + virtual bool wantMassQuery() override; + + virtual void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr accessor = 0) override; + + virtual void addToStore(const ValidPathInfo & info, const ref & nar, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr accessor = 0) override; + + virtual Path addToStore(const string & name, const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; + + virtual Path addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair = NoRepair) override; + + virtual void narFromPath(const Path & path, Sink & sink) override; + + virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; + + virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override; + + virtual void ensurePath(const Path & path) override; + + virtual void addTempRoot(const Path & path) override; + + virtual void addIndirectRoot(const Path & path) override; + + virtual void syncWithGC() override; + + virtual void collectGarbage(const GCOptions & options, GCResults & results) override; + + virtual void optimiseStore() override; + + virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) override; + + virtual ref getFSAccessor() override; + + virtual void addSignatures(const Path & storePath, const StringSet & sigs) override; + + + virtual void computeFSClosure(const PathSet & paths, + PathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override; + + virtual void queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) override; + + virtual std::shared_ptr getBuildLog(const Path & path) override; + + virtual void connect() override; + + virtual int getPriority() override; + + virtual Path toRealPath(const Path & storePath) override; +}; + +class HerculesStore : public WrappingStore { +public: + PathSet ensuredPaths; + void (* builderCallback)(std::vector*, std::exception_ptr *exceptionToThrow); + + HerculesStore(const Params & params, ref storeToWrap); + + // Overrides + + virtual void ensurePath(const Path & path) override; + + virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; + + virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override; + + virtual void queryMissing(const PathSet& targets, + PathSet& willBuild, + PathSet& willSubstitute, + PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) override; + + // Additions + + void printDiagnostics(); + + void setBuilderCallback(void (* newBuilderCallback)(std::vector*, std::exception_ptr *exceptionToThrow)); + + void inhibitBuilds(); + void uninhibitBuilds(); +}; diff --git a/cbits/nix-2.4/hercules-store.cxx b/cbits/nix-2.4/hercules-store.cxx new file mode 100644 --- /dev/null +++ b/cbits/nix-2.4/hercules-store.cxx @@ -0,0 +1,276 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hercules-store.hh" + +using namespace nix; + +WrappingStore::WrappingStore(const Params& params, ref storeToWrap) + : Store(params), wrappedStore(storeToWrap) {} + +WrappingStore::~WrappingStore() {} + +std::string WrappingStore::getUri() { + return "wrapped:" + wrappedStore->getUri(); +}; + +bool WrappingStore::isValidPathUncached(const StorePath & path) { + return wrappedStore->isValidPath(path); // caches again. Not much we can do. +} + +StorePathSet WrappingStore::queryValidPaths(const StorePathSet& paths, + SubstituteFlag maybeSubstitute) { + return wrappedStore->queryValidPaths(paths, maybeSubstitute); +} +StorePathSet WrappingStore::queryAllValidPaths() { + return wrappedStore->queryAllValidPaths(); +} + +// protected: +void WrappingStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept { + + auto callbackPtr = std::make_shared(std::move(callback)); + + wrappedStore->queryPathInfo(path, {[=](std::future> vpi){ + (*callbackPtr)(vpi.get().get_ptr()); + }}); +} + +// public: + +void WrappingStore::queryReferrers(const StorePath& path, StorePathSet& referrers) { + wrappedStore->queryReferrers(path, referrers); +} + +StorePathSet WrappingStore::queryValidDerivers(const StorePath& path) { + return wrappedStore->queryValidDerivers(path); +} + +StorePathSet WrappingStore::queryDerivationOutputs(const StorePath& path) { + return wrappedStore->queryDerivationOutputs(path); +} + +std::optional WrappingStore::queryPathFromHashPart(const std::string & hashPart) { + return wrappedStore->queryPathFromHashPart(hashPart); +} + +StorePathSet WrappingStore::querySubstitutablePaths(const StorePathSet& paths) { + return wrappedStore->querySubstitutablePaths(paths); +} + +void WrappingStore::querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) { + wrappedStore->querySubstitutablePathInfos(paths, infos); +} + +void WrappingStore::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs) { + wrappedStore->addToStore(info, narSource, repair, checkSigs); +} + +StorePath WrappingStore::addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) { + return wrappedStore->addToStore(name, srcPath, method, hashAlgo, filter, repair); +} + +StorePath WrappingStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { + return wrappedStore->addToStoreFromDump(dump, name, method, hashAlgo, repair); +} + +StorePath WrappingStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { + return wrappedStore->addTextToStore(name, s, references, repair); +} + +void WrappingStore::narFromPath(const StorePath& path, Sink& sink) { + wrappedStore->narFromPath(path, sink); +} + +void WrappingStore::buildPaths( + const std::vector & paths, BuildMode buildMode) { + wrappedStore->buildPaths(paths, buildMode); +} + +BuildResult WrappingStore::buildDerivation(const StorePath& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + return wrappedStore->buildDerivation(drvPath, drv, buildMode); +} + +void WrappingStore::ensurePath(const StorePath& path) { + wrappedStore->ensurePath(path); +} + +void WrappingStore::addTempRoot(const StorePath& path) { + wrappedStore->addTempRoot(path); +} + +void WrappingStore::addIndirectRoot(const Path& path) { + wrappedStore->addIndirectRoot(path); +} + +void WrappingStore::syncWithGC() { + wrappedStore->syncWithGC(); +} + +void WrappingStore::collectGarbage(const GCOptions& options, + GCResults& results) { + wrappedStore->collectGarbage(options, results); +} + +void WrappingStore::optimiseStore() { + wrappedStore->optimiseStore(); +}; + +bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) { + return wrappedStore->verifyStore(checkContents, repair); +}; + +ref WrappingStore::getFSAccessor() { + return wrappedStore->getFSAccessor(); +} + +void WrappingStore::addSignatures(const StorePath& storePath, + const StringSet& sigs) { + wrappedStore->addSignatures(storePath, sigs); +}; + +void WrappingStore::computeFSClosure(const StorePathSet& paths, + StorePathSet& out, + bool flipDirection, + bool includeOutputs, + bool includeDerivers) { + wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs, + includeDerivers); +} + +void WrappingStore::queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) { + wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown, + downloadSize, narSize); +} + +std::shared_ptr WrappingStore::getBuildLog(const StorePath& path) { + return wrappedStore->getBuildLog(path); +} + +void WrappingStore::connect() { + wrappedStore->connect(); +}; + +Path WrappingStore::toRealPath(const Path& storePath) { + return wrappedStore->toRealPath(storePath); +}; + +Roots WrappingStore::findRoots(bool censor) { + return wrappedStore->findRoots(censor); +} + +unsigned int WrappingStore::getProtocol() { + return wrappedStore->getProtocol(); +} + +void WrappingStore::createUser(const std::string & userName, uid_t userId) { + wrappedStore->createUser(userName, userId); +} + +///// + +HerculesStore::HerculesStore(const Params& params, ref storeToWrap) + : StoreConfig(params) + , WrappingStore(params, storeToWrap) {} + +const std::string HerculesStore::name() { + return "wrapped " + wrappedStore->name(); +} + +std::optional HerculesStore::queryRealisation(const DrvOutput &drvOut) { + // TODO (ca-derivations) use? + return wrappedStore->queryRealisation(drvOut); +} + +void HerculesStore::ensurePath(const StorePath& path) { + /* We avoid asking substituters for paths, since + those would yield negative pathInfo caches on remote store. + + Instead, we only assert if path exists in the store. + + Once IFD build is performed, we ask for substitution + via ensurePath. + */ + if (!wrappedStore->isValidPath(path)) { + std::exception_ptr exceptionToThrow(nullptr); + // FIXME probably need this + // builderCallback(path, &exceptionToThrow); + if (exceptionToThrow != nullptr) { + std::rethrow_exception(exceptionToThrow); + } + wrappedStore->ensurePath(path); + } + ensuredPaths.insert(path); +}; + +// Avoid substituting in evaluator, see `ensurePath` for more details +void HerculesStore::queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) { +}; + +void HerculesStore::buildPaths(const std::vector & derivedPaths, BuildMode buildMode) { + std::exception_ptr exceptionToThrow(nullptr); + + // responsibility for delete is transferred to builderCallback + auto pathsPtr = new std::vector(); + std::vector &paths = *pathsPtr; + + // TODO: don't ignore the Opaques + for (auto & derivedPath : derivedPaths) { + std::visit(overloaded { + [&](DerivedPath::Built built) { + paths.push_back(StorePathWithOutputs { built.drvPath, built.outputs }); + }, + [&](DerivedPath::Opaque opaque) { + // TODO: pass this to the callback as well + }, + }, derivedPath.raw()); + } + + builderCallback(pathsPtr, &exceptionToThrow); + + if (exceptionToThrow != nullptr) { + std::rethrow_exception(exceptionToThrow); + } +} + +BuildResult HerculesStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) { + + unsupported("buildDerivation"); + // unreachable + return wrappedStore->buildDerivation(drvPath, drv, buildMode); +} + +void HerculesStore::printDiagnostics() { + for (auto path : ensuredPaths) { + std::cerr << path.to_string() << std::endl; + } +} + +void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(std::vector*, std::exception_ptr *exceptionToThrow)) { + builderCallback = newBuilderCallback; +} diff --git a/cbits/nix-2.4/hercules-store.hh b/cbits/nix-2.4/hercules-store.hh new file mode 100644 --- /dev/null +++ b/cbits/nix-2.4/hercules-store.hh @@ -0,0 +1,154 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "HsFFI.h" + +using namespace nix; + +class WrappingStore : public Store { + public: + ref wrappedStore; + + WrappingStore(const Params & params, ref storeToWrap); + + + virtual ~WrappingStore(); + + virtual std::string getUri() override; + +protected: + + virtual bool isValidPathUncached(const StorePath & path) override; + +public: + + virtual StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + virtual StorePathSet queryAllValidPaths() override; + +protected: + virtual void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + +public: + + virtual void queryReferrers(const StorePath & path, + StorePathSet & referrers) override; + + virtual StorePathSet queryValidDerivers(const StorePath & path) override; + + virtual StorePathSet queryDerivationOutputs(const StorePath & path) override; + + virtual std::optional queryPathFromHashPart(const std::string & hashPart) override; + + virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; + + virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) override; + + virtual void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override; + + virtual StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; + + virtual StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + + virtual StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) override; + + virtual void narFromPath(const StorePath & path, Sink & sink) override; + + virtual void buildPaths( + const std::vector & paths, + BuildMode buildMode = bmNormal) override; + + virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override; + + virtual void ensurePath(const StorePath & path) override; + + virtual void addTempRoot(const StorePath & path) override; + + virtual void addIndirectRoot(const Path & path) override; + + virtual void syncWithGC() override; + + virtual Roots findRoots(bool censor) override; + + virtual void collectGarbage(const GCOptions & options, GCResults & results) override; + + virtual void optimiseStore() override; + + virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) override; + + virtual ref getFSAccessor() override; + + virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) override; + + virtual void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override; + + virtual void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) override; + + virtual std::shared_ptr getBuildLog(const StorePath & path) override; + + virtual unsigned int getProtocol() override; + + virtual void connect() override; + + virtual Path toRealPath(const Path & storePath) override; + + virtual void createUser(const std::string & userName, uid_t userId) override; + +}; + +class HerculesStore : public WrappingStore { +public: + StorePathSet ensuredPaths; + void (* builderCallback)(std::vector*, std::exception_ptr *exceptionToThrow); + + HerculesStore(const Params & params, ref storeToWrap); + + virtual const std::string name() override; + + // Overrides + + virtual std::optional queryRealisation(const DrvOutput &) override; + + virtual void ensurePath(const StorePath & path) override; + + virtual void buildPaths( + const std::vector & paths, + BuildMode buildMode = bmNormal) override; + + virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override; + + virtual void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) override; + + // Additions + + void printDiagnostics(); + + void setBuilderCallback(void (* newBuilderCallback)(std::vector*, std::exception_ptr *exceptionToThrow)); + + void inhibitBuilds(); + void uninhibitBuilds(); +}; diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker.hs @@ -14,7 +14,6 @@ import Control.Monad.Except import Control.Monad.IO.Unlift import Control.Monad.Trans.Control -import qualified Data.ByteString as BS import qualified Data.Conduit import Data.Conduit.Extras (sinkChan, sinkChanTerminate, sourceChan) import Data.Conduit.Katip.Orphans () @@ -64,6 +63,9 @@ import Hercules.CNix.Expr.Context (EvalState) import qualified Hercules.CNix.Expr.Raw import Hercules.CNix.Expr.Typed (Value) +import Hercules.CNix.Std.Vector (StdVector) +import qualified Hercules.CNix.Std.Vector as Std.Vector +import Hercules.CNix.Store.Context (NixStorePathWithOutputs) import Hercules.CNix.Util (triggerInterrupt) import Hercules.Error import Katip @@ -82,8 +84,8 @@ import qualified Prelude data HerculesState = HerculesState - { drvsCompleted :: TVar (Map Text (UUID, BuildResult.BuildStatus)), - drvsInProgress :: IORef (Set Text), + { drvsCompleted :: TVar (Map StorePath (UUID, BuildResult.BuildStatus)), + drvsInProgress :: IORef (Set StorePath), herculesStore :: Ptr (Ref HerculesStore), wrappedStore :: Store, shortcutChannel :: Chan (Maybe Event) @@ -252,7 +254,8 @@ logLocM DebugS "Received remote build result" - liftIO $ atomically $ modifyTVar (drvsCompleted herculesState) (M.insert path (attempt, result)) + storePath <- liftIO $ CNix.parseStorePath (wrappedStore herculesState) (encodeUtf8 path) + liftIO $ atomically $ modifyTVar (drvsCompleted herculesState) (M.insert storePath (attempt, result)) _ -> pass Command.Build build -> katipAddNamespace "Build" $ @@ -371,7 +374,7 @@ Eval.LiteralArg s -> ["--argstr", encodeUtf8 k, s] Eval.ExprArg s -> ["--arg", encodeUtf8 k, s] -withDrvInProgress :: MonadUnliftIO m => HerculesState -> Text -> m a -> m a +withDrvInProgress :: MonadUnliftIO m => HerculesState -> StorePath -> m a -> m a withDrvInProgress HerculesState {drvsInProgress = ref} drvPath = bracket acquire release . const where @@ -435,54 +438,66 @@ UnliftIO unlift <- lift askUnliftIO let decode = decodeUtf8With lenientDecode liftIO $ - setBuilderCallback hStore $ \path -> unlift $ - katipAddContext (sl "fullpath" (decode path)) $ do - logLocM DebugS "Building" - let (plainDrv, bangOut) = BS.span (/= fromIntegral (ord '!')) path - outputName = BS.dropWhile (== fromIntegral (ord '!')) bangOut - plainDrvText = decode plainDrv - withDrvInProgress st plainDrvText $ do - liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (decode outputName) Nothing - derivation <- liftIO $ getDerivation store plainDrv - outputPath <- liftIO $ getDerivationOutputPath derivation outputName - katipAddContext (sl "outputPath" (decode outputPath)) $ - logLocM DebugS "Naively calling ensurePath" - liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e0 -> do - katipAddContext (sl "message" (show (e0 :: SomeException) :: Text)) $ - logLocM DebugS "Recovering from failed wrapped.ensurePath" - (attempt0, result) <- - liftIO $ - atomically $ do - c <- readTVar drvsCompl - anyAlternative $ M.lookup plainDrvText c - liftIO $ maybeThrowBuildException result plainDrvText - liftIO clearSubstituterCaches - liftIO $ clearPathInfoCache store - liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e1 -> do - katipAddContext (sl "message" (show (e1 :: SomeException) :: Text)) $ - logLocM DebugS "Recovering from fresh ensurePath" - liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (decode outputName) (Just attempt0) - -- TODO sync - result' <- - liftIO $ - atomically $ do - c <- readTVar drvsCompl - (attempt1, r) <- anyAlternative $ M.lookup plainDrvText c - guard (attempt1 /= attempt0) - pure r - liftIO $ maybeThrowBuildException result' plainDrvText - liftIO clearSubstituterCaches - liftIO $ clearPathInfoCache store - liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e2 -> - liftIO $ - throwIO $ - BuildException - plainDrvText - ( Just $ - "It could not be retrieved on the evaluating agent, despite a successful rebuild. Exception: " - <> show (e2 :: SomeException) - ) - logLocM DebugS "Built" + setBuilderCallback hStore $ + traverseSPWOs $ \storePathWithOutputs -> unlift $ do + drvStorePath <- liftIO $ getStorePath storePathWithOutputs + drvPath <- liftIO $ CNix.storePathToPath store drvStorePath + let pathText = decode drvPath + outputs <- liftIO $ getOutputs storePathWithOutputs + katipAddContext (sl "fullpath" pathText) $ + for_ outputs $ \outputName -> do + logLocM DebugS "Building" + withDrvInProgress st drvStorePath $ do + liftIO $ writeChan shortcutChan $ Just $ Event.Build drvPath (decode outputName) Nothing + derivation <- liftIO $ getDerivation store drvStorePath + drvName <- liftIO $ getDerivationNameFromPath drvStorePath + drvOutputs <- liftIO $ getDerivationOutputs store drvName derivation + outputPath <- + case find (\o -> derivationOutputName o == outputName) drvOutputs of + Nothing -> panic $ "output " <> show outputName <> " does not exist on " <> pathText + Just o -> case derivationOutputPath o of + Just x -> pure x + Nothing -> + -- FIXME ca-derivations + panic $ "output path unknown for output " <> show outputName <> " on " <> pathText <> ". ca-derivations is not supported yet." + katipAddContext (sl "outputPath" (show outputPath :: Text)) $ + logLocM DebugS "Naively calling ensurePath" + liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e0 -> do + katipAddContext (sl "message" (show (e0 :: SomeException) :: Text)) $ + logLocM DebugS "Recovering from failed wrapped.ensurePath" + (attempt0, result) <- + liftIO $ + atomically $ do + c <- readTVar drvsCompl + anyAlternative $ M.lookup drvStorePath c + liftIO $ maybeThrowBuildException result (decode drvPath) + liftIO clearSubstituterCaches + liftIO $ clearPathInfoCache store + liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e1 -> do + katipAddContext (sl "message" (show (e1 :: SomeException) :: Text)) $ + logLocM DebugS "Recovering from fresh ensurePath" + liftIO $ writeChan shortcutChan $ Just $ Event.Build drvPath (decode outputName) (Just attempt0) + -- TODO sync + result' <- + liftIO $ + atomically $ do + c <- readTVar drvsCompl + (attempt1, r) <- anyAlternative $ M.lookup drvStorePath c + guard (attempt1 /= attempt0) + pure r + liftIO $ maybeThrowBuildException result' (decode drvPath) + liftIO clearSubstituterCaches + liftIO $ clearPathInfoCache store + liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e2 -> + liftIO $ + throwIO $ + BuildException + (decode drvPath) + ( Just $ + "It could not be retrieved on the evaluating agent, despite a successful rebuild. Exception: " + <> show (e2 :: SomeException) + ) + logLocM DebugS "Built" withEvalStateConduit store $ \evalState -> do katipAddContext (sl "storeURI" (decode s)) $ logLocM DebugS "EvalState loaded." @@ -493,16 +508,22 @@ do imprt <- liftIO $ evalFile evalState (toS $ Eval.file eval) applied <- liftIO (autoCallFunction evalState imprt args) - walk evalState args applied + walk store evalState args applied yield Event.EvaluationDone +traverseSPWOs :: (StorePathWithOutputs -> IO ()) -> StdVector NixStorePathWithOutputs -> IO () +traverseSPWOs f v = do + v' <- Std.Vector.toListFP v + traverse_ f v' + walk :: (MonadUnliftIO m, KatipContext m) => + Store -> Ptr EvalState -> Value NixAttrs -> RawValue -> ConduitT i Event m () -walk evalState = walk' True [] 10 +walk store evalState = walk' True [] 10 where handleErrors path = Data.Conduit.handleC (yieldAttributeError path) walk' :: @@ -531,7 +552,8 @@ isDeriv <- liftIO $ isDerivation evalState v if isDeriv then do - drvPath <- getDrvFile evalState v + drvStorePath <- getDrvFile evalState v + drvPath <- liftIO $ CNix.storePathToPath store drvStorePath typE <- runExceptT do isEffect <- liftEitherAs Left =<< liftIO (getAttrBool evalState attrValue "isEffect") case isEffect of diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs @@ -5,7 +5,6 @@ import Conduit import Data.Conduit.Katip.Orphans () -import Foreign (ForeignPtr) import Hercules.Agent.Worker.Build.Prefetched (buildDerivation) import qualified Hercules.Agent.Worker.Build.Prefetched as Build import qualified Hercules.Agent.WorkerProtocol.Command.Build as Command.Build @@ -17,8 +16,7 @@ getDerivationOutputs, ) import qualified Hercules.CNix as CNix -import Hercules.CNix.Store (Store, queryPathInfo, validPathInfoNarHash, validPathInfoNarSize) -import Hercules.CNix.Store.Context (Derivation) +import Hercules.CNix.Store (Store, queryPathInfo, validPathInfoNarHash32, validPathInfoNarSize) import Katip import Protolude hiding (yield) @@ -26,8 +24,10 @@ runBuild store build = do let extraPaths = Command.Build.inputDerivationOutputPaths build drvPath = encodeUtf8 $ Command.Build.drvPath build - x <- for extraPaths $ \input -> do - liftIO $ try $ CNix.ensurePath store input + drvStorePath <- liftIO $ CNix.parseStorePath store drvPath + x <- for extraPaths $ \input -> liftIO $ do + storePath <- CNix.parseStorePath store input + try $ CNix.ensurePath store storePath materialize <- case sequenceA x of Right _ -> -- no error, proceed with requested materialization setting @@ -36,32 +36,35 @@ CNix.logInfo $ "while retrieving dependencies: " <> toS (displayException e) CNix.logInfo "unable to retrieve dependency; attempting fallback to local build" pure True - derivationMaybe <- liftIO $ Build.getDerivation store drvPath + drvName <- liftIO $ CNix.getDerivationNameFromPath drvStorePath + derivationMaybe <- liftIO $ Build.getDerivation store drvStorePath derivation <- case derivationMaybe of Just drv -> pure drv - Nothing -> panic $ "Could not retrieve derivation " <> show drvPath <> " from local store or binary caches." - nixBuildResult <- liftIO $ buildDerivation store drvPath derivation (extraPaths <$ guard (not materialize)) + Nothing -> panic $ "Could not retrieve derivation " <> show drvStorePath <> " from local store or binary caches." + nixBuildResult <- liftIO $ buildDerivation store drvStorePath derivation (extraPaths <$ guard (not materialize)) katipAddContext (sl "result" (show nixBuildResult :: Text)) $ logLocM DebugS "Build result" - buildResult <- liftIO $ enrichResult store derivation nixBuildResult + buildResult <- liftIO $ enrichResult store drvName derivation nixBuildResult yield $ Event.BuildResult buildResult -- TODO: case distinction on BuildStatus enumeration -enrichResult :: Store -> ForeignPtr Derivation -> Build.BuildResult -> IO Event.BuildResult.BuildResult -enrichResult _ _ result@Build.BuildResult {isSuccess = False} = +enrichResult :: Store -> ByteString -> CNix.Derivation -> Build.BuildResult -> IO Event.BuildResult.BuildResult +enrichResult _ _ _ result@Build.BuildResult {isSuccess = False} = pure $ Event.BuildResult.BuildFailure {errorMessage = Build.errorMessage result} -enrichResult store derivation _ = do - drvOuts <- getDerivationOutputs derivation +enrichResult store drvName derivation _ = do + drvOuts <- getDerivationOutputs store drvName derivation outputInfos <- for drvOuts $ \drvOut -> do - vpi <- queryPathInfo store (derivationOutputPath drvOut) - hash_ <- validPathInfoNarHash vpi - let size = validPathInfoNarSize vpi + -- FIXME: ca-derivations: always get the built path + vpi <- for (derivationOutputPath drvOut) (queryPathInfo store) + hash_ <- traverse validPathInfoNarHash32 vpi + path <- traverse (CNix.storePathToPath store) (derivationOutputPath drvOut) + let size = fmap validPathInfoNarSize vpi pure Event.BuildResult.OutputInfo { name = derivationOutputName drvOut, - path = derivationOutputPath drvOut, - hash = hash_, - size = size + path = fromMaybe "" path, + hash = fromMaybe "" hash_, + size = fromMaybe 0 size } pure $ Event.BuildResult.BuildSuccess outputInfos diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs @@ -12,6 +12,7 @@ import qualified Data.ByteString.Char8 as C8 import Foreign (FinalizerPtr, ForeignPtr, alloca, newForeignPtr, nullPtr, peek) import Foreign.C (peekCString) +import Hercules.CNix.Encapsulation import Hercules.CNix.Store import qualified Language.C.Inline.Cpp as C import qualified Language.C.Inline.Cpp.Exceptions as C @@ -37,6 +38,8 @@ C.include "" +C.include "" + C.include "" C.using "namespace nix" @@ -90,11 +93,11 @@ nullableForeignPtr _ rawPtr | rawPtr == nullPtr = pure Nothing nullableForeignPtr finalize rawPtr = Just <$> newForeignPtr finalize rawPtr -getDerivation :: Store -> ByteString -> IO (Maybe (ForeignPtr Derivation)) +getDerivation :: Store -> StorePath -> IO (Maybe Derivation) getDerivation (Store store) derivationPath = - nullableForeignPtr finalizeDerivation + nullableMoveToForeignPtrWrapper =<< [C.throwBlock| Derivation *{ - std::string derivationPath($bs-ptr:derivationPath, $bs-len:derivationPath); + StorePath derivationPath = *$fptr-ptr:(nix::StorePath *derivationPath); std::list> stores = getDefaultSubstituters(); stores.push_front(*$(refStore* store)); @@ -102,14 +105,7 @@ for (nix::ref & currentStore : stores) { try { - auto accessor = currentStore->getFSAccessor(); - auto drvText = accessor->readFile(derivationPath); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - Path drvTmpPath = tmpDir + "/drv"; - writeFile(drvTmpPath, drvText, 0600); - derivation = new nix::Derivation(nix::readDerivation(drvTmpPath)); + derivation = new nix::Derivation(currentStore->derivationFromPath(printPath23(*currentStore, derivationPath))); break; } catch (nix::Interrupted &e) { throw e; @@ -126,7 +122,7 @@ }|] -- | @buildDerivation derivationPath derivationText@ -buildDerivation :: Store -> ByteString -> ForeignPtr Derivation -> Maybe [ByteString] -> IO BuildResult +buildDerivation :: Store -> StorePath -> Derivation -> Maybe [ByteString] -> IO BuildResult buildDerivation (Store store) derivationPath derivation extraInputs = let extraInputsMerged = C8.intercalate "\n" (fromMaybe [] extraInputs) materializeDerivation = if isNothing extraInputs then 1 else 0 @@ -142,13 +138,19 @@ const char *&errorMessage = *$(const char **errorMessagePtr); time_t &startTime = *$(time_t *startTimePtr); time_t &stopTime = *$(time_t *stopTimePtr); - std::string derivationPath($bs-ptr:derivationPath, $bs-len:derivationPath); + StorePath derivationPath = *$fptr-ptr:(nix::StorePath *derivationPath); if ($(bool materializeDerivation)) { - store.ensurePath(derivationPath); - nix::PathSet paths{derivationPath}; + store.ensurePath(printPath23(store, derivationPath)); +#ifdef NIX_2_4 + auto derivation = store.derivationFromPath(derivationPath); + StorePathWithOutputs storePathWithOutputs { .path = derivationPath, .outputs = derivation.outputNames() }; + std::vector paths{storePathWithOutputs}; +#else + nix::PathSet paths{printPath23(store, derivationPath)}; +#endif try { - store.buildPaths(paths); + store.buildPaths(toDerivedPaths24(paths)); status = -1; success = true; errorMessage = strdup(""); @@ -170,10 +172,15 @@ std::istringstream stream(extraInputsMerged); while (std::getline(stream, extraInput)) { - derivation->inputSrcs.insert(extraInput); +#ifdef NIX_2_4 + auto path = store.parseStorePath(extraInput); +#else + auto path = extraInput; +#endif + derivation->inputSrcs.insert(path); } - nix::BuildResult result = store.buildDerivation(derivationPath, *derivation); + nix::BuildResult result = store.buildDerivation(printPath23(store, derivationPath), *derivation); switch (result.status) { case nix::BuildResult::Built: diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs @@ -4,13 +4,12 @@ module Hercules.Agent.Worker.Effect where import Control.Monad.Catch (MonadThrow) -import GHC.ForeignPtr (ForeignPtr) import Hercules.Agent.Worker.Build.Prefetched (buildDerivation) import qualified Hercules.Agent.Worker.Build.Prefetched as Build import qualified Hercules.Agent.WorkerProtocol.Command.Effect as Command.Effect import Hercules.CNix (Store) import qualified Hercules.CNix as CNix -import Hercules.CNix.Store.Context (Derivation) +import Hercules.CNix.Store (Derivation) import qualified Hercules.Effect as Effect import Katip (KatipContext) import Protolude @@ -22,28 +21,29 @@ dir <- getCurrentDirectory Effect.runEffect derivation (Command.Effect.token command) (Command.Effect.secretsPath command) (Command.Effect.apiBaseURL command) dir -prepareDerivation :: MonadIO m => Store -> Command.Effect.Effect -> m (ForeignPtr Derivation) +prepareDerivation :: MonadIO m => Store -> Command.Effect.Effect -> m Derivation prepareDerivation store command = do let extraPaths = Command.Effect.inputDerivationOutputPaths command drvPath = encodeUtf8 $ Command.Effect.drvPath command - ensureDeps = for_ extraPaths $ \input -> - liftIO $ CNix.ensurePath store input + ensureDeps = for_ extraPaths $ \input -> liftIO $ do + CNix.ensurePath store =<< CNix.parseStorePath store input + drvStorePath <- liftIO $ CNix.parseStorePath store drvPath liftIO $ do ensureDeps `catch` \e -> do CNix.logInfo $ "while retrieving dependencies: " <> toS (displayException (e :: SomeException)) CNix.logInfo "unable to retrieve dependency; attempting fallback to local build" - CNix.ensurePath store drvPath - derivation <- CNix.getDerivation store drvPath - depDrvPaths <- CNix.getDerivationInputs derivation + CNix.ensurePath store drvStorePath + derivation <- CNix.getDerivation store drvStorePath + depDrvPaths <- CNix.getDerivationInputs store derivation for_ depDrvPaths \(depDrv, _outputs) -> do depDerivation <- CNix.getDerivation store depDrv _nixBuildResult <- liftIO $ buildDerivation store depDrv depDerivation mempty pass derivation <- - liftIO (Build.getDerivation store drvPath) >>= \case + liftIO (Build.getDerivation store drvStorePath) >>= \case Just drv -> pure drv Nothing -> panic $ "Could not retrieve derivation " <> show drvPath <> " from local store or binary caches." - sources <- liftIO $ CNix.getDerivationSources derivation + sources <- liftIO $ CNix.getDerivationSources store derivation for_ sources \src -> do liftIO $ CNix.ensurePath store src pure derivation diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs @@ -11,9 +11,7 @@ import Control.Exception ( catch, ) -import qualified Data.ByteString.Unsafe as BS import Data.Coerce (coerce) -import qualified Foreign.C import Foreign.C.String (withCString) import Foreign.StablePtr (castStablePtrToPtr, newStablePtr) import Hercules.Agent.Worker.HerculesStore.Context @@ -21,8 +19,10 @@ HerculesStore, context, ) -import Hercules.CNix (Store) +import Hercules.CNix.Encapsulation (moveToForeignPtrWrapper) import Hercules.CNix.Expr (Store (Store)) +import Hercules.CNix.Expr.Context (NixStorePathWithOutputs) +import Hercules.CNix.Std.Vector (CStdVector, StdVector) import Hercules.CNix.Store.Context (Ref) import qualified Language.C.Inline.Cpp as C import qualified Language.C.Inline.Cpp.Exceptions as C @@ -47,6 +47,8 @@ C.include "" +C.include "" + C.include "hercules-aliases.h" C.using "namespace nix" @@ -76,21 +78,21 @@ }|] -- TODO catch pure exceptions from displayException -setBuilderCallback :: Ptr (Ref HerculesStore) -> (ByteString -> IO ()) -> IO () +setBuilderCallback :: Ptr (Ref HerculesStore) -> (StdVector NixStorePathWithOutputs -> IO ()) -> IO () setBuilderCallback s callback = do p <- - mkBuilderCallback $ \cstr exceptionToThrowPtr -> - Control.Exception.catch (BS.unsafePackMallocCString cstr >>= callback) $ \e -> + mkBuilderCallback $ \sp exceptionToThrowPtr -> + Control.Exception.catch (callback =<< moveToForeignPtrWrapper sp) $ \e -> withCString (displayException (e :: SomeException)) $ \renderedException -> do stablePtr <- castStablePtrToPtr <$> newStablePtr e [C.block| void { (*$(exception_ptr *exceptionToThrowPtr)) = std::make_exception_ptr(HaskellException(std::string($(const char* renderedException)), $(void* stablePtr))); }|] [C.throwBlock| void { - (*$(refHerculesStore* s))->setBuilderCallback($(void (*p)(const char *, exception_ptr *) )); + (*$(refHerculesStore* s))->setBuilderCallback($(void (*p)(std::vector*, exception_ptr *))); }|] -type BuilderCallback = Foreign.C.CString -> Ptr ExceptionPtr -> IO () +type BuilderCallback = Ptr (CStdVector NixStorePathWithOutputs) -> Ptr ExceptionPtr -> IO () -- Work around a problem in ghcide with foreign imports. #ifndef __GHCIDE__ diff --git a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs --- a/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs +++ b/hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs @@ -6,6 +6,7 @@ import qualified Data.Map as M import Hercules.CNix.Expr.Context (Ref) +import Hercules.CNix.Std.Vector (stdVectorCtx) import qualified Hercules.CNix.Store.Context as Store import qualified Language.C.Inline.Context as C import qualified Language.C.Inline.Cpp as C @@ -20,6 +21,7 @@ context = C.cppCtx <> C.fptrCtx <> C.bsCtx + <> stdVectorCtx <> Store.context <> herculesStoreContext diff --git a/hercules-ci-agent.cabal b/hercules-ci-agent.cabal --- a/hercules-ci-agent.cabal +++ b/hercules-ci-agent.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: hercules-ci-agent -version: 0.8.1 +version: 0.8.2 synopsis: Runs Continuous Integration tasks on your machines category: Nix, CI, Testing, DevOps homepage: https://docs.hercules-ci.com @@ -14,10 +14,15 @@ extra-source-files: CHANGELOG.md cbits/hercules-aliases.h - cbits/hercules-store.hh cbits/hercules-logger.hh + cbits/nix-2.3/hercules-store.hh + cbits/nix-2.4/hercules-store.hh testdata/vm-test-run-agent-test.drv +flag nix-2_4 + description: Build for Nix >=2.4pre* + default: False + source-repository head type: git location: https://github.com/hercules-ci/hercules-ci-agent @@ -183,8 +188,8 @@ , exceptions , filepath , hercules-ci-agent - , hercules-ci-api-core == 0.1.2.0 - , hercules-ci-api-agent == 0.3.1.0 + , hercules-ci-api-core == 0.1.3.0 + , hercules-ci-api-agent == 0.4.0.0 , hostname , http-client , http-client-tls @@ -243,8 +248,16 @@ hs-source-dirs: hercules-ci-agent-worker cxx-sources: - cbits/hercules-store.cxx cbits/hercules-logger.cxx + if flag(nix-2_4) + cpp-options: + -DNIX_2_4 + cxx-sources: + cbits/nix-2.4/hercules-store.cxx + else + cxx-sources: + cbits/nix-2.3/hercules-store.cxx + default-extensions: DeriveGeneric DeriveTraversable DisambiguateRecordFields FlexibleContexts InstanceSigs LambdaCase MultiParamTypeClasses NoImplicitPrelude OverloadedStrings RankNTypes TupleSections TypeApplications TypeOperators ghc-options: -Werror=incomplete-patterns -Werror=missing-fields @@ -259,6 +272,12 @@ -with-rtsopts=-maxN8 include-dirs: cbits + if flag(nix-2_4) + include-dirs: + cbits/nix-2.4 + else + include-dirs: + cbits/nix-2.3 extra-libraries: boost_context build-depends: @@ -352,6 +371,7 @@ , process , protolude , safe-exceptions + , temporary , text , transformers-base , unliftio-core diff --git a/hercules-ci-agent/Hercules/Agent.hs b/hercules-ci-agent/Hercules/Agent.hs --- a/hercules-ci-agent/Hercules/Agent.hs +++ b/hercules-ci-agent/Hercules/Agent.hs @@ -37,6 +37,7 @@ import qualified Hercules.Agent.Build as Build import Hercules.Agent.CabalInfo (herculesAgentVersion) import qualified Hercules.Agent.Cache as Cache +import qualified Hercules.Agent.Cachix.Env as Cachix.Env import Hercules.Agent.Client ( lifeCycleClient, tasksClient, @@ -117,7 +118,9 @@ ServicePayload.StartEvaluation evalTask -> launchTask tasks socket (Task.upcastId $ EvaluateTask.id evalTask) do Cache.withCaches do - Evaluate.performEvaluation evalTask + -- TODO move the store directly under env + store <- asks (Cachix.Env.nixStore . Env.cachixEnv) + Evaluate.performEvaluation store evalTask pure $ TaskStatus.Successful () ServicePayload.StartBuild buildTask -> launchTask tasks socket (Task.upcastId $ BuildTask.id buildTask) do diff --git a/hercules-ci-agent/Hercules/Agent/Build.hs b/hercules-ci-agent/Hercules/Agent/Build.hs --- a/hercules-ci-agent/Hercules/Agent/Build.hs +++ b/hercules-ci-agent/Hercules/Agent/Build.hs @@ -17,6 +17,7 @@ import Hercules.API.TaskStatus (TaskStatus) import qualified Hercules.API.TaskStatus as TaskStatus import qualified Hercules.Agent.Cache as Agent.Cache +import qualified Hercules.Agent.Cachix.Env as Cachix.Env import qualified Hercules.Agent.Client import qualified Hercules.Agent.Config as Config import Hercules.Agent.Env @@ -32,6 +33,7 @@ import qualified Hercules.Agent.WorkerProtocol.Event as Event import qualified Hercules.Agent.WorkerProtocol.Event.BuildResult as BuildResult import qualified Hercules.Agent.WorkerProtocol.LogSettings as LogSettings +import qualified Hercules.CNix.Store as CNix import Hercules.Error (defaultRetry) import qualified Network.URI import Protolude @@ -85,7 +87,8 @@ Just BuildResult.BuildSuccess {outputs = outs'} -> do let outs = convertOutputs (BuildTask.derivationPath buildTask) outs' reportOutputInfos buildTask outs - push buildTask outs + store <- asks (Cachix.Env.nixStore . Env.cachixEnv) + push store buildTask outs reportSuccess buildTask pure $ TaskStatus.Successful () Just BuildResult.BuildFailure {} -> pure $ TaskStatus.Terminated () @@ -102,12 +105,14 @@ hash = decodeUtf8With lenientDecode $ BuildResult.hash oi } -push :: BuildTask -> Map Text OutputInfo -> App () -push buildTask outs = do +push :: CNix.Store -> BuildTask -> Map Text OutputInfo -> App () +push store buildTask outs = do let paths = OutputInfo.path <$> toList outs caches <- activePushCaches forM_ caches $ \cache -> do - Agent.Cache.push cache paths 4 + -- TODO preserve StorePath instead + storePaths <- liftIO $ for paths (CNix.parseStorePath store . encodeUtf8) + Agent.Cache.push cache storePaths 4 emitEvents buildTask [BuildEvent.Pushed $ Pushed.Pushed {cache = cache}] reportSuccess :: BuildTask -> App () diff --git a/hercules-ci-agent/Hercules/Agent/Cache.hs b/hercules-ci-agent/Hercules/Agent/Cache.hs --- a/hercules-ci-agent/Hercules/Agent/Cache.hs +++ b/hercules-ci-agent/Hercules/Agent/Cache.hs @@ -13,6 +13,9 @@ import qualified Hercules.Agent.Nix as Nix import qualified Hercules.Agent.SecureDirectory as SecureDirectory import qualified Hercules.CNix as CNix +import Hercules.CNix.Std.Set (StdSet, toListFP) +import qualified Hercules.CNix.Std.Set as Std.Set +import Hercules.CNix.Store (StorePath) import qualified Hercules.CNix.Store as Store import qualified Hercules.Formats.NixCache as NixCache import Katip @@ -42,7 +45,7 @@ -- | Cache name Text -> -- | Paths, which do not have to form a closure - [Text] -> + [StorePath] -> -- | Number of concurrent upload threads Int -> App () @@ -58,14 +61,13 @@ throwIO $ FatalError $ "Agent does not have a binary cache named " <> show cacheName <> " in its configuration." fromMaybe failNothing (maybeNix <|> maybeCachix) -nixPush :: NixCache.NixCache -> [Text] -> Int -> App () -nixPush cacheConf pathsText _concurrency = do - let paths = map encodeUtf8 pathsText +nixPush :: NixCache.NixCache -> [StorePath] -> Int -> App () +nixPush cacheConf paths _concurrency = do keys <- liftIO $ for (NixCache.signingKeys cacheConf) (CNix.parseSecretKey . encodeUtf8) CNix.withStore $ \store -> do - (Sum total, Sum signed) <- - mconcat <$> for keys \key -> liftIO $ do - pathSet <- paths & newPathSetWith + (Sum total, Sum signed) <- liftIO do + pathSet <- Std.Set.fromListFP paths + mconcat <$> for keys \key -> do signClosure store key pathSet katipAddContext (sl "num-signatures" signed <> sl "num-paths" total) $ logLocM DebugS "Signed" @@ -73,18 +75,11 @@ CNix.withStoreFromURI (NixCache.storeURI cacheConf) $ \cache -> do liftIO (CNix.copyClosure store cache paths) -newPathSetWith :: [ByteString] -> IO Store.PathSet -newPathSetWith paths = do - pathSet <- Store.newEmptyPathSet - for_ paths \path -> - Store.addToPathSet path pathSet - pure pathSet - -signClosure :: CNix.Store -> ForeignPtr CNix.SecretKey -> Store.PathSet -> IO (Sum Int, Sum Int) +signClosure :: CNix.Store -> ForeignPtr CNix.SecretKey -> StdSet Store.NixStorePath -> IO (Sum Int, Sum Int) signClosure store key' pathSet = withForeignPtr key' \key -> do - closure <- Store.computeFSClosure store Store.defaultClosureParams pathSet + closure <- Store.computeFSClosure store Store.defaultClosureParams pathSet >>= toListFP closure - & Store.traversePathSet + & traverse ( \path -> do CNix.signPath store key path ) diff --git a/hercules-ci-agent/Hercules/Agent/Cachix.hs b/hercules-ci-agent/Hercules/Agent/Cachix.hs --- a/hercules-ci-agent/Hercules/Agent/Cachix.hs +++ b/hercules-ci-agent/Hercules/Agent/Cachix.hs @@ -12,11 +12,12 @@ import Hercules.Agent.Env as Agent.Env hiding (activePushCaches) import qualified Hercules.Agent.EnvironmentInfo as EnvInfo import Hercules.Agent.Log +import Hercules.CNix.Store (StorePath) import Hercules.Error import qualified Hercules.Formats.CachixCache as CachixCache import Protolude -push :: Text -> [Text] -> Int -> App () +push :: Text -> [StorePath] -> Int -> App () push cache paths workers = withNamedContext "cache" cache $ do Agent.Cachix.Env { pushCaches = pushCaches, @@ -36,7 +37,7 @@ pushParamsStore = nixStore, pushParamsClientEnv = clientEnv, pushParamsStrategy = \storePath -> - let ctx = withNamedContext "path" storePath + let ctx = withNamedContext "path" (show storePath :: Text) in Cachix.Push.PushStrategy { onAlreadyPresent = pass, onAttempt = \retryStatus size -> @@ -65,7 +66,7 @@ cks <- asks (Agent.Cachix.cacheKeys . Agent.Env.cachixEnv) nixInfo <- liftIO EnvInfo.getNixInfo pure - ( EnvInfo.nixSubstituters nixInfo + ( map (decodeUtf8With lenientDecode) (EnvInfo.nixSubstituters nixInfo) ++ map (\c -> "https://" <> c <> ".cachix.org") (M.keys cks) ) @@ -73,4 +74,7 @@ getTrustedPublicKeys = do cks <- asks (Agent.Cachix.cacheKeys . Agent.Env.cachixEnv) nixInfo <- liftIO EnvInfo.getNixInfo - pure (EnvInfo.nixTrustedPublicKeys nixInfo ++ concatMap CachixCache.publicKeys cks) + pure + ( map (decodeUtf8With lenientDecode) (EnvInfo.nixTrustedPublicKeys nixInfo) + ++ concatMap CachixCache.publicKeys cks + ) diff --git a/hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs b/hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs --- a/hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs +++ b/hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs @@ -1,18 +1,9 @@ +{-# LANGUAGE ScopedTypeVariables #-} + module Hercules.Agent.EnvironmentInfo where -import Control.Lens - ( to, - (^..), - (^?), - ) -import qualified Data.Aeson as Aeson -import Data.Aeson.Lens - ( key, - _Array, - _Number, - _String, - ) -import qualified Data.ByteString.Lazy as LBS +import qualified Data.ByteString as BS +import qualified Data.Set as S import qualified Data.Text as T import qualified Hercules.API.Agent.LifeCycle.AgentInfo as AgentInfo import Hercules.Agent.CabalInfo as CabalInfo @@ -20,10 +11,11 @@ import qualified Hercules.Agent.Config as Config import Hercules.Agent.Env as Env import Hercules.Agent.Log +import qualified Hercules.CNix as CNix +import qualified Hercules.CNix.Settings as Settings import qualified Hercules.CNix.Store as Store import Network.HostName (getHostName) import Protolude hiding (to) -import qualified System.Process as Process extractAgentInfo :: App AgentInfo.AgentInfo extractAgentInfo = do @@ -37,15 +29,15 @@ let s = AgentInfo.AgentInfo { hostname = toS hostname, - agentVersion = CabalInfo.herculesAgentVersion, -- TODO: Add git revision - nixVersion = nixExeVersion nix, + agentVersion = CabalInfo.herculesAgentVersion, + nixVersion = nixLibVersion nix, nixClientProtocolVersion = nixClientProtocolVersion, nixDaemonProtocolVersion = nixStoreProtocolVersion, - platforms = nixPlatforms nix, + platforms = map fromUtf8Lenient $ nixPlatforms nix, cachixPushCaches = cachixPushCaches, pushCaches = pushCaches, - systemFeatures = nixSystemFeatures nix, - substituters = nixSubstituters nix, -- TODO: Add cachix substituters + systemFeatures = map fromUtf8Lenient $ nixSystemFeatures nix, + substituters = map fromUtf8Lenient $ nixSubstituters nix, concurrentTasks = fromIntegral $ Config.concurrentTasks cfg, labels = Config.labels cfg } @@ -53,72 +45,39 @@ pure s data NixInfo = NixInfo - { nixExeVersion :: Text, - nixPlatforms :: [Text], - nixSystemFeatures :: [Text], - nixSubstituters :: [Text], - nixTrustedPublicKeys :: [Text], - nixNarinfoCacheNegativeTTL :: Maybe Integer, - nixNetrcFile :: Maybe Text + { nixLibVersion :: Text, + nixPlatforms :: [ByteString], + nixSystemFeatures :: [ByteString], + nixSubstituters :: [ByteString], + nixTrustedPublicKeys :: [ByteString], + nixNarinfoCacheNegativeTTL :: Word64, + nixNetrcFile :: Maybe ByteString } + deriving (Show) +fromUtf8Lenient :: ByteString -> Text +fromUtf8Lenient = decodeUtf8With lenientDecode + getNixInfo :: IO NixInfo getNixInfo = do - let stdinEmpty = "" - version <- Process.readProcess "nix" ["--version"] stdinEmpty - rawJson <- Process.readProcess "nix" ["show-config", "--json"] stdinEmpty - cfg <- - case Aeson.eitherDecode (LBS.fromStrict $ encodeUtf8 $ toS rawJson) of - Left e -> panic $ "Could not parse nix show-config --json: " <> show e - Right r -> pure r + extraPlatforms <- Settings.getExtraPlatforms + system <- Settings.getSystem + systemFeatures <- Settings.getSystemFeatures + substituters <- Settings.getSubstituters + trustedPublicKeys <- Settings.getTrustedPublicKeys + narinfoCacheNegativeTTL <- Settings.getNarinfoCacheNegativeTtl + netrcFile <- Settings.getNetrcFile pure NixInfo - { nixExeVersion = T.dropAround isSpace (toS version), - nixPlatforms = - ((cfg :: Aeson.Value) ^.. key "system" . key "value" . _String) - <> ( cfg - ^.. key "extra-platforms" - . key "value" - . _Array - . traverse - . _String - ), - nixSystemFeatures = - cfg - ^.. key "system-features" - . key "value" - . _Array - . traverse - . _String, - nixSubstituters = - cfg - ^.. key "substituters" - . key "value" - . _Array - . traverse - . _String - . to cleanUrl, - nixTrustedPublicKeys = - cfg - ^.. key "trusted-public-keys" - . key "value" - . _Array - . traverse - . _String - . to cleanUrl, - nixNarinfoCacheNegativeTTL = - cfg - ^? key "narinfo-cache-negative-ttl" - . key "value" - . _Number - . to floor, - nixNetrcFile = - cfg - ^? key "netrc-file" - . key "value" - . _String + { nixLibVersion = T.dropAround isSpace (fromUtf8Lenient CNix.nixVersion), + nixPlatforms = toList (S.singleton system <> extraPlatforms), + nixSystemFeatures = toList systemFeatures, + nixSubstituters = map cleanUrl substituters, + nixTrustedPublicKeys = trustedPublicKeys, + nixNarinfoCacheNegativeTTL = narinfoCacheNegativeTTL, + nixNetrcFile = guard (netrcFile /= "") $> netrcFile } -cleanUrl :: Text -> Text -cleanUrl t | "@" `T.isInfixOf` t = "" +cleanUrl :: ByteString -> ByteString +cleanUrl t | "@" `BS.isInfixOf` t = "" cleanUrl t = t diff --git a/hercules-ci-agent/Hercules/Agent/Evaluate.hs b/hercules-ci-agent/Hercules/Agent/Evaluate.hs --- a/hercules-ci-agent/Hercules/Agent/Evaluate.hs +++ b/hercules-ci-agent/Hercules/Agent/Evaluate.hs @@ -40,9 +40,11 @@ import qualified Hercules.API.Agent.Evaluate.EvaluateTask as EvaluateTask import Hercules.API.Servant (noContent) import qualified Hercules.Agent.Cache as Agent.Cache +import qualified Hercules.Agent.Cachix.Env as Cachix.Env import qualified Hercules.Agent.Client import Hercules.Agent.Env import qualified Hercules.Agent.Env as Env +import Hercules.Agent.Evaluate.TraversalQueue (Queue) import qualified Hercules.Agent.Evaluate.TraversalQueue as TraversalQueue import Hercules.Agent.Files import Hercules.Agent.Log @@ -66,7 +68,7 @@ import qualified Hercules.Agent.WorkerProtocol.Event.Attribute as WorkerAttribute import qualified Hercules.Agent.WorkerProtocol.Event.AttributeError as WorkerAttributeError import qualified Hercules.Agent.WorkerProtocol.LogSettings as LogSettings -import Hercules.CNix (withStore) +import Hercules.CNix.Store (Store, StorePath, parseStorePath) import Hercules.Error (defaultRetry, quickRetry) import qualified Network.HTTP.Client.Conduit as HTTP.Conduit import qualified Network.HTTP.Simple as HTTP.Simple @@ -83,18 +85,19 @@ pushEvalWorkers :: Int pushEvalWorkers = 16 -performEvaluation :: EvaluateTask.EvaluateTask -> App () -performEvaluation task' = - withProducer (produceEvaluationTaskEvents task') $ \producer -> +performEvaluation :: Store -> EvaluateTask.EvaluateTask -> App () +performEvaluation store task' = + withProducer (produceEvaluationTaskEvents store task') $ \producer -> withBoundedDelayBatchProducer (1000 * 1000) 1000 producer $ \batchProducer -> fix $ \continue -> joinSTM $ listen batchProducer (\b -> withSync b (postBatch task' . catMaybes) *> continue) pure produceEvaluationTaskEvents :: + Store -> EvaluateTask.EvaluateTask -> (Syncing EvaluateEvent.EvaluateEvent -> App ()) -> App () -produceEvaluationTaskEvents task writeToBatch = withWorkDir "eval" $ \tmpdir -> do +produceEvaluationTaskEvents store task writeToBatch = withWorkDir "eval" $ \tmpdir -> do logLocM DebugS "Retrieving evaluation task" let emitSingle = writeToBatch . Syncable sync = syncer writeToBatch @@ -206,7 +209,7 @@ Message.typ = Message.Error, Message.message = e } - Right file -> TraversalQueue.with $ \derivationQueue -> + Right file -> TraversalQueue.with $ \(derivationQueue :: Queue StorePath) -> let doIt = do Async.Lifted.concurrently_ evaluation emitDrvs -- derivationInfo upload has finished @@ -241,16 +244,17 @@ emit $ EvaluateEvent.PushedAll $ PushedAll.PushedAll {cache = cache} captureAttrDrvAndEmit msg = do case msg of - EvaluateEvent.Attribute ae -> - addTopDerivation (AttributeEvent.derivationPath ae) + EvaluateEvent.Attribute ae -> do + storePath <- liftIO $ parseStorePath store (encodeUtf8 $ AttributeEvent.derivationPath ae) + addTopDerivation storePath _ -> pass emit msg emitDrvs = - withStore $ \store -> TraversalQueue.work derivationQueue $ \recurse drvPath -> do + TraversalQueue.work derivationQueue $ \recurse drvPath -> do drvInfo <- retrieveDerivationInfo store drvPath - forM_ - (M.keys $ DerivationInfo.inputDerivations drvInfo) - recurse -- asynchronously + for_ (M.keys $ DerivationInfo.inputDerivations drvInfo) \inp -> do + inputStorePath <- liftIO $ parseStorePath store (encodeUtf8 inp) + recurse inputStorePath -- asynchronously emit $ EvaluateEvent.DerivationInfo drvInfo in doIt @@ -273,7 +277,7 @@ ] -> (EvaluateEvent.EvaluateEvent -> App ()) -> -- | Upload a derivation, return when done - (Text -> App ()) -> + (StorePath -> App ()) -> App () -> Text -> App () @@ -339,13 +343,16 @@ } continue Event.Build drv outputName notAttempt -> do + store <- asks (Cachix.Env.nixStore . Env.cachixEnv) + storePath <- liftIO (parseStorePath store drv) + let drvText = decode drv status <- - withNamedContext "derivation" drv $ do + withNamedContext "derivation" (decode drv) $ do currentIndex <- liftIO $ atomicModifyIORef buildRequiredIndex (\i -> (i + 1, i)) emit $ EvaluateEvent.BuildRequired BuildRequired.BuildRequired - { BuildRequired.derivationPath = drv, + { BuildRequired.derivationPath = drvText, BuildRequired.index = currentIndex, BuildRequired.outputName = outputName } @@ -353,21 +360,21 @@ caches <- activePushCaches forM_ caches $ \cache -> do withNamedContext "cache" cache $ logLocM DebugS "Pushing derivations for import from derivation" - Agent.Cache.push cache [drv] pushEvalWorkers + Agent.Cache.push cache [storePath] pushEvalWorkers Async.Lifted.concurrently_ - (uploadDerivationInfos drv) + (uploadDerivationInfos storePath) pushDerivations emit $ EvaluateEvent.BuildRequest BuildRequest.BuildRequest - { derivationPath = drv, + { derivationPath = drvText, forceRebuild = isJust notAttempt } flush - status <- drvPoller notAttempt drv + status <- drvPoller notAttempt drvText logLocM DebugS $ "Got derivation status " <> logStr (show status :: Text) return status - writeChan commandChan $ Just $ Command.BuildResult $ uncurry (BuildResult.BuildResult drv) status + writeChan commandChan $ Just $ Command.BuildResult $ uncurry (BuildResult.BuildResult drvText) status continue Event.Exception e -> panic e -- Unused during eval diff --git a/hercules-ci-agent/Hercules/Agent/Nix.hs b/hercules-ci-agent/Hercules/Agent/Nix.hs --- a/hercules-ci-agent/Hercules/Agent/Nix.hs +++ b/hercules-ci-agent/Hercules/Agent/Nix.hs @@ -8,8 +8,6 @@ import Hercules.Agent.Nix.Env as Nix.Env import Protolude import System.Directory (doesFileExist) -import System.Process (readProcess) -import qualified System.Process withExtraOptions :: [(Text, Text)] -> App a -> App a withExtraOptions extraOpts = local $ \env -> @@ -23,35 +21,10 @@ askExtraOptions :: MonadReader Agent.Env.Env m => m [(Text, Text)] askExtraOptions = asks (extraOptions . nixEnv) -getExtraOptionArguments :: MonadReader Agent.Env.Env m => m [Text] -getExtraOptionArguments = do - asks (f . extraOptions . nixEnv) - where - f = concatMap $ \(opt, val) -> ["--option", opt, val] - -readNixProcess :: - -- | Command - Text -> - -- | Options - [Text] -> - -- | Paths - [Text] -> - -- | Stdin - Text -> - App Text -readNixProcess cmd opts paths stdinText = do - extraOpts <- getExtraOptionArguments - toS <$> liftIO (readProcess (toS cmd) (map toS (opts <> extraOpts <> ["--"] <> paths)) (toS stdinText)) - -nixProc :: Text -> [Text] -> [Text] -> App System.Process.CreateProcess -nixProc exe opts args = do - extraOpts <- getExtraOptionArguments - pure $ System.Process.proc (toS exe) (map toS (opts <> extraOpts <> ["--"] <> args)) - getNetrcLines :: App [Text] getNetrcLines = liftIO $ do info <- getNixInfo - let fm = toS <$> nixNetrcFile info + let fm = toS . decodeUtf8With lenientDecode <$> nixNetrcFile info fmap fold <$> runMaybeT $ do f <- MaybeT $ pure fm exs <- lift $ doesFileExist f diff --git a/hercules-ci-agent/Hercules/Agent/Nix/Init.hs b/hercules-ci-agent/Hercules/Agent/Nix/Init.hs --- a/hercules-ci-agent/Hercules/Agent/Nix/Init.hs +++ b/hercules-ci-agent/Hercules/Agent/Nix/Init.hs @@ -7,7 +7,8 @@ newEnv :: IO Env newEnv = do nixInfo <- EnvironmentInfo.getNixInfo - when (EnvironmentInfo.nixNarinfoCacheNegativeTTL nixInfo /= Just 0) $ do + print nixInfo + when (EnvironmentInfo.nixNarinfoCacheNegativeTTL nixInfo /= 0) $ do putErrText "\n\ \We have detected that the setting narinfo-cache-negative-ttl is non-zero.\n\ diff --git a/hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs b/hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs --- a/hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs +++ b/hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs @@ -1,8 +1,9 @@ +{-# LANGUAGE BlockArguments #-} + module Hercules.Agent.Nix.RetrieveDerivationInfo where import qualified Data.Map as M import qualified Data.Text as T -import Foreign.ForeignPtr (ForeignPtr) import Hercules.API.Agent.Evaluate.EvaluateEvent.DerivationInfo ( DerivationInfo (DerivationInfo), ) @@ -13,39 +14,49 @@ retrieveDerivationInfo :: MonadIO m => Store -> - DerivationInfo.DerivationPathText -> + StorePath -> m DerivationInfo retrieveDerivationInfo store drvPath = liftIO $ do - drv <- getDerivation store (encodeUtf8 drvPath) - retrieveDerivationInfo' drvPath drv + drv <- getDerivation store drvPath + path <- storePathToPath store drvPath + drvName <- getDerivationNameFromPath drvPath + retrieveDerivationInfo' store path drvName drv -retrieveDerivationInfo' :: Text -> ForeignPtr Derivation -> IO DerivationInfo -retrieveDerivationInfo' drvPath drv = do - sourcePaths <- getDerivationSources drv - inputDrvPaths <- getDerivationInputs drv - outputs <- getDerivationOutputs drv +retrieveDerivationInfo' :: Store -> ByteString -> ByteString -> Derivation -> IO DerivationInfo +retrieveDerivationInfo' store drvPath drvName drv = do + sourceStorePaths <- getDerivationSources store drv + inputDrvStorePaths <- getDerivationInputs store drv + outputs <- getDerivationOutputs store drvName drv env <- getDerivationEnv drv platform <- getDerivationPlatform drv let requiredSystemFeatures = maybe [] splitFeatures $ M.lookup "requiredSystemFeatures" env splitFeatures = filter (not . T.null) . T.split (== ' ') . decode decode = decodeUtf8With lenientDecode + inputDrvPaths <- for inputDrvStorePaths \(storePath, inputOutputs) -> do + path <- Hercules.CNix.storePathToPath store storePath + pure (path, inputOutputs) + sourcePaths <- for sourceStorePaths (Hercules.CNix.storePathToPath store) + outputs' <- for outputs \output -> do + path <- for (derivationOutputPath output) (Hercules.CNix.storePathToPath store) + let isFixed = case derivationOutputDetail output of + (DerivationOutputInputAddressed _s) -> False + (DerivationOutputCAFixed _f _s) -> True + (DerivationOutputCAFloating _f _h) -> False + DerivationOutputDeferred -> False + pure + ( decode $ derivationOutputName output, + DerivationInfo.OutputInfo + { DerivationInfo.path = decode <$> path, + DerivationInfo.isFixed = isFixed + } + ) pure $ DerivationInfo - { derivationPath = drvPath, + { derivationPath = decode drvPath, platform = decode platform, requiredSystemFeatures = requiredSystemFeatures, inputDerivations = inputDrvPaths & map (bimap decode (map decode)) & M.fromList, inputSources = map decode sourcePaths, outputs = - outputs - & map - ( \output -> - ( decode $ derivationOutputName output, - DerivationInfo.OutputInfo - { DerivationInfo.path = decode $ derivationOutputPath output, - DerivationInfo.isFixed = derivationOutputHashAlgo output /= "" - } - ) - ) - & M.fromList + outputs' & M.fromList } diff --git a/src/Hercules/Agent/WorkerProtocol/Event.hs b/src/Hercules/Agent/WorkerProtocol/Event.hs --- a/src/Hercules/Agent/WorkerProtocol/Event.hs +++ b/src/Hercules/Agent/WorkerProtocol/Event.hs @@ -15,7 +15,7 @@ | AttributeError AttributeError | EvaluationDone | Error Text - | Build Text Text (Maybe UUID) + | Build ByteString Text (Maybe UUID) | BuildResult BuildResult | EffectResult Int | Exception Text diff --git a/src/Hercules/Effect.hs b/src/Hercules/Effect.hs --- a/src/Hercules/Effect.hs +++ b/src/Hercules/Effect.hs @@ -8,10 +8,9 @@ import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BL import qualified Data.Map as M -import Foreign (ForeignPtr) import Hercules.Agent.Sensitive (Sensitive (Sensitive, reveal), revealContainer) +import Hercules.CNix (Derivation) import Hercules.CNix.Store (getDerivationArguments, getDerivationBuilder, getDerivationEnv) -import Hercules.CNix.Store.Context (Derivation) import Hercules.Effect.Container (BindMount (BindMount)) import qualified Hercules.Effect.Container as Container import Hercules.Error (escalateAs) @@ -63,7 +62,7 @@ { data_ = Formats.Secret.data_ secret } -runEffect :: (MonadThrow m, KatipContext m) => ForeignPtr Derivation -> Sensitive Text -> FilePath -> Text -> FilePath -> m ExitCode +runEffect :: (MonadThrow m, KatipContext m) => Derivation -> Sensitive Text -> FilePath -> Text -> FilePath -> m ExitCode runEffect derivation token secretsPath apiBaseURL dir = do drvBuilder <- liftIO $ getDerivationBuilder derivation drvArgs <- liftIO $ getDerivationArguments derivation diff --git a/src/Hercules/Effect/Container.hs b/src/Hercules/Effect/Container.hs --- a/src/Hercules/Effect/Container.hs +++ b/src/Hercules/Effect/Container.hs @@ -15,6 +15,7 @@ import Protolude import System.Directory (createDirectory) import System.FilePath (()) +import System.IO (hClose) import System.IO.Error (ioeGetErrorType) import System.Posix.IO (closeFd, fdToHandle) import System.Posix.Terminal (openPseudoTerminal) @@ -92,27 +93,21 @@ Right a -> pure a Left e -> throwIO (FatalError $ "decoding runc config.json template: " <> show e) let configJson = effectToRuncSpec config template - BS.writeFile (configJsonPath) (BL.toStrict $ encode configJson) + BS.writeFile configJsonPath (BL.toStrict $ encode configJson) createDirectory rootfsPath createDirectory runcRootPath name <- do uuid <- UUID.nextRandom pure $ "hercules-ci-" <> show uuid - (exitCode, _) <- bracket - openPseudoTerminal - ( \(fd1, fd2) -> handle (\(_e :: SomeException) -> pass) do - closeFd fd1 - when (fd2 /= fd1) (closeFd fd2) - ) - $ \(master, terminal) -> do + (exitCode, _) <- withPseudoTerminalHandles $ + \(master, terminal) -> do concurrently ( do - terminalHandle <- fdToHandle terminal let createProcSpec = (System.Process.proc runcExe ["--root", runcRootPath, "run", name]) - { std_in = UseHandle terminalHandle, -- can't pass /dev/null :( - std_out = UseHandle terminalHandle, - std_err = UseHandle terminalHandle, + { std_in = UseHandle terminal, -- can't pass /dev/null :( + std_out = UseHandle terminal, + std_err = UseHandle terminal, cwd = Just dir } withCreateProcess createProcSpec \_subStdin _noOut _noErr processHandle -> do @@ -128,9 +123,8 @@ ) ) ( do - masterHandle <- fdToHandle master let shovel = - handleEOF (BS.hGetLine masterHandle) >>= \case + handleEOF (BS.hGetLine master) >>= \case "" -> pass someBytes | "@nix" `BS.isPrefixOf` someBytes -> do -- TODO use it (example @nix { "action": "setPhase", "phase": "effectPhase" }) @@ -142,3 +136,29 @@ shovel ) pure exitCode + +-- | Like 'openPseudoTerminalHandles' but closes the handles after the +-- function is done. +withPseudoTerminalHandles :: ((Handle, Handle) -> IO a) -> IO a +withPseudoTerminalHandles = + bracket + openPseudoTerminalHandles + ( \(master, terminal) -> do + hClose master `catch` \(_ :: SomeException) -> pass + hClose terminal `catch` \(_ :: SomeException) -> pass + ) + +-- | Like 'openPseudoTerminal' but returning handles, in a resource-safe manner. +openPseudoTerminalHandles :: IO (Handle, Handle) +openPseudoTerminalHandles = + mask_ do + (masterFd, terminalFd) <- openPseudoTerminal + + ( do + master <- fdToHandle masterFd + terminal <- fdToHandle terminalFd + pure (master, terminal) + ) + `onException` do + closeFd masterFd + when (terminalFd /= masterFd) (closeFd terminalFd) diff --git a/test/Hercules/Agent/Nix/RetrieveDerivationInfoSpec.hs b/test/Hercules/Agent/Nix/RetrieveDerivationInfoSpec.hs --- a/test/Hercules/Agent/Nix/RetrieveDerivationInfoSpec.hs +++ b/test/Hercules/Agent/Nix/RetrieveDerivationInfoSpec.hs @@ -1,23 +1,33 @@ +{-# LANGUAGE BlockArguments #-} {-# LANGUAGE OverloadedStrings #-} module Hercules.Agent.Nix.RetrieveDerivationInfoSpec where +import qualified Data.ByteString as BS import qualified Data.Map as M import Hercules.API.Agent.Evaluate.EvaluateEvent.DerivationInfo import Hercules.Agent.Nix.RetrieveDerivationInfo import Hercules.CNix import Protolude +import System.IO.Temp (withSystemTempDirectory) import Test.Hspec spec :: Spec spec = do describe "retrieveDerivationInfo" $ do - it "parses vm-test-run-agent-test.drv correctly" $ do - drv <- getDerivationFromFile "testdata/vm-test-run-agent-test.drv" - d <- retrieveDerivationInfo' "testdata/vm-test-run-agent-test.drv" drv - derivationPath d `shouldBe` "testdata/vm-test-run-agent-test.drv" - platform d `shouldBe` "x86_64-linux" - requiredSystemFeatures d `shouldBe` ["kvm", "nixos-test"] - inputDerivations d `shouldBe` M.fromList [("/nix/store/0si75icim8ajxcsp25d9c52m42kqg1xj-stdenv-linux.drv", ["out"]), ("/nix/store/1kircip4wskspsqqzxbmh6ss73iqh9ah-bash-4.4-p23.drv", ["out"]), ("/nix/store/5kpp7mly0qad7l451xhr60k0wbv6vivi-jquery-1.11.3.drv", ["out"]), ("/nix/store/id49f28qvggmzpmlxp511v6yhc070lkf-jquery-ui-1.11.4.drv", ["out"]), ("/nix/store/kyxayhm3r6gpzbg1d57fwb8m7gkmpwcy-nixos-test-driver-agent-test.drv", ["out"]), ("/nix/store/qzvim3ca1s5zxbbvpzzipizfnfpdf5r2-libxslt-1.1.33.drv", ["dev"])] - inputSources d `shouldBe` ["/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh", "/nix/store/fs6a6m9s5n367dslsvsl9lg89h3ns3ya-logfile.css", "/nix/store/jjh6h82n5rhw45badlpbj1yv7y6m48h2-log2html.xsl", "/nix/store/kdangwwh47ngwxk859ibbvkj4vmm9rzr-treebits.js"] - outputs d `shouldBe` M.fromList [("out", OutputInfo {path = "/nix/store/7vg7ij5933dhg6dh22fvs90bbzmzg8kj-vm-test-run-agent-test", isFixed = False})] + it "parses vm-test-run-agent-test.drv correctly" $ + withTempStore $ \store -> do + s <- BS.readFile "testdata/vm-test-run-agent-test.drv" + drv <- getDerivationFromString store "vm-test-run-agent-test" s + d <- retrieveDerivationInfo' store "testdata/vm-test-run-agent-test.drv" "vm-test-run-agent-test" drv + derivationPath d `shouldBe` "testdata/vm-test-run-agent-test.drv" + platform d `shouldBe` "x86_64-linux" + requiredSystemFeatures d `shouldBe` ["kvm", "nixos-test"] + inputDerivations d `shouldBe` M.fromList [("/nix/store/0si75icim8ajxcsp25d9c52m42kqg1xj-stdenv-linux.drv", ["out"]), ("/nix/store/1kircip4wskspsqqzxbmh6ss73iqh9ah-bash-4.4-p23.drv", ["out"]), ("/nix/store/5kpp7mly0qad7l451xhr60k0wbv6vivi-jquery-1.11.3.drv", ["out"]), ("/nix/store/id49f28qvggmzpmlxp511v6yhc070lkf-jquery-ui-1.11.4.drv", ["out"]), ("/nix/store/kyxayhm3r6gpzbg1d57fwb8m7gkmpwcy-nixos-test-driver-agent-test.drv", ["out"]), ("/nix/store/qzvim3ca1s5zxbbvpzzipizfnfpdf5r2-libxslt-1.1.33.drv", ["dev"])] + inputSources d `shouldBe` ["/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh", "/nix/store/fs6a6m9s5n367dslsvsl9lg89h3ns3ya-logfile.css", "/nix/store/jjh6h82n5rhw45badlpbj1yv7y6m48h2-log2html.xsl", "/nix/store/kdangwwh47ngwxk859ibbvkj4vmm9rzr-treebits.js"] + outputs d `shouldBe` M.fromList [("out", OutputInfo {path = Just "/nix/store/7vg7ij5933dhg6dh22fvs90bbzmzg8kj-vm-test-run-agent-test", isFixed = False})] + +withTempStore :: (Store -> IO a) -> IO a +withTempStore f = + withSystemTempDirectory "cnix-test-store" \d -> + withStoreFromURI (toS d) f