hercules-ci-agent 0.8.1 → 0.8.2

raw patch · 29 files changed

+1248/−745 lines, 29 filesdep ~hercules-ci-api-agentdep ~hercules-ci-api-core

Dependency ranges changed: hercules-ci-api-agent, hercules-ci-api-core

Files

• CHANGELOG.md +10/−0
• cbits/hercules-logger.cxx +9/−5
• cbits/hercules-logger.hh +10/−1
• cbits/hercules-store.cxx +0/−271
• cbits/hercules-store.hh +0/−150
• cbits/nix-2.3/hercules-store.cxx +277/−0
• cbits/nix-2.3/hercules-store.hh +151/−0
• cbits/nix-2.4/hercules-store.cxx +276/−0
• cbits/nix-2.4/hercules-store.hh +154/−0
• hercules-ci-agent-worker/Hercules/Agent/Worker.hs +78/−56
• hercules-ci-agent-worker/Hercules/Agent/Worker/Build.hs +22/−19
• hercules-ci-agent-worker/Hercules/Agent/Worker/Build/Prefetched.hs +25/−18
• hercules-ci-agent-worker/Hercules/Agent/Worker/Effect.hs +10/−10
• hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore.hs +10/−8
• hercules-ci-agent-worker/Hercules/Agent/Worker/HerculesStore/Context.hs +2/−0
• hercules-ci-agent.cabal +25/−5
• hercules-ci-agent/Hercules/Agent.hs +4/−1
• hercules-ci-agent/Hercules/Agent/Build.hs +9/−4
• hercules-ci-agent/Hercules/Agent/Cache.hs +12/−17
• hercules-ci-agent/Hercules/Agent/Cachix.hs +8/−4
• hercules-ci-agent/Hercules/Agent/EnvironmentInfo.hs +38/−79
• hercules-ci-agent/Hercules/Agent/Evaluate.hs +27/−20
• hercules-ci-agent/Hercules/Agent/Nix.hs +1/−28
• hercules-ci-agent/Hercules/Agent/Nix/Init.hs +2/−1
• hercules-ci-agent/Hercules/Agent/Nix/RetrieveDerivationInfo.hs +32/−21
• src/Hercules/Agent/WorkerProtocol/Event.hs +1/−1
• src/Hercules/Effect.hs +2/−3
• src/Hercules/Effect/Container.hs +34/−14
• test/Hercules/Agent/Nix/RetrieveDerivationInfoSpec.hs +19/−9

@@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.2]++### Added++ - Preparations for the next Nix version++### Fixed++ - #304, `message:epollControl: invalid argument (Bad file descriptor)` in effect task+ ## [0.8.1]  ### Added

@@ -1,10 +1,5 @@ #include "hercules-logger.hh" -HerculesLogger::HerculesLogger()-{--}- void HerculesLogger::push(std::unique_ptr<LogEntry> entry) {   auto state(state_.lock());   state->queue.push(std::move(entry));@@ -25,6 +20,15 @@     .text = fs.s   })); }++#ifdef NIX_2_4+// TODO structured+void HerculesLogger::logEI(const nix::ErrorInfo & ei) {+  std::stringstream oss;+  showErrorInfo(oss, ei, false);+  log(ei.level, oss.str());+}+#endif  void HerculesLogger::startActivity(nix::ActivityId act, nix::Verbosity lvl, nix::ActivityType type,     const std::string & s, const Fields & fields, nix::ActivityId parent) {

@@ -1,6 +1,11 @@ #pragma once  #include <nix/config.h>++#ifdef NIX_2_4+#include <nix/error.hh>+#endif+ #include <nix/shared.hh> #include <nix/sync.hh> #include <nix/logging.hh>@@ -27,7 +32,7 @@   uint64_t getMs();   public:-  HerculesLogger();+  inline HerculesLogger() {};    struct LogEntry {     int entryType;@@ -45,6 +50,10 @@       const std::string & s, const Fields & fields, nix::ActivityId parent) override;   void stopActivity(nix::ActivityId act) override;   void result(nix::ActivityId act, nix::ResultType type, const Fields & fields) override;++#ifdef NIX_2_4+  void logEI(const nix::ErrorInfo &ei) override;+#endif    std::unique_ptr<LogEntry> pop();   void popMany(int max, std::queue<std::unique_ptr<LogEntry>> &out);

@@ -1,271 +0,0 @@-#include <cstdio>-#include <cstring>-#include <math.h>-#include <nix/config.h>-#include <nix/shared.hh>-#include <nix/store-api.hh>-#include <nix/common-eval-args.hh>-#include <nix/get-drvs.hh>-#include <nix/derivations.hh>-#include <nix/affinity.hh>-#include <nix/globals.hh>--#include "hercules-store.hh"--using namespace nix;--WrappingStore::WrappingStore(const Params& params, ref<Store> storeToWrap)-    : Store(params), wrappedStore(storeToWrap) {}--WrappingStore::~WrappingStore() {}--std::string WrappingStore::getUri() {-  return "wrapped:" + wrappedStore->getUri();-};--bool WrappingStore::isValidPathUncached(const Path& path) {-  return wrappedStore->isValidPath(path);  // not ideal-}--PathSet WrappingStore::queryValidPaths(const PathSet& paths,-                                       SubstituteFlag maybeSubstitute) {-  return wrappedStore->queryValidPaths(paths, maybeSubstitute);-}-PathSet WrappingStore::queryAllValidPaths() {-  return wrappedStore->queryAllValidPaths();-}--// protected:-void WrappingStore::queryPathInfoUncached(-    const Path& path,-    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {-  unsupported("queryPathInfoUncached");-  /*-      Callback<ref<ValidPathInfo>>-     callback2([&callback](std::future<ref<ValidPathInfo>> vpi){-        std::shared_ptr<ValidPathInfo> !@#$%^&*-        callback(vpi);-      });-      return wrappedStore->queryPathInfo(path, callback2); // not ideal-  */-}--// public:--void WrappingStore::queryReferrers(const Path& path, PathSet& referrers) {-  wrappedStore->queryReferrers(path, referrers);-}--PathSet WrappingStore::queryValidDerivers(const Path& path) {-  return wrappedStore->queryValidDerivers(path);-}--PathSet WrappingStore::queryDerivationOutputs(const Path& path) {-  return wrappedStore->queryDerivationOutputs(path);-}--StringSet WrappingStore::queryDerivationOutputNames(const Path& path) {-  return wrappedStore->queryDerivationOutputNames(path);-}--Path WrappingStore::queryPathFromHashPart(const string& hashPart) {-  return wrappedStore->queryPathFromHashPart(hashPart);-}--PathSet WrappingStore::querySubstitutablePaths(const PathSet& paths) {-  return wrappedStore->querySubstitutablePaths(paths);-}--void WrappingStore::querySubstitutablePathInfos(const PathSet& paths,-                                                SubstitutablePathInfos& infos) {-  wrappedStore->querySubstitutablePathInfos(paths, infos);-}--bool WrappingStore::wantMassQuery() {-  return wrappedStore->wantMassQuery();-}--void WrappingStore::addToStore(const ValidPathInfo& info,-                               Source& narSource,-                               RepairFlag repair,-                               CheckSigsFlag checkSigs,-                               std::shared_ptr<FSAccessor> accessor) {-  wrappedStore->addToStore(info, narSource, repair, checkSigs, accessor);-}--void WrappingStore::addToStore(const ValidPathInfo& info,-                               const ref<std::string>& nar,-                               RepairFlag repair,-                               CheckSigsFlag checkSigs,-                               std::shared_ptr<FSAccessor> accessor) {-  wrappedStore->addToStore(info, nar, repair, checkSigs, accessor);-}--Path WrappingStore::addToStore(const string& name,-                               const Path& srcPath,-                               bool recursive,-                               HashType hashAlgo,-                               PathFilter& filter,-                               RepairFlag repair) {-  return wrappedStore->addToStore(name, srcPath, recursive, hashAlgo, filter,-                                  repair);-}--Path WrappingStore::addTextToStore(const string& name,-                                   const string& s,-                                   const PathSet& references,-                                   RepairFlag repair) {-  return wrappedStore->addTextToStore(name, s, references, repair);-}--void WrappingStore::narFromPath(const Path& path, Sink& sink) {-  wrappedStore->narFromPath(path, sink);-}--void WrappingStore::buildPaths(const PathSet& paths, BuildMode buildMode) {-  wrappedStore->buildPaths(paths, buildMode);-}--BuildResult WrappingStore::buildDerivation(const Path& drvPath,-                                           const BasicDerivation& drv,-                                           BuildMode buildMode) {-  return wrappedStore->buildDerivation(drvPath, drv, buildMode);-}--void WrappingStore::ensurePath(const Path& path) {-  wrappedStore->ensurePath(path);-}--void WrappingStore::addTempRoot(const Path& path) {-  wrappedStore->addTempRoot(path);-}--void WrappingStore::addIndirectRoot(const Path& path) {-  wrappedStore->addIndirectRoot(path);-}--void WrappingStore::syncWithGC() {-  wrappedStore->syncWithGC();-}--void WrappingStore::collectGarbage(const GCOptions& options,-                                   GCResults& results) {-  wrappedStore->collectGarbage(options, results);-}--void WrappingStore::optimiseStore() {-  wrappedStore->optimiseStore();-};--bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) {-  return wrappedStore->verifyStore(checkContents, repair);-};--ref<FSAccessor> WrappingStore::getFSAccessor() {-  return wrappedStore->getFSAccessor();-}--void WrappingStore::addSignatures(const Path& storePath,-                                  const StringSet& sigs) {-  wrappedStore->addSignatures(storePath, sigs);-};--void WrappingStore::computeFSClosure(const PathSet& paths,-                                     PathSet& out,-                                     bool flipDirection,-                                     bool includeOutputs,-                                     bool includeDerivers) {-  wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs,-                                 includeDerivers);-}--void WrappingStore::queryMissing(const PathSet& targets,-                                 PathSet& willBuild,-                                 PathSet& willSubstitute,-                                 PathSet& unknown,-                                 unsigned long long& downloadSize,-                                 unsigned long long& narSize) {-  wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown,-                             downloadSize, narSize);-}--std::shared_ptr<std::string> WrappingStore::getBuildLog(const Path& path) {-  return wrappedStore->getBuildLog(path);-}--void WrappingStore::connect() {-  wrappedStore->connect();-};--int WrappingStore::getPriority() {-  return wrappedStore->getPriority();-}--Path WrappingStore::toRealPath(const Path& storePath) {-  return wrappedStore->toRealPath(storePath);-};--/////--HerculesStore::HerculesStore(const Params& params, ref<Store> storeToWrap)-    : WrappingStore(params, storeToWrap) {}--void HerculesStore::ensurePath(const Path& path) {-  /* We avoid asking substituters for paths, since-     those would yield negative pathInfo caches on remote store.--     Instead, we only assert if path exists in the store.--     Once IFD build is performed, we ask for substitution-     via ensurePath.-  */-  if (!wrappedStore->isValidPath(path)) {-    std::exception_ptr exceptionToThrow(nullptr);-    builderCallback(strdup(path.c_str()), &exceptionToThrow);-    if (exceptionToThrow != nullptr) {-      std::rethrow_exception(exceptionToThrow);-    }-    wrappedStore->ensurePath(path);-  }-  ensuredPaths.insert(path);-};--// Avoid substituting in evaluator, see `ensurePath` for more details-void HerculesStore::queryMissing(const PathSet& targets,-                                 PathSet& willBuild,-                                 PathSet& willSubstitute,-                                 PathSet& unknown,-                                 unsigned long long& downloadSize,-                                 unsigned long long& narSize) {-};--void HerculesStore::buildPaths(const PathSet& paths, BuildMode buildMode) {-  for (Path path : paths) {-    std::exception_ptr exceptionToThrow(nullptr);-    builderCallback(strdup(path.c_str()), &exceptionToThrow);-    if (exceptionToThrow != nullptr) {-      std::rethrow_exception(exceptionToThrow);-    }-  }-}--BuildResult HerculesStore::buildDerivation(const Path& drvPath,-                                           const BasicDerivation& drv,-                                           BuildMode buildMode) {-  unsupported("buildDerivation");--  std::cerr << "building derivation " << drvPath << std::endl;-  auto r = wrappedStore->buildDerivation(drvPath, drv, buildMode);-  std::cerr << "built derivation " << drvPath << std::endl;-  return r;-}--void HerculesStore::printDiagnostics() {-  for (std::string path : ensuredPaths) {-    std::cerr << path << std::endl;-  }-}--void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(const char *, std::exception_ptr *exceptionToThrow)) {-  builderCallback = newBuilderCallback;-}

@@ -1,150 +0,0 @@-#include <cstdio>-#include <cstring>-#include <math.h>-#include <nix/config.h>-#include <nix/shared.hh>-#include <nix/store-api.hh>-#include <nix/common-eval-args.hh>-#include <nix/get-drvs.hh>-#include <nix/derivations.hh>-#include <nix/affinity.hh>-#include <nix/globals.hh>-#include "HsFFI.h"--using namespace nix;--class WrappingStore : public Store {- public:-  ref<Store> wrappedStore;--  WrappingStore(const Params & params, ref<Store> storeToWrap);---  virtual ~WrappingStore();--  virtual std::string getUri() override;--protected:--  virtual bool isValidPathUncached(const Path & path) override;--public:--  virtual PathSet queryValidPaths(const PathSet & paths,-      SubstituteFlag maybeSubstitute = NoSubstitute) override;-  virtual PathSet queryAllValidPaths() override;--protected:-  virtual void queryPathInfoUncached(const Path & path,-        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;--public:--  virtual void queryReferrers(const Path & path,-      PathSet & referrers) override;--  virtual PathSet queryValidDerivers(const Path & path) override;--  virtual PathSet queryDerivationOutputs(const Path & path) override;--  virtual StringSet queryDerivationOutputNames(const Path & path) override;--  virtual Path queryPathFromHashPart(const string & hashPart) override;--  virtual PathSet querySubstitutablePaths(const PathSet & paths) override;--  virtual void querySubstitutablePathInfos(const PathSet & paths,-        SubstitutablePathInfos & infos) override;--  virtual bool wantMassQuery() override;--  virtual void addToStore(const ValidPathInfo & info, Source & narSource,-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,-        std::shared_ptr<FSAccessor> accessor = 0) override;--  virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,-        std::shared_ptr<FSAccessor> accessor = 0) override;--  virtual Path addToStore(const string & name, const Path & srcPath,-        bool recursive = true, HashType hashAlgo = htSHA256,-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;--  virtual Path addTextToStore(const string & name, const string & s,-        const PathSet & references, RepairFlag repair = NoRepair) override;--  virtual void narFromPath(const Path & path, Sink & sink) override;--  virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;--  virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,-        BuildMode buildMode = bmNormal) override;--  virtual void ensurePath(const Path & path) override;--  virtual void addTempRoot(const Path & path) override;--  virtual void addIndirectRoot(const Path & path) override;--  virtual void syncWithGC() override;--  virtual void collectGarbage(const GCOptions & options, GCResults & results) override;--  virtual void optimiseStore() override;--  virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) override;--  virtual ref<FSAccessor> getFSAccessor() override;--  virtual void addSignatures(const Path & storePath, const StringSet & sigs) override;--    -  virtual void computeFSClosure(const PathSet & paths,-      PathSet & out, bool flipDirection = false,-      bool includeOutputs = false, bool includeDerivers = false) override;--  virtual void queryMissing(const PathSet & targets,-        PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,-        unsigned long long & downloadSize, unsigned long long & narSize) override;--  virtual std::shared_ptr<std::string> getBuildLog(const Path & path) override;--  virtual void connect() override;--  virtual int getPriority() override;--  virtual Path toRealPath(const Path & storePath) override;-};--class HerculesStore : public WrappingStore {-public:-  PathSet ensuredPaths;-  void (* builderCallback)(const char *, std::exception_ptr *exceptionToThrow);--  HerculesStore(const Params & params, ref<Store> storeToWrap);--  // Overrides--  virtual void ensurePath(const Path & path) override;--  virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;--  virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,-        BuildMode buildMode = bmNormal) override;--  virtual void queryMissing(const PathSet& targets,-                            PathSet& willBuild,-                            PathSet& willSubstitute,-                            PathSet& unknown,-                            unsigned long long& downloadSize,-                            unsigned long long& narSize) override;--  // Additions--  void printDiagnostics();--  void setBuilderCallback(void (* newBuilderCallback)(const char *, std::exception_ptr *exceptionToThrow));--  void inhibitBuilds();-  void uninhibitBuilds();-};

@@ -0,0 +1,277 @@+#include <cstdio>+#include <cstring>+#include <math.h>+#include <nix/config.h>+#include <nix/shared.hh>+#include <nix/store-api.hh>+#include <nix/common-eval-args.hh>+#include <nix/get-drvs.hh>+#include <nix/derivations.hh>+#include <nix/affinity.hh>+#include <nix/globals.hh>++#include "hercules-store.hh"++using namespace nix;++WrappingStore::WrappingStore(const Params& params, ref<Store> storeToWrap)+    : Store(params), wrappedStore(storeToWrap) {}++WrappingStore::~WrappingStore() {}++std::string WrappingStore::getUri() {+  return "wrapped:" + wrappedStore->getUri();+};++bool WrappingStore::isValidPathUncached(const Path& path) {+  return wrappedStore->isValidPath(path);  // not ideal+}++PathSet WrappingStore::queryValidPaths(const PathSet& paths,+                                       SubstituteFlag maybeSubstitute) {+  return wrappedStore->queryValidPaths(paths, maybeSubstitute);+}+PathSet WrappingStore::queryAllValidPaths() {+  return wrappedStore->queryAllValidPaths();+}++// protected:+void WrappingStore::queryPathInfoUncached(+    const Path& path,+    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {+  unsupported("queryPathInfoUncached");+  /*+      Callback<ref<ValidPathInfo>>+     callback2([&callback](std::future<ref<ValidPathInfo>> vpi){+        std::shared_ptr<ValidPathInfo> !@#$%^&*+        callback(vpi);+      });+      return wrappedStore->queryPathInfo(path, callback2); // not ideal+  */+}++// public:++void WrappingStore::queryReferrers(const Path& path, PathSet& referrers) {+  wrappedStore->queryReferrers(path, referrers);+}++PathSet WrappingStore::queryValidDerivers(const Path& path) {+  return wrappedStore->queryValidDerivers(path);+}++PathSet WrappingStore::queryDerivationOutputs(const Path& path) {+  return wrappedStore->queryDerivationOutputs(path);+}++StringSet WrappingStore::queryDerivationOutputNames(const Path& path) {+  return wrappedStore->queryDerivationOutputNames(path);+}++Path WrappingStore::queryPathFromHashPart(const string& hashPart) {+  return wrappedStore->queryPathFromHashPart(hashPart);+}++PathSet WrappingStore::querySubstitutablePaths(const PathSet& paths) {+  return wrappedStore->querySubstitutablePaths(paths);+}++void WrappingStore::querySubstitutablePathInfos(const PathSet& paths,+                                                SubstitutablePathInfos& infos) {+  wrappedStore->querySubstitutablePathInfos(paths, infos);+}++bool WrappingStore::wantMassQuery() {+  return wrappedStore->wantMassQuery();+}++void WrappingStore::addToStore(const ValidPathInfo& info,+                               Source& narSource,+                               RepairFlag repair,+                               CheckSigsFlag checkSigs,+                               std::shared_ptr<FSAccessor> accessor) {+  wrappedStore->addToStore(info, narSource, repair, checkSigs, accessor);+}++void WrappingStore::addToStore(const ValidPathInfo& info,+                               const ref<std::string>& nar,+                               RepairFlag repair,+                               CheckSigsFlag checkSigs,+                               std::shared_ptr<FSAccessor> accessor) {+  wrappedStore->addToStore(info, nar, repair, checkSigs, accessor);+}++Path WrappingStore::addToStore(const string& name,+                               const Path& srcPath,+                               bool recursive,+                               HashType hashAlgo,+                               PathFilter& filter,+                               RepairFlag repair) {+  return wrappedStore->addToStore(name, srcPath, recursive, hashAlgo, filter,+                                  repair);+}++Path WrappingStore::addTextToStore(const string& name,+                                   const string& s,+                                   const PathSet& references,+                                   RepairFlag repair) {+  return wrappedStore->addTextToStore(name, s, references, repair);+}++void WrappingStore::narFromPath(const Path& path, Sink& sink) {+  wrappedStore->narFromPath(path, sink);+}++void WrappingStore::buildPaths(const PathSet& paths, BuildMode buildMode) {+  wrappedStore->buildPaths(paths, buildMode);+}++BuildResult WrappingStore::buildDerivation(const Path& drvPath,+                                           const BasicDerivation& drv,+                                           BuildMode buildMode) {+  return wrappedStore->buildDerivation(drvPath, drv, buildMode);+}++void WrappingStore::ensurePath(const Path& path) {+  wrappedStore->ensurePath(path);+}++void WrappingStore::addTempRoot(const Path& path) {+  wrappedStore->addTempRoot(path);+}++void WrappingStore::addIndirectRoot(const Path& path) {+  wrappedStore->addIndirectRoot(path);+}++void WrappingStore::syncWithGC() {+  wrappedStore->syncWithGC();+}++void WrappingStore::collectGarbage(const GCOptions& options,+                                   GCResults& results) {+  wrappedStore->collectGarbage(options, results);+}++void WrappingStore::optimiseStore() {+  wrappedStore->optimiseStore();+};++bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) {+  return wrappedStore->verifyStore(checkContents, repair);+};++ref<FSAccessor> WrappingStore::getFSAccessor() {+  return wrappedStore->getFSAccessor();+}++void WrappingStore::addSignatures(const Path& storePath,+                                  const StringSet& sigs) {+  wrappedStore->addSignatures(storePath, sigs);+};++void WrappingStore::computeFSClosure(const PathSet& paths,+                                     PathSet& out,+                                     bool flipDirection,+                                     bool includeOutputs,+                                     bool includeDerivers) {+  wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs,+                                 includeDerivers);+}++void WrappingStore::queryMissing(const PathSet& targets,+                                 PathSet& willBuild,+                                 PathSet& willSubstitute,+                                 PathSet& unknown,+                                 unsigned long long& downloadSize,+                                 unsigned long long& narSize) {+  wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown,+                             downloadSize, narSize);+}++std::shared_ptr<std::string> WrappingStore::getBuildLog(const Path& path) {+  return wrappedStore->getBuildLog(path);+}++void WrappingStore::connect() {+  wrappedStore->connect();+};++int WrappingStore::getPriority() {+  return wrappedStore->getPriority();+}++Path WrappingStore::toRealPath(const Path& storePath) {+  return wrappedStore->toRealPath(storePath);+};++/////++HerculesStore::HerculesStore(const Params& params, ref<Store> storeToWrap)+    : WrappingStore(params, storeToWrap) {}++void HerculesStore::ensurePath(const Path& path) {+  /* We avoid asking substituters for paths, since+     those would yield negative pathInfo caches on remote store.++     Instead, we only assert if path exists in the store.++     Once IFD build is performed, we ask for substitution+     via ensurePath.+  */+  if (!wrappedStore->isValidPath(path)) {+    std::exception_ptr exceptionToThrow(nullptr);+    auto pathWO = nix::parsePathWithOutputs(*this, path);+    auto pathsPtr = new std::vector<StorePathWithOutputs>{pathWO};+    // TODO place outside the loop+    builderCallback(pathsPtr, &exceptionToThrow);+    if (exceptionToThrow != nullptr) {+      std::rethrow_exception(exceptionToThrow);+    }+    wrappedStore->ensurePath(path);+  }+  ensuredPaths.insert(path);+};++// Avoid substituting in evaluator, see `ensurePath` for more details+void HerculesStore::queryMissing(const PathSet& targets,+                                 PathSet& willBuild,+                                 PathSet& willSubstitute,+                                 PathSet& unknown,+                                 unsigned long long& downloadSize,+                                 unsigned long long& narSize) {+};++void HerculesStore::buildPaths(const PathSet& paths, BuildMode buildMode) {+  for (Path path : paths) {+    std::exception_ptr exceptionToThrow(nullptr);+    auto pathWO = nix::parsePathWithOutputs(*this, path);+    auto pathsPtr = new std::vector<StorePathWithOutputs>{pathWO};+    // TODO place outside the loop+    builderCallback(pathsPtr, &exceptionToThrow);+    if (exceptionToThrow != nullptr) {+      std::rethrow_exception(exceptionToThrow);+    }+  }+}++BuildResult HerculesStore::buildDerivation(const Path& drvPath,+                                           const BasicDerivation& drv,+                                           BuildMode buildMode) {+  unsupported("buildDerivation");++  std::cerr << "building derivation " << drvPath << std::endl;+  auto r = wrappedStore->buildDerivation(drvPath, drv, buildMode);+  std::cerr << "built derivation " << drvPath << std::endl;+  return r;+}++void HerculesStore::printDiagnostics() {+  for (std::string path : ensuredPaths) {+    std::cerr << path << std::endl;+  }+}++void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(std::vector<nix::StorePathWithOutputs>*, std::exception_ptr *exceptionToThrow)) {+  builderCallback = newBuilderCallback;+}

@@ -0,0 +1,151 @@+#include <cstdio>+#include <cstring>+#include <math.h>+#include <nix/config.h>+#include <nix/shared.hh>+#include <nix/store-api.hh>+#include <nix/common-eval-args.hh>+#include <nix/get-drvs.hh>+#include <nix/derivations.hh>+#include <nix/affinity.hh>+#include <nix/globals.hh>+#include <nix-compat.hh>+#include "HsFFI.h"++using namespace nix;++class WrappingStore : public Store {+ public:+  ref<Store> wrappedStore;++  WrappingStore(const Params & params, ref<Store> storeToWrap);+++  virtual ~WrappingStore();++  virtual std::string getUri() override;++protected:++  virtual bool isValidPathUncached(const Path & path) override;++public:++  virtual PathSet queryValidPaths(const PathSet & paths,+      SubstituteFlag maybeSubstitute = NoSubstitute) override;+  virtual PathSet queryAllValidPaths() override;++protected:+  virtual void queryPathInfoUncached(const Path & path,+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;++public:++  virtual void queryReferrers(const Path & path,+      PathSet & referrers) override;++  virtual PathSet queryValidDerivers(const Path & path) override;++  virtual PathSet queryDerivationOutputs(const Path & path) override;++  virtual StringSet queryDerivationOutputNames(const Path & path) override;++  virtual Path queryPathFromHashPart(const string & hashPart) override;++  virtual PathSet querySubstitutablePaths(const PathSet & paths) override;++  virtual void querySubstitutablePathInfos(const PathSet & paths,+        SubstitutablePathInfos & infos) override;++  virtual bool wantMassQuery() override;++  virtual void addToStore(const ValidPathInfo & info, Source & narSource,+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,+        std::shared_ptr<FSAccessor> accessor = 0) override;++  virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,+        std::shared_ptr<FSAccessor> accessor = 0) override;++  virtual Path addToStore(const string & name, const Path & srcPath,+        bool recursive = true, HashType hashAlgo = htSHA256,+        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;++  virtual Path addTextToStore(const string & name, const string & s,+        const PathSet & references, RepairFlag repair = NoRepair) override;++  virtual void narFromPath(const Path & path, Sink & sink) override;++  virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;++  virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,+        BuildMode buildMode = bmNormal) override;++  virtual void ensurePath(const Path & path) override;++  virtual void addTempRoot(const Path & path) override;++  virtual void addIndirectRoot(const Path & path) override;++  virtual void syncWithGC() override;++  virtual void collectGarbage(const GCOptions & options, GCResults & results) override;++  virtual void optimiseStore() override;++  virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) override;++  virtual ref<FSAccessor> getFSAccessor() override;++  virtual void addSignatures(const Path & storePath, const StringSet & sigs) override;++    +  virtual void computeFSClosure(const PathSet & paths,+      PathSet & out, bool flipDirection = false,+      bool includeOutputs = false, bool includeDerivers = false) override;++  virtual void queryMissing(const PathSet & targets,+        PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,+        unsigned long long & downloadSize, unsigned long long & narSize) override;++  virtual std::shared_ptr<std::string> getBuildLog(const Path & path) override;++  virtual void connect() override;++  virtual int getPriority() override;++  virtual Path toRealPath(const Path & storePath) override;+};++class HerculesStore : public WrappingStore {+public:+  PathSet ensuredPaths;+  void (* builderCallback)(std::vector<nix::StorePathWithOutputs>*, std::exception_ptr *exceptionToThrow);++  HerculesStore(const Params & params, ref<Store> storeToWrap);++  // Overrides++  virtual void ensurePath(const Path & path) override;++  virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;++  virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,+        BuildMode buildMode = bmNormal) override;++  virtual void queryMissing(const PathSet& targets,+                            PathSet& willBuild,+                            PathSet& willSubstitute,+                            PathSet& unknown,+                            unsigned long long& downloadSize,+                            unsigned long long& narSize) override;++  // Additions++  void printDiagnostics();++  void setBuilderCallback(void (* newBuilderCallback)(std::vector<nix::StorePathWithOutputs>*, std::exception_ptr *exceptionToThrow));++  void inhibitBuilds();+  void uninhibitBuilds();+};

@@ -0,0 +1,276 @@+#include <cstdio>+#include <cstring>+#include <memory>+#include <math.h>+#include <nix/config.h>+#include <nix/shared.hh>+#include <nix/store-api.hh>+#include <nix/common-eval-args.hh>+#include <nix/get-drvs.hh>+#include <nix/derivations.hh>+#include <nix/affinity.hh>+#include <nix/globals.hh>+#include <nix/callback.hh>++#include "hercules-store.hh"++using namespace nix;++WrappingStore::WrappingStore(const Params& params, ref<Store> storeToWrap)+    : Store(params), wrappedStore(storeToWrap) {}++WrappingStore::~WrappingStore() {}++std::string WrappingStore::getUri() {+  return "wrapped:" + wrappedStore->getUri();+};++bool WrappingStore::isValidPathUncached(const StorePath & path) {+  return wrappedStore->isValidPath(path);  // caches again. Not much we can do.+}++StorePathSet WrappingStore::queryValidPaths(const StorePathSet& paths,+                                       SubstituteFlag maybeSubstitute) {+  return wrappedStore->queryValidPaths(paths, maybeSubstitute);+}+StorePathSet WrappingStore::queryAllValidPaths() {+  return wrappedStore->queryAllValidPaths();+}++// protected:+void WrappingStore::queryPathInfoUncached(const StorePath & path,+      Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept {++  auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));++  wrappedStore->queryPathInfo(path, {[=](std::future<ref<const ValidPathInfo>> vpi){+    (*callbackPtr)(vpi.get().get_ptr());+  }});+}++// public:++void WrappingStore::queryReferrers(const StorePath& path, StorePathSet& referrers) {+  wrappedStore->queryReferrers(path, referrers);+}++StorePathSet WrappingStore::queryValidDerivers(const StorePath& path) {+  return wrappedStore->queryValidDerivers(path);+}++StorePathSet WrappingStore::queryDerivationOutputs(const StorePath& path) {+  return wrappedStore->queryDerivationOutputs(path);+}++std::optional<StorePath> WrappingStore::queryPathFromHashPart(const std::string & hashPart) {+  return wrappedStore->queryPathFromHashPart(hashPart);+}++StorePathSet WrappingStore::querySubstitutablePaths(const StorePathSet& paths) {+  return wrappedStore->querySubstitutablePaths(paths);+}++void WrappingStore::querySubstitutablePathInfos(const StorePathCAMap & paths,+      SubstitutablePathInfos & infos) {+  wrappedStore->querySubstitutablePathInfos(paths, infos);+}++void WrappingStore::addToStore(const ValidPathInfo & info, Source & narSource,+    RepairFlag repair, CheckSigsFlag checkSigs) {+  wrappedStore->addToStore(info, narSource, repair, checkSigs);+}++StorePath WrappingStore::addToStore(const string & name, const Path & srcPath,+      FileIngestionMethod method, HashType hashAlgo,+      PathFilter & filter, RepairFlag repair) {+  return wrappedStore->addToStore(name, srcPath, method, hashAlgo, filter, repair);+}++StorePath WrappingStore::addToStoreFromDump(Source & dump, const string & name,+      FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) {+  return wrappedStore->addToStoreFromDump(dump, name, method, hashAlgo, repair);+}++StorePath WrappingStore::addTextToStore(const string & name, const string & s,+      const StorePathSet & references, RepairFlag repair) {+  return wrappedStore->addTextToStore(name, s, references, repair);+}++void WrappingStore::narFromPath(const StorePath& path, Sink& sink) {+  wrappedStore->narFromPath(path, sink);+}++void WrappingStore::buildPaths(+      const std::vector<DerivedPath> & paths, BuildMode buildMode) {+  wrappedStore->buildPaths(paths, buildMode);+}++BuildResult WrappingStore::buildDerivation(const StorePath& drvPath,+                                           const BasicDerivation& drv,+                                           BuildMode buildMode) {+  return wrappedStore->buildDerivation(drvPath, drv, buildMode);+}++void WrappingStore::ensurePath(const StorePath& path) {+  wrappedStore->ensurePath(path);+}++void WrappingStore::addTempRoot(const StorePath& path) {+  wrappedStore->addTempRoot(path);+}++void WrappingStore::addIndirectRoot(const Path& path) {+  wrappedStore->addIndirectRoot(path);+}++void WrappingStore::syncWithGC() {+  wrappedStore->syncWithGC();+}++void WrappingStore::collectGarbage(const GCOptions& options,+                                   GCResults& results) {+  wrappedStore->collectGarbage(options, results);+}++void WrappingStore::optimiseStore() {+  wrappedStore->optimiseStore();+};++bool WrappingStore::verifyStore(bool checkContents, RepairFlag repair) {+  return wrappedStore->verifyStore(checkContents, repair);+};++ref<FSAccessor> WrappingStore::getFSAccessor() {+  return wrappedStore->getFSAccessor();+}++void WrappingStore::addSignatures(const StorePath& storePath,+                                  const StringSet& sigs) {+  wrappedStore->addSignatures(storePath, sigs);+};++void WrappingStore::computeFSClosure(const StorePathSet& paths,+                                     StorePathSet& out,+                                     bool flipDirection,+                                     bool includeOutputs,+                                     bool includeDerivers) {+  wrappedStore->computeFSClosure(paths, out, flipDirection, includeOutputs,+                                 includeDerivers);+}++void WrappingStore::queryMissing(const std::vector<DerivedPath> & targets,+      StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,+      uint64_t & downloadSize, uint64_t & narSize) {+  wrappedStore->queryMissing(targets, willBuild, willSubstitute, unknown,+                             downloadSize, narSize);+}++std::shared_ptr<std::string> WrappingStore::getBuildLog(const StorePath& path) {+  return wrappedStore->getBuildLog(path);+}++void WrappingStore::connect() {+  wrappedStore->connect();+};++Path WrappingStore::toRealPath(const Path& storePath) {+  return wrappedStore->toRealPath(storePath);+};++Roots WrappingStore::findRoots(bool censor) {+  return wrappedStore->findRoots(censor);+}++unsigned int WrappingStore::getProtocol() {+  return wrappedStore->getProtocol();+}++void WrappingStore::createUser(const std::string & userName, uid_t userId) {+  wrappedStore->createUser(userName, userId);+}++/////++HerculesStore::HerculesStore(const Params& params, ref<Store> storeToWrap)+    : StoreConfig(params)+    , WrappingStore(params, storeToWrap) {}++const std::string HerculesStore::name() {+  return "wrapped " + wrappedStore->name();+}++std::optional<const Realisation> HerculesStore::queryRealisation(const DrvOutput &drvOut) {+  // TODO (ca-derivations) use?+  return wrappedStore->queryRealisation(drvOut);+}++void HerculesStore::ensurePath(const StorePath& path) {+  /* We avoid asking substituters for paths, since+     those would yield negative pathInfo caches on remote store.++     Instead, we only assert if path exists in the store.++     Once IFD build is performed, we ask for substitution+     via ensurePath.+  */+  if (!wrappedStore->isValidPath(path)) {+    std::exception_ptr exceptionToThrow(nullptr);+    // FIXME probably need this+    // builderCallback(path, &exceptionToThrow);+    if (exceptionToThrow != nullptr) {+      std::rethrow_exception(exceptionToThrow);+    }+    wrappedStore->ensurePath(path);+  }+  ensuredPaths.insert(path);+};++// Avoid substituting in evaluator, see `ensurePath` for more details+void HerculesStore::queryMissing(const std::vector<DerivedPath> & targets,+      StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,+      uint64_t & downloadSize, uint64_t & narSize) {+};++void HerculesStore::buildPaths(const std::vector<DerivedPath> & derivedPaths, BuildMode buildMode) {+  std::exception_ptr exceptionToThrow(nullptr);++  // responsibility for delete is transferred to builderCallback+  auto pathsPtr = new std::vector<StorePathWithOutputs>();+  std::vector<StorePathWithOutputs> &paths = *pathsPtr;++  // TODO: don't ignore the Opaques+  for (auto & derivedPath : derivedPaths) {+    std::visit(overloaded {+      [&](DerivedPath::Built built) {+          paths.push_back(StorePathWithOutputs { built.drvPath, built.outputs });+      },+      [&](DerivedPath::Opaque opaque) {+          // TODO: pass this to the callback as well+      },+    }, derivedPath.raw());+  }++  builderCallback(pathsPtr, &exceptionToThrow);++  if (exceptionToThrow != nullptr) {+    std::rethrow_exception(exceptionToThrow);+  }+}++BuildResult HerculesStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,+    BuildMode buildMode) {++  unsupported("buildDerivation");+  // unreachable+  return wrappedStore->buildDerivation(drvPath, drv, buildMode);+}++void HerculesStore::printDiagnostics() {+  for (auto path : ensuredPaths) {+    std::cerr << path.to_string() << std::endl;+  }+}++void HerculesStore::setBuilderCallback(void (* newBuilderCallback)(std::vector<nix::StorePathWithOutputs>*, std::exception_ptr *exceptionToThrow)) {+  builderCallback = newBuilderCallback;+}

@@ -0,0 +1,154 @@+#include <cstdio>+#include <cstring>+#include <math.h>+#include <nix/config.h>+#include <nix/shared.hh>+#include <nix/store-api.hh>+#include <nix/common-eval-args.hh>+#include <nix/get-drvs.hh>+#include <nix/derivations.hh>+#include <nix/affinity.hh>+#include <nix/globals.hh>+#include <nix-compat.hh>+#include "HsFFI.h"++using namespace nix;++class WrappingStore : public Store {+ public:+  ref<Store> wrappedStore;++  WrappingStore(const Params & params, ref<Store> storeToWrap);+++  virtual ~WrappingStore();++  virtual std::string getUri() override;++protected:++  virtual bool isValidPathUncached(const StorePath & path) override;++public:++  virtual StorePathSet queryValidPaths(const StorePathSet & paths,+      SubstituteFlag maybeSubstitute = NoSubstitute) override;+  virtual StorePathSet queryAllValidPaths() override;++protected:+  virtual void queryPathInfoUncached(const StorePath & path,+      Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;++public:++  virtual void queryReferrers(const StorePath & path,+      StorePathSet & referrers) override;++  virtual StorePathSet queryValidDerivers(const StorePath & path) override;++  virtual StorePathSet queryDerivationOutputs(const StorePath & path) override;++  virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;++  virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;++  virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,+      SubstitutablePathInfos & infos) override;++  virtual void addToStore(const ValidPathInfo & info, Source & narSource,+      RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override;++  virtual StorePath addToStore(const string & name, const Path & srcPath,+      FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,+      PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;++  virtual StorePath addToStoreFromDump(Source & dump, const string & name,+      FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;++  virtual StorePath addTextToStore(const string & name, const string & s,+      const StorePathSet & references, RepairFlag repair = NoRepair) override;++  virtual void narFromPath(const StorePath & path, Sink & sink) override;++  virtual void buildPaths(+      const std::vector<DerivedPath> & paths,+      BuildMode buildMode = bmNormal) override;++  virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,+      BuildMode buildMode = bmNormal) override;++  virtual void ensurePath(const StorePath & path) override;++  virtual void addTempRoot(const StorePath & path) override;++  virtual void addIndirectRoot(const Path & path) override;++  virtual void syncWithGC() override;++  virtual Roots findRoots(bool censor) override;++  virtual void collectGarbage(const GCOptions & options, GCResults & results) override;++  virtual void optimiseStore() override;++  virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) override;++  virtual ref<FSAccessor> getFSAccessor() override;++  virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) override;++  virtual void computeFSClosure(const StorePathSet & paths,+      StorePathSet & out, bool flipDirection = false,+      bool includeOutputs = false, bool includeDerivers = false) override;++  virtual void queryMissing(const std::vector<DerivedPath> & targets,+      StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,+      uint64_t & downloadSize, uint64_t & narSize) override;++  virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;++  virtual unsigned int getProtocol() override;++  virtual void connect() override;++  virtual Path toRealPath(const Path & storePath) override;++  virtual void createUser(const std::string & userName, uid_t userId) override;++};++class HerculesStore : public WrappingStore {+public:+  StorePathSet ensuredPaths;+  void (* builderCallback)(std::vector<nix::StorePathWithOutputs>*, std::exception_ptr *exceptionToThrow);++  HerculesStore(const Params & params, ref<Store> storeToWrap);++  virtual const std::string name() override;++  // Overrides++  virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) override;++  virtual void ensurePath(const StorePath & path) override;++  virtual void buildPaths(+      const std::vector<DerivedPath> & paths,+      BuildMode buildMode = bmNormal) override;++  virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,+      BuildMode buildMode = bmNormal) override;++  virtual void queryMissing(const std::vector<DerivedPath> & targets,+      StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,+      uint64_t & downloadSize, uint64_t & narSize) override;++  // Additions++  void printDiagnostics();++  void setBuilderCallback(void (* newBuilderCallback)(std::vector<nix::StorePathWithOutputs>*, std::exception_ptr *exceptionToThrow));++  void inhibitBuilds();+  void uninhibitBuilds();+};

@@ -14,7 +14,6 @@ import Control.Monad.Except import Control.Monad.IO.Unlift import Control.Monad.Trans.Control-import qualified Data.ByteString as BS import qualified Data.Conduit import Data.Conduit.Extras (sinkChan, sinkChanTerminate, sourceChan) import Data.Conduit.Katip.Orphans ()@@ -64,6 +63,9 @@ import Hercules.CNix.Expr.Context (EvalState) import qualified Hercules.CNix.Expr.Raw import Hercules.CNix.Expr.Typed (Value)+import Hercules.CNix.Std.Vector (StdVector)+import qualified Hercules.CNix.Std.Vector as Std.Vector+import Hercules.CNix.Store.Context (NixStorePathWithOutputs) import Hercules.CNix.Util (triggerInterrupt) import Hercules.Error import Katip@@ -82,8 +84,8 @@ import qualified Prelude  data HerculesState = HerculesState-  { drvsCompleted :: TVar (Map Text (UUID, BuildResult.BuildStatus)),-    drvsInProgress :: IORef (Set Text),+  { drvsCompleted :: TVar (Map StorePath (UUID, BuildResult.BuildStatus)),+    drvsInProgress :: IORef (Set StorePath),     herculesStore :: Ptr (Ref HerculesStore),     wrappedStore :: Store,     shortcutChannel :: Chan (Maybe Event)@@ -252,7 +254,8 @@                 logLocM                   DebugS                   "Received remote build result"-              liftIO $ atomically $ modifyTVar (drvsCompleted herculesState) (M.insert path (attempt, result))+              storePath <- liftIO $ CNix.parseStorePath (wrappedStore herculesState) (encodeUtf8 path)+              liftIO $ atomically $ modifyTVar (drvsCompleted herculesState) (M.insert storePath (attempt, result))             _ -> pass     Command.Build build ->       katipAddNamespace "Build" $@@ -371,7 +374,7 @@     Eval.LiteralArg s -> ["--argstr", encodeUtf8 k, s]     Eval.ExprArg s -> ["--arg", encodeUtf8 k, s] -withDrvInProgress :: MonadUnliftIO m => HerculesState -> Text -> m a -> m a+withDrvInProgress :: MonadUnliftIO m => HerculesState -> StorePath -> m a -> m a withDrvInProgress HerculesState {drvsInProgress = ref} drvPath =   bracket acquire release . const   where@@ -435,54 +438,66 @@   UnliftIO unlift <- lift askUnliftIO   let decode = decodeUtf8With lenientDecode   liftIO $-    setBuilderCallback hStore $ \path -> unlift $-      katipAddContext (sl "fullpath" (decode path)) $ do-        logLocM DebugS "Building"-        let (plainDrv, bangOut) = BS.span (/= fromIntegral (ord '!')) path-            outputName = BS.dropWhile (== fromIntegral (ord '!')) bangOut-            plainDrvText = decode plainDrv-        withDrvInProgress st plainDrvText $ do-          liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (decode outputName) Nothing-          derivation <- liftIO $ getDerivation store plainDrv-          outputPath <- liftIO $ getDerivationOutputPath derivation outputName-          katipAddContext (sl "outputPath" (decode outputPath)) $-            logLocM DebugS "Naively calling ensurePath"-          liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e0 -> do-            katipAddContext (sl "message" (show (e0 :: SomeException) :: Text)) $-              logLocM DebugS "Recovering from failed wrapped.ensurePath"-            (attempt0, result) <--              liftIO $-                atomically $ do-                  c <- readTVar drvsCompl-                  anyAlternative $ M.lookup plainDrvText c-            liftIO $ maybeThrowBuildException result plainDrvText-            liftIO clearSubstituterCaches-            liftIO $ clearPathInfoCache store-            liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e1 -> do-              katipAddContext (sl "message" (show (e1 :: SomeException) :: Text)) $-                logLocM DebugS "Recovering from fresh ensurePath"-              liftIO $ writeChan shortcutChan $ Just $ Event.Build plainDrvText (decode outputName) (Just attempt0)-              -- TODO sync-              result' <--                liftIO $-                  atomically $ do-                    c <- readTVar drvsCompl-                    (attempt1, r) <- anyAlternative $ M.lookup plainDrvText c-                    guard (attempt1 /= attempt0)-                    pure r-              liftIO $ maybeThrowBuildException result' plainDrvText-              liftIO clearSubstituterCaches-              liftIO $ clearPathInfoCache store-              liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e2 ->-                liftIO $-                  throwIO $-                    BuildException-                      plainDrvText-                      ( Just $-                          "It could not be retrieved on the evaluating agent, despite a successful rebuild. Exception: "-                            <> show (e2 :: SomeException)-                      )-        logLocM DebugS "Built"+    setBuilderCallback hStore $+      traverseSPWOs $ \storePathWithOutputs -> unlift $ do+        drvStorePath <- liftIO $ getStorePath storePathWithOutputs+        drvPath <- liftIO $ CNix.storePathToPath store drvStorePath+        let pathText = decode drvPath+        outputs <- liftIO $ getOutputs storePathWithOutputs+        katipAddContext (sl "fullpath" pathText) $+          for_ outputs $ \outputName -> do+            logLocM DebugS "Building"+            withDrvInProgress st drvStorePath $ do+              liftIO $ writeChan shortcutChan $ Just $ Event.Build drvPath (decode outputName) Nothing+              derivation <- liftIO $ getDerivation store drvStorePath+              drvName <- liftIO $ getDerivationNameFromPath drvStorePath+              drvOutputs <- liftIO $ getDerivationOutputs store drvName derivation+              outputPath <-+                case find (\o -> derivationOutputName o == outputName) drvOutputs of+                  Nothing -> panic $ "output " <> show outputName <> " does not exist on " <> pathText+                  Just o -> case derivationOutputPath o of+                    Just x -> pure x+                    Nothing ->+                      -- FIXME ca-derivations+                      panic $ "output path unknown for output " <> show outputName <> " on " <> pathText <> ". ca-derivations is not supported yet."+              katipAddContext (sl "outputPath" (show outputPath :: Text)) $+                logLocM DebugS "Naively calling ensurePath"+              liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e0 -> do+                katipAddContext (sl "message" (show (e0 :: SomeException) :: Text)) $+                  logLocM DebugS "Recovering from failed wrapped.ensurePath"+                (attempt0, result) <-+                  liftIO $+                    atomically $ do+                      c <- readTVar drvsCompl+                      anyAlternative $ M.lookup drvStorePath c+                liftIO $ maybeThrowBuildException result (decode drvPath)+                liftIO clearSubstituterCaches+                liftIO $ clearPathInfoCache store+                liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e1 -> do+                  katipAddContext (sl "message" (show (e1 :: SomeException) :: Text)) $+                    logLocM DebugS "Recovering from fresh ensurePath"+                  liftIO $ writeChan shortcutChan $ Just $ Event.Build drvPath (decode outputName) (Just attempt0)+                  -- TODO sync+                  result' <-+                    liftIO $+                      atomically $ do+                        c <- readTVar drvsCompl+                        (attempt1, r) <- anyAlternative $ M.lookup drvStorePath c+                        guard (attempt1 /= attempt0)+                        pure r+                  liftIO $ maybeThrowBuildException result' (decode drvPath)+                  liftIO clearSubstituterCaches+                  liftIO $ clearPathInfoCache store+                  liftIO (ensurePath (wrappedStore st) outputPath) `catch` \e2 ->+                    liftIO $+                      throwIO $+                        BuildException+                          (decode drvPath)+                          ( Just $+                              "It could not be retrieved on the evaluating agent, despite a successful rebuild. Exception: "+                                <> show (e2 :: SomeException)+                          )+            logLocM DebugS "Built"   withEvalStateConduit store $ \evalState -> do     katipAddContext (sl "storeURI" (decode s)) $       logLocM DebugS "EvalState loaded."@@ -493,16 +508,22 @@       do         imprt <- liftIO $ evalFile evalState (toS $ Eval.file eval)         applied <- liftIO (autoCallFunction evalState imprt args)-        walk evalState args applied+        walk store evalState args applied     yield Event.EvaluationDone +traverseSPWOs :: (StorePathWithOutputs -> IO ()) -> StdVector NixStorePathWithOutputs -> IO ()+traverseSPWOs f v = do+  v' <- Std.Vector.toListFP v+  traverse_ f v'+ walk ::   (MonadUnliftIO m, KatipContext m) =>+  Store ->   Ptr EvalState ->   Value NixAttrs ->   RawValue ->   ConduitT i Event m ()-walk evalState = walk' True [] 10+walk store evalState = walk' True [] 10   where     handleErrors path = Data.Conduit.handleC (yieldAttributeError path)     walk' ::@@ -531,7 +552,8 @@                 isDeriv <- liftIO $ isDerivation evalState v                 if isDeriv                   then do-                    drvPath <- getDrvFile evalState v+                    drvStorePath <- getDrvFile evalState v+                    drvPath <- liftIO $ CNix.storePathToPath store drvStorePath                     typE <- runExceptT do                       isEffect <- liftEitherAs Left =<< liftIO (getAttrBool evalState attrValue "isEffect")                       case isEffect of

@@ -5,7 +5,6 @@  import Conduit import Data.Conduit.Katip.Orphans ()-import Foreign (ForeignPtr) import Hercules.Agent.Worker.Build.Prefetched (buildDerivation) import qualified Hercules.Agent.Worker.Build.Prefetched as Build import qualified Hercules.Agent.WorkerProtocol.Command.Build as Command.Build@@ -17,8 +16,7 @@     getDerivationOutputs,   ) import qualified Hercules.CNix as CNix-import Hercules.CNix.Store (Store, queryPathInfo, validPathInfoNarHash, validPathInfoNarSize)-import Hercules.CNix.Store.Context (Derivation)+import Hercules.CNix.Store (Store, queryPathInfo, validPathInfoNarHash32, validPathInfoNarSize) import Katip import Protolude hiding (yield) @@ -26,8 +24,10 @@ runBuild store build = do   let extraPaths = Command.Build.inputDerivationOutputPaths build       drvPath = encodeUtf8 $ Command.Build.drvPath build-  x <- for extraPaths $ \input -> do-    liftIO $ try $ CNix.ensurePath store input+  drvStorePath <- liftIO $ CNix.parseStorePath store drvPath+  x <- for extraPaths $ \input -> liftIO $ do+    storePath <- CNix.parseStorePath store input+    try $ CNix.ensurePath store storePath   materialize <- case sequenceA x of     Right _ ->       -- no error, proceed with requested materialization setting@@ -36,32 +36,35 @@       CNix.logInfo $ "while retrieving dependencies: " <> toS (displayException e)       CNix.logInfo "unable to retrieve dependency; attempting fallback to local build"       pure True-  derivationMaybe <- liftIO $ Build.getDerivation store drvPath+  drvName <- liftIO $ CNix.getDerivationNameFromPath drvStorePath+  derivationMaybe <- liftIO $ Build.getDerivation store drvStorePath   derivation <- case derivationMaybe of     Just drv -> pure drv-    Nothing -> panic $ "Could not retrieve derivation " <> show drvPath <> " from local store or binary caches."-  nixBuildResult <- liftIO $ buildDerivation store drvPath derivation (extraPaths <$ guard (not materialize))+    Nothing -> panic $ "Could not retrieve derivation " <> show drvStorePath <> " from local store or binary caches."+  nixBuildResult <- liftIO $ buildDerivation store drvStorePath derivation (extraPaths <$ guard (not materialize))   katipAddContext (sl "result" (show nixBuildResult :: Text)) $     logLocM DebugS "Build result"-  buildResult <- liftIO $ enrichResult store derivation nixBuildResult+  buildResult <- liftIO $ enrichResult store drvName derivation nixBuildResult   yield $ Event.BuildResult buildResult  -- TODO: case distinction on BuildStatus enumeration-enrichResult :: Store -> ForeignPtr Derivation -> Build.BuildResult -> IO Event.BuildResult.BuildResult-enrichResult _ _ result@Build.BuildResult {isSuccess = False} =+enrichResult :: Store -> ByteString -> CNix.Derivation -> Build.BuildResult -> IO Event.BuildResult.BuildResult+enrichResult _ _ _ result@Build.BuildResult {isSuccess = False} =   pure $     Event.BuildResult.BuildFailure {errorMessage = Build.errorMessage result}-enrichResult store derivation _ = do-  drvOuts <- getDerivationOutputs derivation+enrichResult store drvName derivation _ = do+  drvOuts <- getDerivationOutputs store drvName derivation   outputInfos <- for drvOuts $ \drvOut -> do-    vpi <- queryPathInfo store (derivationOutputPath drvOut)-    hash_ <- validPathInfoNarHash vpi-    let size = validPathInfoNarSize vpi+    -- FIXME: ca-derivations: always get the built path+    vpi <- for (derivationOutputPath drvOut) (queryPathInfo store)+    hash_ <- traverse validPathInfoNarHash32 vpi+    path <- traverse (CNix.storePathToPath store) (derivationOutputPath drvOut)+    let size = fmap validPathInfoNarSize vpi     pure       Event.BuildResult.OutputInfo         { name = derivationOutputName drvOut,-          path = derivationOutputPath drvOut,-          hash = hash_,-          size = size+          path = fromMaybe "" path,+          hash = fromMaybe "" hash_,+          size = fromMaybe 0 size         }   pure $ Event.BuildResult.BuildSuccess outputInfos

@@ -12,6 +12,7 @@ import qualified Data.ByteString.Char8 as C8 import Foreign (FinalizerPtr, ForeignPtr, alloca, newForeignPtr, nullPtr, peek) import Foreign.C (peekCString)+import Hercules.CNix.Encapsulation import Hercules.CNix.Store import qualified Language.C.Inline.Cpp as C import qualified Language.C.Inline.Cpp.Exceptions as C@@ -37,6 +38,8 @@  C.include "<nix/fs-accessor.hh>" +C.include "<nix-compat.hh>"+ C.include "<hercules-ci-cnix/store.hxx>"  C.using "namespace nix"@@ -90,11 +93,11 @@ nullableForeignPtr _ rawPtr | rawPtr == nullPtr = pure Nothing nullableForeignPtr finalize rawPtr = Just <$> newForeignPtr finalize rawPtr -getDerivation :: Store -> ByteString -> IO (Maybe (ForeignPtr Derivation))+getDerivation :: Store -> StorePath -> IO (Maybe Derivation) getDerivation (Store store) derivationPath =-  nullableForeignPtr finalizeDerivation+  nullableMoveToForeignPtrWrapper     =<< [C.throwBlock| Derivation *{-      std::string derivationPath($bs-ptr:derivationPath, $bs-len:derivationPath);+      StorePath derivationPath = *$fptr-ptr:(nix::StorePath *derivationPath);       std::list<nix::ref<nix::Store>> stores = getDefaultSubstituters();       stores.push_front(*$(refStore* store)); @@ -102,14 +105,7 @@        for (nix::ref<nix::Store> & currentStore : stores) {         try {-          auto accessor = currentStore->getFSAccessor();-          auto drvText = accessor->readFile(derivationPath);--          Path tmpDir = createTempDir();-          AutoDelete delTmpDir(tmpDir, true);-          Path drvTmpPath = tmpDir + "/drv";-          writeFile(drvTmpPath, drvText, 0600);-          derivation = new nix::Derivation(nix::readDerivation(drvTmpPath));+          derivation = new nix::Derivation(currentStore->derivationFromPath(printPath23(*currentStore, derivationPath)));           break;         } catch (nix::Interrupted &e) {           throw e;@@ -126,7 +122,7 @@     }|]  -- | @buildDerivation derivationPath derivationText@-buildDerivation :: Store -> ByteString -> ForeignPtr Derivation -> Maybe [ByteString] -> IO BuildResult+buildDerivation :: Store -> StorePath -> Derivation -> Maybe [ByteString] -> IO BuildResult buildDerivation (Store store) derivationPath derivation extraInputs =   let extraInputsMerged = C8.intercalate "\n" (fromMaybe [] extraInputs)       materializeDerivation = if isNothing extraInputs then 1 else 0@@ -142,13 +138,19 @@       const char *&errorMessage = *$(const char **errorMessagePtr);       time_t &startTime = *$(time_t *startTimePtr);       time_t &stopTime = *$(time_t *stopTimePtr);-      std::string derivationPath($bs-ptr:derivationPath, $bs-len:derivationPath);+      StorePath derivationPath = *$fptr-ptr:(nix::StorePath *derivationPath);        if ($(bool materializeDerivation)) {-        store.ensurePath(derivationPath);-        nix::PathSet paths{derivationPath};+        store.ensurePath(printPath23(store, derivationPath));+#ifdef NIX_2_4+        auto derivation = store.derivationFromPath(derivationPath);+        StorePathWithOutputs storePathWithOutputs { .path = derivationPath, .outputs = derivation.outputNames() };+        std::vector<nix::StorePathWithOutputs> paths{storePathWithOutputs};+#else+        nix::PathSet paths{printPath23(store, derivationPath)};+#endif         try {-          store.buildPaths(paths);+          store.buildPaths(toDerivedPaths24(paths));           status = -1;           success = true;           errorMessage = strdup("");@@ -170,10 +172,15 @@         std::istringstream stream(extraInputsMerged);          while (std::getline(stream, extraInput)) {-          derivation->inputSrcs.insert(extraInput);+#ifdef NIX_2_4+          auto path = store.parseStorePath(extraInput);+#else+          auto path = extraInput;+#endif+          derivation->inputSrcs.insert(path);         } -        nix::BuildResult result = store.buildDerivation(derivationPath, *derivation);+        nix::BuildResult result = store.buildDerivation(printPath23(store, derivationPath), *derivation);          switch (result.status) {           case nix::BuildResult::Built:

@@ -4,13 +4,12 @@ module Hercules.Agent.Worker.Effect where  import Control.Monad.Catch (MonadThrow)-import GHC.ForeignPtr (ForeignPtr) import Hercules.Agent.Worker.Build.Prefetched (buildDerivation) import qualified Hercules.Agent.Worker.Build.Prefetched as Build import qualified Hercules.Agent.WorkerProtocol.Command.Effect as Command.Effect import Hercules.CNix (Store) import qualified Hercules.CNix as CNix-import Hercules.CNix.Store.Context (Derivation)+import Hercules.CNix.Store (Derivation) import qualified Hercules.Effect as Effect import Katip (KatipContext) import Protolude@@ -22,28 +21,29 @@   dir <- getCurrentDirectory   Effect.runEffect derivation (Command.Effect.token command) (Command.Effect.secretsPath command) (Command.Effect.apiBaseURL command) dir -prepareDerivation :: MonadIO m => Store -> Command.Effect.Effect -> m (ForeignPtr Derivation)+prepareDerivation :: MonadIO m => Store -> Command.Effect.Effect -> m Derivation prepareDerivation store command = do   let extraPaths = Command.Effect.inputDerivationOutputPaths command       drvPath = encodeUtf8 $ Command.Effect.drvPath command-      ensureDeps = for_ extraPaths $ \input ->-        liftIO $ CNix.ensurePath store input+      ensureDeps = for_ extraPaths $ \input -> liftIO $ do+        CNix.ensurePath store =<< CNix.parseStorePath store input+  drvStorePath <- liftIO $ CNix.parseStorePath store drvPath   liftIO $ do     ensureDeps `catch` \e -> do       CNix.logInfo $ "while retrieving dependencies: " <> toS (displayException (e :: SomeException))       CNix.logInfo "unable to retrieve dependency; attempting fallback to local build"-      CNix.ensurePath store drvPath-      derivation <- CNix.getDerivation store drvPath-      depDrvPaths <- CNix.getDerivationInputs derivation+      CNix.ensurePath store drvStorePath+      derivation <- CNix.getDerivation store drvStorePath+      depDrvPaths <- CNix.getDerivationInputs store derivation       for_ depDrvPaths \(depDrv, _outputs) -> do         depDerivation <- CNix.getDerivation store depDrv         _nixBuildResult <- liftIO $ buildDerivation store depDrv depDerivation mempty         pass   derivation <--    liftIO (Build.getDerivation store drvPath) >>= \case+    liftIO (Build.getDerivation store drvStorePath) >>= \case       Just drv -> pure drv       Nothing -> panic $ "Could not retrieve derivation " <> show drvPath <> " from local store or binary caches."-  sources <- liftIO $ CNix.getDerivationSources derivation+  sources <- liftIO $ CNix.getDerivationSources store derivation   for_ sources \src -> do     liftIO $ CNix.ensurePath store src   pure derivation

@@ -11,9 +11,7 @@ import Control.Exception   ( catch,   )-import qualified Data.ByteString.Unsafe as BS import Data.Coerce (coerce)-import qualified Foreign.C import Foreign.C.String (withCString) import Foreign.StablePtr (castStablePtrToPtr, newStablePtr) import Hercules.Agent.Worker.HerculesStore.Context@@ -21,8 +19,10 @@     HerculesStore,     context,   )-import Hercules.CNix (Store)+import Hercules.CNix.Encapsulation (moveToForeignPtrWrapper) import Hercules.CNix.Expr (Store (Store))+import Hercules.CNix.Expr.Context (NixStorePathWithOutputs)+import Hercules.CNix.Std.Vector (CStdVector, StdVector) import Hercules.CNix.Store.Context (Ref) import qualified Language.C.Inline.Cpp as C import qualified Language.C.Inline.Cpp.Exceptions as C@@ -47,6 +47,8 @@  C.include "<nix/globals.hh>" +C.include "<nix-compat.hh>"+ C.include "hercules-aliases.h"  C.using "namespace nix"@@ -76,21 +78,21 @@   }|]  -- TODO catch pure exceptions from displayException-setBuilderCallback :: Ptr (Ref HerculesStore) -> (ByteString -> IO ()) -> IO ()+setBuilderCallback :: Ptr (Ref HerculesStore) -> (StdVector NixStorePathWithOutputs -> IO ()) -> IO () setBuilderCallback s callback = do   p <--    mkBuilderCallback $ \cstr exceptionToThrowPtr ->-      Control.Exception.catch (BS.unsafePackMallocCString cstr >>= callback) $ \e ->+    mkBuilderCallback $ \sp exceptionToThrowPtr ->+      Control.Exception.catch (callback =<< moveToForeignPtrWrapper sp) $ \e ->         withCString (displayException (e :: SomeException)) $ \renderedException -> do           stablePtr <- castStablePtrToPtr <$> newStablePtr e           [C.block| void {             (*$(exception_ptr *exceptionToThrowPtr)) = std::make_exception_ptr(HaskellException(std::string($(const char* renderedException)), $(void* stablePtr)));           }|]   [C.throwBlock| void {-    (*$(refHerculesStore* s))->setBuilderCallback($(void (*p)(const char *, exception_ptr *) ));+    (*$(refHerculesStore* s))->setBuilderCallback($(void (*p)(std::vector<nix::StorePathWithOutputs>*, exception_ptr *)));   }|] -type BuilderCallback = Foreign.C.CString -> Ptr ExceptionPtr -> IO ()+type BuilderCallback = Ptr (CStdVector NixStorePathWithOutputs) -> Ptr ExceptionPtr -> IO ()  -- Work around a problem in ghcide with foreign imports. #ifndef __GHCIDE__

@@ -6,6 +6,7 @@  import qualified Data.Map as M import Hercules.CNix.Expr.Context (Ref)+import Hercules.CNix.Std.Vector (stdVectorCtx) import qualified Hercules.CNix.Store.Context as Store import qualified Language.C.Inline.Context as C import qualified Language.C.Inline.Cpp as C@@ -20,6 +21,7 @@ context =   C.cppCtx <> C.fptrCtx     <> C.bsCtx+    <> stdVectorCtx     <> Store.context     <> herculesStoreContext 

@@ -1,7 +1,7 @@ cabal-version: 2.4  name:           hercules-ci-agent-version:        0.8.1+version:        0.8.2 synopsis:       Runs Continuous Integration tasks on your machines category:       Nix, CI, Testing, DevOps homepage:       https://docs.hercules-ci.com@@ -14,10 +14,15 @@ extra-source-files:     CHANGELOG.md     cbits/hercules-aliases.h-    cbits/hercules-store.hh     cbits/hercules-logger.hh+    cbits/nix-2.3/hercules-store.hh+    cbits/nix-2.4/hercules-store.hh     testdata/vm-test-run-agent-test.drv +flag nix-2_4+  description: Build for Nix >=2.4pre*+  default: False+ source-repository head   type: git   location: https://github.com/hercules-ci/hercules-ci-agent@@ -183,8 +188,8 @@     , exceptions     , filepath     , hercules-ci-agent-    , hercules-ci-api-core == 0.1.2.0-    , hercules-ci-api-agent == 0.3.1.0+    , hercules-ci-api-core == 0.1.3.0+    , hercules-ci-api-agent == 0.4.0.0     , hostname     , http-client     , http-client-tls@@ -243,8 +248,16 @@   hs-source-dirs:       hercules-ci-agent-worker   cxx-sources:-      cbits/hercules-store.cxx       cbits/hercules-logger.cxx+  if flag(nix-2_4)+    cpp-options:+        -DNIX_2_4+    cxx-sources:+      cbits/nix-2.4/hercules-store.cxx+  else+    cxx-sources:+      cbits/nix-2.3/hercules-store.cxx+   default-extensions: DeriveGeneric DeriveTraversable DisambiguateRecordFields FlexibleContexts InstanceSigs LambdaCase MultiParamTypeClasses NoImplicitPrelude OverloadedStrings RankNTypes TupleSections TypeApplications TypeOperators   ghc-options:     -Werror=incomplete-patterns -Werror=missing-fields@@ -259,6 +272,12 @@     -with-rtsopts=-maxN8   include-dirs:       cbits+  if flag(nix-2_4)+    include-dirs:+      cbits/nix-2.4+  else+    include-dirs:+      cbits/nix-2.3   extra-libraries:       boost_context   build-depends:@@ -352,6 +371,7 @@     , process     , protolude     , safe-exceptions+    , temporary     , text     , transformers-base     , unliftio-core

@@ -37,6 +37,7 @@ import qualified Hercules.Agent.Build as Build import Hercules.Agent.CabalInfo (herculesAgentVersion) import qualified Hercules.Agent.Cache as Cache+import qualified Hercules.Agent.Cachix.Env as Cachix.Env import Hercules.Agent.Client   ( lifeCycleClient,     tasksClient,@@ -117,7 +118,9 @@                       ServicePayload.StartEvaluation evalTask ->                         launchTask tasks socket (Task.upcastId $ EvaluateTask.id evalTask) do                           Cache.withCaches do-                            Evaluate.performEvaluation evalTask+                            -- TODO move the store directly under env+                            store <- asks (Cachix.Env.nixStore . Env.cachixEnv)+                            Evaluate.performEvaluation store evalTask                             pure $ TaskStatus.Successful ()                       ServicePayload.StartBuild buildTask ->                         launchTask tasks socket (Task.upcastId $ BuildTask.id buildTask) do

@@ -17,6 +17,7 @@ import Hercules.API.TaskStatus (TaskStatus) import qualified Hercules.API.TaskStatus as TaskStatus import qualified Hercules.Agent.Cache as Agent.Cache+import qualified Hercules.Agent.Cachix.Env as Cachix.Env import qualified Hercules.Agent.Client import qualified Hercules.Agent.Config as Config import Hercules.Agent.Env@@ -32,6 +33,7 @@ import qualified Hercules.Agent.WorkerProtocol.Event as Event import qualified Hercules.Agent.WorkerProtocol.Event.BuildResult as BuildResult import qualified Hercules.Agent.WorkerProtocol.LogSettings as LogSettings+import qualified Hercules.CNix.Store as CNix import Hercules.Error (defaultRetry) import qualified Network.URI import Protolude@@ -85,7 +87,8 @@     Just BuildResult.BuildSuccess {outputs = outs'} -> do       let outs = convertOutputs (BuildTask.derivationPath buildTask) outs'       reportOutputInfos buildTask outs-      push buildTask outs+      store <- asks (Cachix.Env.nixStore . Env.cachixEnv)+      push store buildTask outs       reportSuccess buildTask       pure $ TaskStatus.Successful ()     Just BuildResult.BuildFailure {} -> pure $ TaskStatus.Terminated ()@@ -102,12 +105,14 @@         hash = decodeUtf8With lenientDecode $ BuildResult.hash oi       } -push :: BuildTask -> Map Text OutputInfo -> App ()-push buildTask outs = do+push :: CNix.Store -> BuildTask -> Map Text OutputInfo -> App ()+push store buildTask outs = do   let paths = OutputInfo.path <$> toList outs   caches <- activePushCaches   forM_ caches $ \cache -> do-    Agent.Cache.push cache paths 4+    -- TODO preserve StorePath instead+    storePaths <- liftIO $ for paths (CNix.parseStorePath store . encodeUtf8)+    Agent.Cache.push cache storePaths 4     emitEvents buildTask [BuildEvent.Pushed $ Pushed.Pushed {cache = cache}]  reportSuccess :: BuildTask -> App ()

@@ -13,6 +13,9 @@ import qualified Hercules.Agent.Nix as Nix import qualified Hercules.Agent.SecureDirectory as SecureDirectory import qualified Hercules.CNix as CNix+import Hercules.CNix.Std.Set (StdSet, toListFP)+import qualified Hercules.CNix.Std.Set as Std.Set+import Hercules.CNix.Store (StorePath) import qualified Hercules.CNix.Store as Store import qualified Hercules.Formats.NixCache as NixCache import Katip@@ -42,7 +45,7 @@   -- | Cache name   Text ->   -- | Paths, which do not have to form a closure-  [Text] ->+  [StorePath] ->   -- | Number of concurrent upload threads   Int ->   App ()@@ -58,14 +61,13 @@         throwIO $ FatalError $ "Agent does not have a binary cache named " <> show cacheName <> " in its configuration."   fromMaybe failNothing (maybeNix <|> maybeCachix) -nixPush :: NixCache.NixCache -> [Text] -> Int -> App ()-nixPush cacheConf pathsText _concurrency = do-  let paths = map encodeUtf8 pathsText+nixPush :: NixCache.NixCache -> [StorePath] -> Int -> App ()+nixPush cacheConf paths _concurrency = do   keys <- liftIO $ for (NixCache.signingKeys cacheConf) (CNix.parseSecretKey . encodeUtf8)   CNix.withStore $ \store -> do-    (Sum total, Sum signed) <--      mconcat <$> for keys \key -> liftIO $ do-        pathSet <- paths & newPathSetWith+    (Sum total, Sum signed) <- liftIO do+      pathSet <- Std.Set.fromListFP paths+      mconcat <$> for keys \key -> do         signClosure store key pathSet     katipAddContext (sl "num-signatures" signed <> sl "num-paths" total) $       logLocM DebugS "Signed"@@ -73,18 +75,11 @@     CNix.withStoreFromURI (NixCache.storeURI cacheConf) $ \cache -> do       liftIO (CNix.copyClosure store cache paths) -newPathSetWith :: [ByteString] -> IO Store.PathSet-newPathSetWith paths = do-  pathSet <- Store.newEmptyPathSet-  for_ paths \path ->-    Store.addToPathSet path pathSet-  pure pathSet--signClosure :: CNix.Store -> ForeignPtr CNix.SecretKey -> Store.PathSet -> IO (Sum Int, Sum Int)+signClosure :: CNix.Store -> ForeignPtr CNix.SecretKey -> StdSet Store.NixStorePath -> IO (Sum Int, Sum Int) signClosure store key' pathSet = withForeignPtr key' \key -> do-  closure <- Store.computeFSClosure store Store.defaultClosureParams pathSet+  closure <- Store.computeFSClosure store Store.defaultClosureParams pathSet >>= toListFP   closure-    & Store.traversePathSet+    & traverse       ( \path -> do           CNix.signPath store key path       )

@@ -12,11 +12,12 @@ import Hercules.Agent.Env as Agent.Env hiding (activePushCaches) import qualified Hercules.Agent.EnvironmentInfo as EnvInfo import Hercules.Agent.Log+import Hercules.CNix.Store (StorePath) import Hercules.Error import qualified Hercules.Formats.CachixCache as CachixCache import Protolude -push :: Text -> [Text] -> Int -> App ()+push :: Text -> [StorePath] -> Int -> App () push cache paths workers = withNamedContext "cache" cache $ do   Agent.Cachix.Env     { pushCaches = pushCaches,@@ -36,7 +37,7 @@             pushParamsStore = nixStore,             pushParamsClientEnv = clientEnv,             pushParamsStrategy = \storePath ->-              let ctx = withNamedContext "path" storePath+              let ctx = withNamedContext "path" (show storePath :: Text)                in Cachix.Push.PushStrategy                     { onAlreadyPresent = pass,                       onAttempt = \retryStatus size ->@@ -65,7 +66,7 @@   cks <- asks (Agent.Cachix.cacheKeys . Agent.Env.cachixEnv)   nixInfo <- liftIO EnvInfo.getNixInfo   pure-    ( EnvInfo.nixSubstituters nixInfo+    ( map (decodeUtf8With lenientDecode) (EnvInfo.nixSubstituters nixInfo)         ++ map (\c -> "https://" <> c <> ".cachix.org") (M.keys cks)     ) @@ -73,4 +74,7 @@ getTrustedPublicKeys = do   cks <- asks (Agent.Cachix.cacheKeys . Agent.Env.cachixEnv)   nixInfo <- liftIO EnvInfo.getNixInfo-  pure (EnvInfo.nixTrustedPublicKeys nixInfo ++ concatMap CachixCache.publicKeys cks)+  pure+    ( map (decodeUtf8With lenientDecode) (EnvInfo.nixTrustedPublicKeys nixInfo)+        ++ concatMap CachixCache.publicKeys cks+    )

@@ -1,18 +1,9 @@+{-# LANGUAGE ScopedTypeVariables #-}+ module Hercules.Agent.EnvironmentInfo where -import Control.Lens-  ( to,-    (^..),-    (^?),-  )-import qualified Data.Aeson as Aeson-import Data.Aeson.Lens-  ( key,-    _Array,-    _Number,-    _String,-  )-import qualified Data.ByteString.Lazy as LBS+import qualified Data.ByteString as BS+import qualified Data.Set as S import qualified Data.Text as T import qualified Hercules.API.Agent.LifeCycle.AgentInfo as AgentInfo import Hercules.Agent.CabalInfo as CabalInfo@@ -20,10 +11,11 @@ import qualified Hercules.Agent.Config as Config import Hercules.Agent.Env as Env import Hercules.Agent.Log+import qualified Hercules.CNix as CNix+import qualified Hercules.CNix.Settings as Settings import qualified Hercules.CNix.Store as Store import Network.HostName (getHostName) import Protolude hiding (to)-import qualified System.Process as Process  extractAgentInfo :: App AgentInfo.AgentInfo extractAgentInfo = do@@ -37,15 +29,15 @@   let s =         AgentInfo.AgentInfo           { hostname = toS hostname,-            agentVersion = CabalInfo.herculesAgentVersion, -- TODO: Add git revision-            nixVersion = nixExeVersion nix,+            agentVersion = CabalInfo.herculesAgentVersion,+            nixVersion = nixLibVersion nix,             nixClientProtocolVersion = nixClientProtocolVersion,             nixDaemonProtocolVersion = nixStoreProtocolVersion,-            platforms = nixPlatforms nix,+            platforms = map fromUtf8Lenient $ nixPlatforms nix,             cachixPushCaches = cachixPushCaches,             pushCaches = pushCaches,-            systemFeatures = nixSystemFeatures nix,-            substituters = nixSubstituters nix, -- TODO: Add cachix substituters+            systemFeatures = map fromUtf8Lenient $ nixSystemFeatures nix,+            substituters = map fromUtf8Lenient $ nixSubstituters nix,             concurrentTasks = fromIntegral $ Config.concurrentTasks cfg,             labels = Config.labels cfg           }@@ -53,72 +45,39 @@   pure s  data NixInfo = NixInfo-  { nixExeVersion :: Text,-    nixPlatforms :: [Text],-    nixSystemFeatures :: [Text],-    nixSubstituters :: [Text],-    nixTrustedPublicKeys :: [Text],-    nixNarinfoCacheNegativeTTL :: Maybe Integer,-    nixNetrcFile :: Maybe Text+  { nixLibVersion :: Text,+    nixPlatforms :: [ByteString],+    nixSystemFeatures :: [ByteString],+    nixSubstituters :: [ByteString],+    nixTrustedPublicKeys :: [ByteString],+    nixNarinfoCacheNegativeTTL :: Word64,+    nixNetrcFile :: Maybe ByteString   }+  deriving (Show) +fromUtf8Lenient :: ByteString -> Text+fromUtf8Lenient = decodeUtf8With lenientDecode+ getNixInfo :: IO NixInfo getNixInfo = do-  let stdinEmpty = ""-  version <- Process.readProcess "nix" ["--version"] stdinEmpty-  rawJson <- Process.readProcess "nix" ["show-config", "--json"] stdinEmpty-  cfg <--    case Aeson.eitherDecode (LBS.fromStrict $ encodeUtf8 $ toS rawJson) of-      Left e -> panic $ "Could not parse nix show-config --json: " <> show e-      Right r -> pure r+  extraPlatforms <- Settings.getExtraPlatforms+  system <- Settings.getSystem+  systemFeatures <- Settings.getSystemFeatures+  substituters <- Settings.getSubstituters+  trustedPublicKeys <- Settings.getTrustedPublicKeys+  narinfoCacheNegativeTTL <- Settings.getNarinfoCacheNegativeTtl+  netrcFile <- Settings.getNetrcFile   pure     NixInfo-      { nixExeVersion = T.dropAround isSpace (toS version),-        nixPlatforms =-          ((cfg :: Aeson.Value) ^.. key "system" . key "value" . _String)-            <> ( cfg-                   ^.. key "extra-platforms"-                     . key "value"-                     . _Array-                     . traverse-                     . _String-               ),-        nixSystemFeatures =-          cfg-            ^.. key "system-features"-              . key "value"-              . _Array-              . traverse-              . _String,-        nixSubstituters =-          cfg-            ^.. key "substituters"-              . key "value"-              . _Array-              . traverse-              . _String-              . to cleanUrl,-        nixTrustedPublicKeys =-          cfg-            ^.. key "trusted-public-keys"-              . key "value"-              . _Array-              . traverse-              . _String-              . to cleanUrl,-        nixNarinfoCacheNegativeTTL =-          cfg-            ^? key "narinfo-cache-negative-ttl"-              . key "value"-              . _Number-              . to floor,-        nixNetrcFile =-          cfg-            ^? key "netrc-file"-              . key "value"-              . _String+      { nixLibVersion = T.dropAround isSpace (fromUtf8Lenient CNix.nixVersion),+        nixPlatforms = toList (S.singleton system <> extraPlatforms),+        nixSystemFeatures = toList systemFeatures,+        nixSubstituters = map cleanUrl substituters,+        nixTrustedPublicKeys = trustedPublicKeys,+        nixNarinfoCacheNegativeTTL = narinfoCacheNegativeTTL,+        nixNetrcFile = guard (netrcFile /= "") $> netrcFile       } -cleanUrl :: Text -> Text-cleanUrl t | "@" `T.isInfixOf` t = "<URI censored; might contain secret>"+cleanUrl :: ByteString -> ByteString+cleanUrl t | "@" `BS.isInfixOf` t = "<URI censored; might contain secret>" cleanUrl t = t

@@ -40,9 +40,11 @@ import qualified Hercules.API.Agent.Evaluate.EvaluateTask as EvaluateTask import Hercules.API.Servant (noContent) import qualified Hercules.Agent.Cache as Agent.Cache+import qualified Hercules.Agent.Cachix.Env as Cachix.Env import qualified Hercules.Agent.Client import Hercules.Agent.Env import qualified Hercules.Agent.Env as Env+import Hercules.Agent.Evaluate.TraversalQueue (Queue) import qualified Hercules.Agent.Evaluate.TraversalQueue as TraversalQueue import Hercules.Agent.Files import Hercules.Agent.Log@@ -66,7 +68,7 @@ import qualified Hercules.Agent.WorkerProtocol.Event.Attribute as WorkerAttribute import qualified Hercules.Agent.WorkerProtocol.Event.AttributeError as WorkerAttributeError import qualified Hercules.Agent.WorkerProtocol.LogSettings as LogSettings-import Hercules.CNix (withStore)+import Hercules.CNix.Store (Store, StorePath, parseStorePath) import Hercules.Error (defaultRetry, quickRetry) import qualified Network.HTTP.Client.Conduit as HTTP.Conduit import qualified Network.HTTP.Simple as HTTP.Simple@@ -83,18 +85,19 @@ pushEvalWorkers :: Int pushEvalWorkers = 16 -performEvaluation :: EvaluateTask.EvaluateTask -> App ()-performEvaluation task' =-  withProducer (produceEvaluationTaskEvents task') $ \producer ->+performEvaluation :: Store -> EvaluateTask.EvaluateTask -> App ()+performEvaluation store task' =+  withProducer (produceEvaluationTaskEvents store task') $ \producer ->     withBoundedDelayBatchProducer (1000 * 1000) 1000 producer $ \batchProducer ->       fix $ \continue ->         joinSTM $ listen batchProducer (\b -> withSync b (postBatch task' . catMaybes) *> continue) pure  produceEvaluationTaskEvents ::+  Store ->   EvaluateTask.EvaluateTask ->   (Syncing EvaluateEvent.EvaluateEvent -> App ()) ->   App ()-produceEvaluationTaskEvents task writeToBatch = withWorkDir "eval" $ \tmpdir -> do+produceEvaluationTaskEvents store task writeToBatch = withWorkDir "eval" $ \tmpdir -> do   logLocM DebugS "Retrieving evaluation task"   let emitSingle = writeToBatch . Syncable       sync = syncer writeToBatch@@ -206,7 +209,7 @@               Message.typ = Message.Error,               Message.message = e             }-    Right file -> TraversalQueue.with $ \derivationQueue ->+    Right file -> TraversalQueue.with $ \(derivationQueue :: Queue StorePath) ->       let doIt = do             Async.Lifted.concurrently_ evaluation emitDrvs             -- derivationInfo upload has finished@@ -241,16 +244,17 @@               emit $ EvaluateEvent.PushedAll $ PushedAll.PushedAll {cache = cache}           captureAttrDrvAndEmit msg = do             case msg of-              EvaluateEvent.Attribute ae ->-                addTopDerivation (AttributeEvent.derivationPath ae)+              EvaluateEvent.Attribute ae -> do+                storePath <- liftIO $ parseStorePath store (encodeUtf8 $ AttributeEvent.derivationPath ae)+                addTopDerivation storePath               _ -> pass             emit msg           emitDrvs =-            withStore $ \store -> TraversalQueue.work derivationQueue $ \recurse drvPath -> do+            TraversalQueue.work derivationQueue $ \recurse drvPath -> do               drvInfo <- retrieveDerivationInfo store drvPath-              forM_-                (M.keys $ DerivationInfo.inputDerivations drvInfo)-                recurse -- asynchronously+              for_ (M.keys $ DerivationInfo.inputDerivations drvInfo) \inp -> do+                inputStorePath <- liftIO $ parseStorePath store (encodeUtf8 inp)+                recurse inputStorePath -- asynchronously               emit $ EvaluateEvent.DerivationInfo drvInfo        in doIt @@ -273,7 +277,7 @@   ] ->   (EvaluateEvent.EvaluateEvent -> App ()) ->   -- | Upload a derivation, return when done-  (Text -> App ()) ->+  (StorePath -> App ()) ->   App () ->   Text ->   App ()@@ -339,13 +343,16 @@                       }                 continue               Event.Build drv outputName notAttempt -> do+                store <- asks (Cachix.Env.nixStore . Env.cachixEnv)+                storePath <- liftIO (parseStorePath store drv)+                let drvText = decode drv                 status <--                  withNamedContext "derivation" drv $ do+                  withNamedContext "derivation" (decode drv) $ do                     currentIndex <- liftIO $ atomicModifyIORef buildRequiredIndex (\i -> (i + 1, i))                     emit $                       EvaluateEvent.BuildRequired                         BuildRequired.BuildRequired-                          { BuildRequired.derivationPath = drv,+                          { BuildRequired.derivationPath = drvText,                             BuildRequired.index = currentIndex,                             BuildRequired.outputName = outputName                           }@@ -353,21 +360,21 @@                           caches <- activePushCaches                           forM_ caches $ \cache -> do                             withNamedContext "cache" cache $ logLocM DebugS "Pushing derivations for import from derivation"-                            Agent.Cache.push cache [drv] pushEvalWorkers+                            Agent.Cache.push cache [storePath] pushEvalWorkers                     Async.Lifted.concurrently_-                      (uploadDerivationInfos drv)+                      (uploadDerivationInfos storePath)                       pushDerivations                     emit $                       EvaluateEvent.BuildRequest                         BuildRequest.BuildRequest-                          { derivationPath = drv,+                          { derivationPath = drvText,                             forceRebuild = isJust notAttempt                           }                     flush-                    status <- drvPoller notAttempt drv+                    status <- drvPoller notAttempt drvText                     logLocM DebugS $ "Got derivation status " <> logStr (show status :: Text)                     return status-                writeChan commandChan $ Just $ Command.BuildResult $ uncurry (BuildResult.BuildResult drv) status+                writeChan commandChan $ Just $ Command.BuildResult $ uncurry (BuildResult.BuildResult drvText) status                 continue               Event.Exception e -> panic e               -- Unused during eval

@@ -8,8 +8,6 @@ import Hercules.Agent.Nix.Env as Nix.Env import Protolude import System.Directory (doesFileExist)-import System.Process (readProcess)-import qualified System.Process  withExtraOptions :: [(Text, Text)] -> App a -> App a withExtraOptions extraOpts = local $ \env ->@@ -23,35 +21,10 @@ askExtraOptions :: MonadReader Agent.Env.Env m => m [(Text, Text)] askExtraOptions = asks (extraOptions . nixEnv) -getExtraOptionArguments :: MonadReader Agent.Env.Env m => m [Text]-getExtraOptionArguments = do-  asks (f . extraOptions . nixEnv)-  where-    f = concatMap $ \(opt, val) -> ["--option", opt, val]--readNixProcess ::-  -- | Command-  Text ->-  -- | Options-  [Text] ->-  -- | Paths-  [Text] ->-  -- | Stdin-  Text ->-  App Text-readNixProcess cmd opts paths stdinText = do-  extraOpts <- getExtraOptionArguments-  toS <$> liftIO (readProcess (toS cmd) (map toS (opts <> extraOpts <> ["--"] <> paths)) (toS stdinText))--nixProc :: Text -> [Text] -> [Text] -> App System.Process.CreateProcess-nixProc exe opts args = do-  extraOpts <- getExtraOptionArguments-  pure $ System.Process.proc (toS exe) (map toS (opts <> extraOpts <> ["--"] <> args))- getNetrcLines :: App [Text] getNetrcLines = liftIO $ do   info <- getNixInfo-  let fm = toS <$> nixNetrcFile info+  let fm = toS . decodeUtf8With lenientDecode <$> nixNetrcFile info   fmap fold <$> runMaybeT $ do     f <- MaybeT $ pure fm     exs <- lift $ doesFileExist f

@@ -7,7 +7,8 @@ newEnv :: IO Env newEnv = do   nixInfo <- EnvironmentInfo.getNixInfo-  when (EnvironmentInfo.nixNarinfoCacheNegativeTTL nixInfo /= Just 0) $ do+  print nixInfo+  when (EnvironmentInfo.nixNarinfoCacheNegativeTTL nixInfo /= 0) $ do     putErrText       "\n\       \We have detected that the setting narinfo-cache-negative-ttl is non-zero.\n\

@@ -1,8 +1,9 @@+{-# LANGUAGE BlockArguments #-}+ module Hercules.Agent.Nix.RetrieveDerivationInfo where  import qualified Data.Map as M import qualified Data.Text as T-import Foreign.ForeignPtr (ForeignPtr) import Hercules.API.Agent.Evaluate.EvaluateEvent.DerivationInfo   ( DerivationInfo (DerivationInfo),   )@@ -13,39 +14,49 @@ retrieveDerivationInfo ::   MonadIO m =>   Store ->-  DerivationInfo.DerivationPathText ->+  StorePath ->   m DerivationInfo retrieveDerivationInfo store drvPath = liftIO $ do-  drv <- getDerivation store (encodeUtf8 drvPath)-  retrieveDerivationInfo' drvPath drv+  drv <- getDerivation store drvPath+  path <- storePathToPath store drvPath+  drvName <- getDerivationNameFromPath drvPath+  retrieveDerivationInfo' store path drvName drv -retrieveDerivationInfo' :: Text -> ForeignPtr Derivation -> IO DerivationInfo-retrieveDerivationInfo' drvPath drv = do-  sourcePaths <- getDerivationSources drv-  inputDrvPaths <- getDerivationInputs drv-  outputs <- getDerivationOutputs drv+retrieveDerivationInfo' :: Store -> ByteString -> ByteString -> Derivation -> IO DerivationInfo+retrieveDerivationInfo' store drvPath drvName drv = do+  sourceStorePaths <- getDerivationSources store drv+  inputDrvStorePaths <- getDerivationInputs store drv+  outputs <- getDerivationOutputs store drvName drv   env <- getDerivationEnv drv   platform <- getDerivationPlatform drv   let requiredSystemFeatures = maybe [] splitFeatures $ M.lookup "requiredSystemFeatures" env       splitFeatures = filter (not . T.null) . T.split (== ' ') . decode       decode = decodeUtf8With lenientDecode+  inputDrvPaths <- for inputDrvStorePaths \(storePath, inputOutputs) -> do+    path <- Hercules.CNix.storePathToPath store storePath+    pure (path, inputOutputs)+  sourcePaths <- for sourceStorePaths (Hercules.CNix.storePathToPath store)+  outputs' <- for outputs \output -> do+    path <- for (derivationOutputPath output) (Hercules.CNix.storePathToPath store)+    let isFixed = case derivationOutputDetail output of+          (DerivationOutputInputAddressed _s) -> False+          (DerivationOutputCAFixed _f _s) -> True+          (DerivationOutputCAFloating _f _h) -> False+          DerivationOutputDeferred -> False+    pure+      ( decode $ derivationOutputName output,+        DerivationInfo.OutputInfo+          { DerivationInfo.path = decode <$> path,+            DerivationInfo.isFixed = isFixed+          }+      )   pure $     DerivationInfo-      { derivationPath = drvPath,+      { derivationPath = decode drvPath,         platform = decode platform,         requiredSystemFeatures = requiredSystemFeatures,         inputDerivations = inputDrvPaths & map (bimap decode (map decode)) & M.fromList,         inputSources = map decode sourcePaths,         outputs =-          outputs-            & map-              ( \output ->-                  ( decode $ derivationOutputName output,-                    DerivationInfo.OutputInfo-                      { DerivationInfo.path = decode $ derivationOutputPath output,-                        DerivationInfo.isFixed = derivationOutputHashAlgo output /= ""-                      }-                  )-              )-            & M.fromList+          outputs' & M.fromList       }

@@ -15,7 +15,7 @@   | AttributeError AttributeError   | EvaluationDone   | Error Text-  | Build Text Text (Maybe UUID)+  | Build ByteString Text (Maybe UUID)   | BuildResult BuildResult   | EffectResult Int   | Exception Text

@@ -8,10 +8,9 @@ import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BL import qualified Data.Map as M-import Foreign (ForeignPtr) import Hercules.Agent.Sensitive (Sensitive (Sensitive, reveal), revealContainer)+import Hercules.CNix (Derivation) import Hercules.CNix.Store (getDerivationArguments, getDerivationBuilder, getDerivationEnv)-import Hercules.CNix.Store.Context (Derivation) import Hercules.Effect.Container (BindMount (BindMount)) import qualified Hercules.Effect.Container as Container import Hercules.Error (escalateAs)@@ -63,7 +62,7 @@                       { data_ = Formats.Secret.data_ secret                       } -runEffect :: (MonadThrow m, KatipContext m) => ForeignPtr Derivation -> Sensitive Text -> FilePath -> Text -> FilePath -> m ExitCode+runEffect :: (MonadThrow m, KatipContext m) => Derivation -> Sensitive Text -> FilePath -> Text -> FilePath -> m ExitCode runEffect derivation token secretsPath apiBaseURL dir = do   drvBuilder <- liftIO $ getDerivationBuilder derivation   drvArgs <- liftIO $ getDerivationArguments derivation

@@ -15,6 +15,7 @@ import Protolude import System.Directory (createDirectory) import System.FilePath ((</>))+import System.IO (hClose) import System.IO.Error (ioeGetErrorType) import System.Posix.IO (closeFd, fdToHandle) import System.Posix.Terminal (openPseudoTerminal)@@ -92,27 +93,21 @@     Right a -> pure a     Left e -> throwIO (FatalError $ "decoding runc config.json template: " <> show e)   let configJson = effectToRuncSpec config template-  BS.writeFile (configJsonPath) (BL.toStrict $ encode configJson)+  BS.writeFile configJsonPath (BL.toStrict $ encode configJson)   createDirectory rootfsPath   createDirectory runcRootPath   name <- do     uuid <- UUID.nextRandom     pure $ "hercules-ci-" <> show uuid-  (exitCode, _) <- bracket-    openPseudoTerminal-    ( \(fd1, fd2) -> handle (\(_e :: SomeException) -> pass) do-        closeFd fd1-        when (fd2 /= fd1) (closeFd fd2)-    )-    $ \(master, terminal) -> do+  (exitCode, _) <- withPseudoTerminalHandles $+    \(master, terminal) -> do       concurrently         ( do-            terminalHandle <- fdToHandle terminal             let createProcSpec =                   (System.Process.proc runcExe ["--root", runcRootPath, "run", name])-                    { std_in = UseHandle terminalHandle, -- can't pass /dev/null :(-                      std_out = UseHandle terminalHandle,-                      std_err = UseHandle terminalHandle,+                    { std_in = UseHandle terminal, -- can't pass /dev/null :(+                      std_out = UseHandle terminal,+                      std_err = UseHandle terminal,                       cwd = Just dir                     }             withCreateProcess createProcSpec \_subStdin _noOut _noErr processHandle -> do@@ -128,9 +123,8 @@                               )         )         ( do-            masterHandle <- fdToHandle master             let shovel =-                  handleEOF (BS.hGetLine masterHandle) >>= \case+                  handleEOF (BS.hGetLine master) >>= \case                     "" -> pass                     someBytes | "@nix" `BS.isPrefixOf` someBytes -> do                       -- TODO use it (example @nix { "action": "setPhase", "phase": "effectPhase" })@@ -142,3 +136,29 @@             shovel         )   pure exitCode++-- | Like 'openPseudoTerminalHandles' but closes the handles after the+-- function is done.+withPseudoTerminalHandles :: ((Handle, Handle) -> IO a) -> IO a+withPseudoTerminalHandles =+  bracket+    openPseudoTerminalHandles+    ( \(master, terminal) -> do+        hClose master `catch` \(_ :: SomeException) -> pass+        hClose terminal `catch` \(_ :: SomeException) -> pass+    )++-- | Like 'openPseudoTerminal' but returning handles, in a resource-safe manner.+openPseudoTerminalHandles :: IO (Handle, Handle)+openPseudoTerminalHandles =+  mask_ do+    (masterFd, terminalFd) <- openPseudoTerminal++    ( do+        master <- fdToHandle masterFd+        terminal <- fdToHandle terminalFd+        pure (master, terminal)+      )+      `onException` do+        closeFd masterFd+        when (terminalFd /= masterFd) (closeFd terminalFd)

@@ -1,23 +1,33 @@+{-# LANGUAGE BlockArguments #-} {-# LANGUAGE OverloadedStrings #-}  module Hercules.Agent.Nix.RetrieveDerivationInfoSpec where +import qualified Data.ByteString as BS import qualified Data.Map as M import Hercules.API.Agent.Evaluate.EvaluateEvent.DerivationInfo import Hercules.Agent.Nix.RetrieveDerivationInfo import Hercules.CNix import Protolude+import System.IO.Temp (withSystemTempDirectory) import Test.Hspec  spec :: Spec spec = do   describe "retrieveDerivationInfo" $ do-    it "parses vm-test-run-agent-test.drv correctly" $ do-      drv <- getDerivationFromFile "testdata/vm-test-run-agent-test.drv"-      d <- retrieveDerivationInfo' "testdata/vm-test-run-agent-test.drv" drv-      derivationPath d `shouldBe` "testdata/vm-test-run-agent-test.drv"-      platform d `shouldBe` "x86_64-linux"-      requiredSystemFeatures d `shouldBe` ["kvm", "nixos-test"]-      inputDerivations d `shouldBe` M.fromList [("/nix/store/0si75icim8ajxcsp25d9c52m42kqg1xj-stdenv-linux.drv", ["out"]), ("/nix/store/1kircip4wskspsqqzxbmh6ss73iqh9ah-bash-4.4-p23.drv", ["out"]), ("/nix/store/5kpp7mly0qad7l451xhr60k0wbv6vivi-jquery-1.11.3.drv", ["out"]), ("/nix/store/id49f28qvggmzpmlxp511v6yhc070lkf-jquery-ui-1.11.4.drv", ["out"]), ("/nix/store/kyxayhm3r6gpzbg1d57fwb8m7gkmpwcy-nixos-test-driver-agent-test.drv", ["out"]), ("/nix/store/qzvim3ca1s5zxbbvpzzipizfnfpdf5r2-libxslt-1.1.33.drv", ["dev"])]-      inputSources d `shouldBe` ["/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh", "/nix/store/fs6a6m9s5n367dslsvsl9lg89h3ns3ya-logfile.css", "/nix/store/jjh6h82n5rhw45badlpbj1yv7y6m48h2-log2html.xsl", "/nix/store/kdangwwh47ngwxk859ibbvkj4vmm9rzr-treebits.js"]-      outputs d `shouldBe` M.fromList [("out", OutputInfo {path = "/nix/store/7vg7ij5933dhg6dh22fvs90bbzmzg8kj-vm-test-run-agent-test", isFixed = False})]+    it "parses vm-test-run-agent-test.drv correctly" $+      withTempStore $ \store -> do+        s <- BS.readFile "testdata/vm-test-run-agent-test.drv"+        drv <- getDerivationFromString store "vm-test-run-agent-test" s+        d <- retrieveDerivationInfo' store "testdata/vm-test-run-agent-test.drv" "vm-test-run-agent-test" drv+        derivationPath d `shouldBe` "testdata/vm-test-run-agent-test.drv"+        platform d `shouldBe` "x86_64-linux"+        requiredSystemFeatures d `shouldBe` ["kvm", "nixos-test"]+        inputDerivations d `shouldBe` M.fromList [("/nix/store/0si75icim8ajxcsp25d9c52m42kqg1xj-stdenv-linux.drv", ["out"]), ("/nix/store/1kircip4wskspsqqzxbmh6ss73iqh9ah-bash-4.4-p23.drv", ["out"]), ("/nix/store/5kpp7mly0qad7l451xhr60k0wbv6vivi-jquery-1.11.3.drv", ["out"]), ("/nix/store/id49f28qvggmzpmlxp511v6yhc070lkf-jquery-ui-1.11.4.drv", ["out"]), ("/nix/store/kyxayhm3r6gpzbg1d57fwb8m7gkmpwcy-nixos-test-driver-agent-test.drv", ["out"]), ("/nix/store/qzvim3ca1s5zxbbvpzzipizfnfpdf5r2-libxslt-1.1.33.drv", ["dev"])]+        inputSources d `shouldBe` ["/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh", "/nix/store/fs6a6m9s5n367dslsvsl9lg89h3ns3ya-logfile.css", "/nix/store/jjh6h82n5rhw45badlpbj1yv7y6m48h2-log2html.xsl", "/nix/store/kdangwwh47ngwxk859ibbvkj4vmm9rzr-treebits.js"]+        outputs d `shouldBe` M.fromList [("out", OutputInfo {path = Just "/nix/store/7vg7ij5933dhg6dh22fvs90bbzmzg8kj-vm-test-run-agent-test", isFixed = False})]++withTempStore :: (Store -> IO a) -> IO a+withTempStore f =+  withSystemTempDirectory "cnix-test-store" \d ->+    withStoreFromURI (toS d) f