nixfromnpm 0.2.1 → 0.10.1
raw patch · 24 files changed
+1049/−1585 lines, 24 filesdep +SHAdep +ansi-terminaldep +curldep −error-listdep −filepathdep −githubdep ~hnixbinary-added
Dependencies added: SHA, ansi-terminal, curl, data-fix, lifted-base, monad-control, semver-range, temporary, transformers, unix
Dependencies removed: error-list, filepath, github, hspec, hspec-expectations, http-client-streams, io-streams, nixfromnpm
Dependency ranges changed: hnix
Files
- LICENSE +21/−0
- nix-libs/nodeLib/buildNodePackage.nix +521/−0
- nix-libs/nodeLib/checkPackageJson.js +18/−0
- nix-libs/nodeLib/default.nix +152/−0
- nix-libs/nodeLib/fetch.py +21/−0
- nix-libs/nodeLib/fetchUrlWithHeaders.nix +71/−0
- nix-libs/nodeLib/installBinaries.py +41/−0
- nix-libs/nodeLib/npm3.tar.gz binary
- nix-libs/nodeLib/removeImpureDependencies.js +49/−0
- nixfromnpm.cabal +37/−78
- src/Main.hs +10/−5
- src/NixFromNpm.hs +0/−21
- src/NixFromNpm/Common.hs +0/−177
- src/NixFromNpm/ConvertToNix.hs +0/−295
- src/NixFromNpm/NpmLookup.hs +0/−478
- src/NixFromNpm/NpmTypes.hs +0/−108
- src/NixFromNpm/NpmVersion.hs +0/−17
- src/NixFromNpm/Options.hs +0/−73
- src/NixFromNpm/Parsers/Common.hs +0/−49
- src/NixFromNpm/Parsers/NpmVersion.hs +0/−79
- src/NixFromNpm/Parsers/SemVer.hs +0/−87
- src/NixFromNpm/SemVer.hs +0/−117
- test/Spec.hs +0/−1
- top_packages.txt +108/−0
+ LICENSE view
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Allen Nelson++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
+ nix-libs/nodeLib/buildNodePackage.nix view
@@ -0,0 +1,521 @@+{+ # System packages.+ pkgs,+ # Derivation for nodejs and npm.+ nodejs,+ # Which version of npm to use.+ npm ? nodejs,+ # List of required native build inputs.+ neededNatives,+ # Self-reference for overriding purposes.+ buildNodePackage,+}:++let+ inherit (pkgs) stdenv;+ inherit (pkgs.lib) showVal;+ # This expression builds the raw C headers and source files for the base+ # node.js installation. Node packages which use the C API for node need to+ # link against these files and use the headers.+ nodejsSources = pkgs.runCommand "node-sources" {} ''+ tar --no-same-owner --no-same-permissions -xf ${nodejs.src}+ mv $(find . -type d -mindepth 1 -maxdepth 1) $out+ '';++ # The path within $out/lib to find a package. If the package does not+ # have a namespace, it will simply be in `node_modules`, and otherwise it+ # will appear in `node_modules/@namespace`.+ modulePath = pkg:+ if !(builtins.hasAttr "namespace" pkg)+ then throw ''+ Package dependency does not appear to be a node package: ${showVal pkg}.+ Use "buildInputs" or "propagatedBuildInputs" for non-node dependencies,+ instead of "deps".+ ''+ else if pkg.namespace == null then "node_modules"+ else "node_modules/@${pkg.namespace}";++ # The path to the package within its modulePath. Just appending the name+ # of the package.+ pathInModulePath = pkg: "${modulePath pkg}/${pkg.basicName}";++ # By default, when checking we'll run npm test.+ defaultCheckPhase = ''+ runHook preCheck+ npm test+ runHook postCheck+ '';+in++{+ # The name of the package. If it's a private package with a namespace,+ # this should not contain the namespace.+ name,++ # Used for private packages. Indicated in the name field of the+ # package.json, e.g. if the name in that file is "@mynamespace/mypackage",+ # then the namespace is 'mynamespace' and the name is 'mypackage'. Public+ # packages will not need this.+ namespace ? null,++ # Version of the package. This should follow the semver standard, although+ # we don't explicitly enforce that in this function.+ version,++ # Source of the package; can be a tarball or a folder on the filesystem.+ src,++ # The suffix to the name as it appears in `nix-env -q` and the nix store. By+ # default, the name of nodejs interpreter e.g:+ # "<package name>-<package version>-nodejs-<nodejs version>"+ nameSuffix ? "-${nodejs.name}",++ # If there's a namespace, by default it will be prepended to the package+ # name. Otherwise, a prefix can be given explicitly.+ namePrefix ? (if namespace == null then "" else "${namespace}-"),++ # List or attribute set of (runtime) dependencies.+ deps ? {},++ # List or attribute set of peer dependencies. See:+ # https://nodejs.org/en/blog/npm/peer-dependencies/+ peerDependencies ? {},++ # List or attribute set of optional dependencies.+ optionalDependencies ? {},++ # List of optional dependencies to skip. List of strings, where a string+ # should contain the `name` of the derivation to skip (not a version or+ # namespace).+ skipOptionalDependencies ? [],++ # List or set of development dependencies (or null). These will only be+ # installed when `installDevDependencies` is true, which is provided by+ # the `.env` attribute.+ devDependencies ? null,++ # If true and devDependencies are defined, the package will only be+ # installed contingent on successfully running tests.+ doCheck ? false,++ # Test command.+ checkPhase ? defaultCheckPhase,++ # Additional flags passed to npm install. A list of strings.+ extraNpmFlags ? [],++ # Same as https://docs.npmjs.com/files/package.json#os+ os ? [],++ # Attribute set of already resolved dependencies, for avoiding infinite+ # recursion. Used internally (don't use this argument explicitly).+ resolvedDeps ? {},++ # Build inputs to propagate in addition to nodejs and non-dev dependencies.+ propagatedBuildInputs ? [],++ # Build inputs in addition to npm and dev dependencies.+ buildInputs ? [],++ # Whether to strip debugging symbols from binaries and patch ELF executables.+ # These should both probably be true but can be overridden.+ # Doc for details: https://nixos.org/wiki/NixPkgs_Standard_Environment.+ dontStrip ? true, dontPatchELF ? true,++ # Optional attributes to pass through to downstream derivations.+ passthru ? {},++ # A set of dependencies to patch.+ patchDependencies ? {},++ # Any remaining flags are passed through to mkDerivation.+ ...+} @ args:++let+ # The package name as it appears in the package.json. This contains a+ # namespace if there is one, so it will be a distinct identifier for+ # different packages.+ packageJsonName = if namespace == null then name+ else "@${namespace}/${name}";++ # The package name with a version appended. This should be unique amongst+ # all packages.+ uniqueName = "${packageJsonName}@${version}";++in++if doCheck && (devDependencies == null)+then throw ("${uniqueName}: Can't run tests because devDependencies have " ++ "not been defined.")+else++let+ inherit (stdenv.lib) fold removePrefix hasPrefix subtractLists isList flip+ intersectLists isAttrs listToAttrs nameValuePair+ mapAttrs filterAttrs attrNames elem concatMapStrings+ attrValues getVersion flatten remove concatStringsSep;++ dependencyTypes = ["dependencies" "devDependencies" "peerDependencies"+ "optionalDependencies"];+++ # These arguments are intended as directives to this function and not+ # to be passed through to mkDerivation. They are removed below.+ attrsToRemove = ["deps" "resolvedDeps" "flags" "os" "skipOptionalDependencies"+ "passthru" "doCheck" "installDevDependencies" "version"+ "namespace" "patchDependencies"] ++ dependencyTypes;++ # We create a `self` object for self-referential expressions. It+ # bottoms out in a call to `mkDerivation` at the end.+ self = let+ platforms = if os == [] then nodejs.meta.platforms else+ fold (entry: platforms:+ let+ filterPlatforms =+ stdenv.lib.platforms.${removePrefix "!" entry} or [];+ in+ # Ignore unknown platforms+ if filterPlatforms == [] then (if platforms == [] then nodejs.meta.platforms else platforms)+ else+ if hasPrefix "!" entry then+ subtractLists (intersectLists filterPlatforms nodejs.meta.platforms) platforms+ else+ platforms ++ (intersectLists filterPlatforms nodejs.meta.platforms)+ ) [] os;+++ strict = x: builtins.seq x x;++ toAttrSet = obj: if isAttrs obj then obj else+ (listToAttrs (map (x: nameValuePair x.name x) obj));++ mapDependencies = deps: filterFunc: let+ attrDeps = toAttrSet deps;+ in rec {+ # All required node modules, without already resolved dependencies+ # Also override with already resolved dependencies+ requiredDeps = strict (mapAttrs (name: dep:+ dep.overrideNodePackage {+ resolvedDeps = resolvedDeps // {"${name}" = self;};+ }+ ) (filterAttrs filterFunc+ (removeAttrs attrDeps (attrNames resolvedDeps))));++ # Recursive dependencies that we want to avoid with shim creation+ recursiveDeps = strict (filterAttrs filterFunc+ (removeAttrs attrDeps (attrNames requiredDeps)));+ };++ # Filter out self-referential dependencies.+ _dependencies = strict (mapDependencies deps (name: dep: dep.uniqueName != uniqueName));++ # Filter out self-referential peer dependencies.+ _peerDependencies = strict (mapDependencies peerDependencies (name: dep:+ dep.uniqueName != uniqueName));++ # Filter out any optional dependencies which don't build correctly.+ _optionalDependencies = strict (mapDependencies optionalDependencies (name: dep:+ (builtins.tryEval dep).success &&+ !(elem dep.basicName skipOptionalDependencies)+ ));++ # Grab development dependencies if doCheck is true.+ _devDependencies = let+ filterFunc = name: dep: dep.uniqueName != uniqueName;+ depSet = if doCheck then devDependencies else [];+ in+ strict (mapDependencies depSet filterFunc);++ # Depencencies we need to propagate (all except devDependencies)+ propagatedDependencies = strict (+ _dependencies.requiredDeps //+ _optionalDependencies.requiredDeps //+ _peerDependencies.requiredDeps);++ # Required dependencies are those that we haven't filtered yet.+ requiredDependencies =+ strict (_devDependencies.requiredDeps // propagatedDependencies);++ # Recursive dependencies. These are turned into "shims" or fake packages,+ # which allows us to have dependency cycles, something npm allows.+ recursiveDependencies = strict (+ _devDependencies.recursiveDeps //+ _dependencies.recursiveDeps //+ _optionalDependencies.recursiveDeps //+ _peerDependencies.recursiveDeps);++ # Flags that we will pass to `npm install`.+ npmFlags = concatStringsSep " " ([+ # We point the registry at something that doesn't exist. This will+ # mean that NPM will fail if any of the dependencies aren't met, as it+ # will attempt to hit this registry for the missing dependency.+ "--registry=http://notaregistry.$UNIQNAME.derp"+ # These flags make failure fast, as otherwise NPM will spin for a while.+ "--fetch-retry-mintimeout=0" "--fetch-retry-maxtimeout=10" "--fetch-retries=0"+ # This will disable any user-level npm configuration.+ "--userconfig=/dev/null"+ # This flag is used for packages which link against the node headers.+ "--nodedir=${nodejsSources}"+ # This will tell npm not to run pre/post publish hooks+ # "--ignore-scripts"+ ] +++ # Use the --production flag if we're not running tests; this will make+ # npm skip the dev dependencies.+ (if !doCheck then ["--production"] else []) +++ # Add any extra headers that the user has passed in.+ extraNpmFlags);++ # A bit of bash to check that variables are set.+ checkSet = vars: concatStringsSep "\n" (flip map vars (var: ''+ [[ -z $${var} ]] && { echo "${var} is not set."; exit 1; }+ ''));++ patchPhase = ''+ runHook prePatch+ patchShebangs $PWD++ # Ensure that the package name matches what is in the package.json.+ node ${./checkPackageJson.js} ${packageJsonName}++ # Remove any impure dependencies from the package.json (see script+ # for details)+ node ${./removeImpureDependencies.js}++ # We do not handle shrinkwraps yet+ rm npm-shrinkwrap.json 2>/dev/null || true++ ${if patchDependencies == {} then "" else ''+ cat <<EOF | python+ import json+ with open("package.json") as f:+ package_json = json.load(f)+ ${flip concatMapStrings (attrNames patchDependencies) (name: let+ version = patchDependencies.${name};+ in flip concatMapStrings dependencyTypes (depType: ''+ if "${name}" in package_json.setdefault("${depType}", {}):+ ${if version == null then ''+ print("removing ${name} from ${depType}")+ package_json["${depType}"].pop("${name}", None)+ '' else ''+ print("Patching ${depType} ${name} to version ${version}")+ package_json["dependencies"]["${name}"] = "${version}"+ ''}+ ''))}+ with open("package.json", "w") as f:+ f.write(json.dumps(package_json))+ EOF+ ''}++ runHook postPatch+ '';++ configurePhase = ''+ runHook preConfigure+ # Symlink or copy dependencies for node modules+ # copy is needed if dependency has recursive dependencies,+ # because node can't follow symlinks while resolving recursive deps.+ ${+ let+ linkCmd = p: if p.recursiveDeps == [] then "ln -sv" else "cp -r";+ link = dep: ''+ if ! [[ -e ${pathInModulePath dep} ]]; then+ ${linkCmd dep} ${dep}/lib/${pathInModulePath dep} ${modulePath dep}+ if [[ -d ${dep}/bin ]]; then+ find -L ${dep}/bin -maxdepth 1 -type f -executable \+ | while read exec_file; do+ echo "Symlinking $exec_file binary to node_modules/.bin"+ mkdir -p node_modules/.bin+ ln -s $exec_file node_modules/.bin/$(basename $exec_file)+ done+ fi+ fi+ '';+ in+ flip concatMapStrings (attrValues requiredDependencies) (dep:+ # Create symlinks (or copies) of all of the required dependencies.+ ''+ mkdir -p ${modulePath dep}+ ${link dep}+ ${concatMapStrings link (attrValues dep.peerDependencies)}+ '')}++ # Remove recursive dependencies from package.json+ ${if recursiveDependencies == {} then "" else ''+ python <<EOF+ import json+ with open("package.json") as f:+ package_json = json.load(f)+ dep_names = [${concatStringsSep ", "+ (map (d: "'${d.packageJsonName}'")+ (attrValues recursiveDependencies))}]+ for name in dep_names:+ print("Removing recursive dependency on {} from package.json"+ .format(name))+ for k in ("dependencies", "devDependencies", "peerDependencies",+ "optionalDependencies"):+ package_json[k] = package_json.setdefault(k, {})+ package_json[k].pop(name, None)+ with open("package.json", "w") as f:+ f.write(json.dumps(package_json))+ EOF+ ''}++ runHook postConfigure+ '';++ buildPhase = ''+ runHook preBuild+ (+ # NPM reads the `HOME` environment variable and fails if it doesn't+ # exist, so set it here.+ export HOME=$PWD+ echo npm install $npmFlags++ npm install $npmFlags || {+ echo "NPM installation of ${name}@${version} failed!"+ echo "Rerunning with verbose logging:"+ npm install . $npmFlags --loglevel=verbose+ if [[ -d node_modules ]]; then+ echo "node_modules contains these files:"+ ls -l node_modules+ fi+ exit 1+ }+ )+ runHook postBuild+ '';++ installPhase = ''+ runHook preInstall++ # Install the package that we just built.+ mkdir -p $out/lib/${modulePath self}++ # Copy the folder that was created for this path to $out/lib.+ cp -r $PWD $out/lib/${pathInModulePath self}++ # Remove the node_modules subfolder from there, and instead put things+ # in $PWD/node_modules into that folder.+ if [ -e "$out/lib/${pathInModulePath self}/man" ]; then+ echo "Linking manpages..."+ mkdir -p $out/share+ for dir in $out/lib/${pathInModulePath self}/man/*; do #*/+ mkdir -p $out/share/man/$(basename "$dir")+ for page in $dir/*; do #*/+ ln -sv $page $out/share/man/$(basename "$dir")+ done+ done+ fi++ # Move peer dependencies to node_modules+ ${concatMapStrings (dep: ''+ mkdir -p ${modulePath dep}+ mv ${pathInModulePath dep} $out/lib/${modulePath dep}+ '') (attrValues _peerDependencies.requiredDeps)}++ # Install binaries using the `bin` object in the package.json+ python ${./installBinaries.py}++ runHook postInstall+ '';++ # These are the arguments that we will pass to `stdenv.mkDerivation`.+ mkDerivationArgs = {+ inherit+ src+ patchPhase+ configurePhase+ buildPhase+ checkPhase+ installPhase+ doCheck;++ # Tell mkDerivation to run `setVariables` prior to other phases.+ prePhases = ["setVariables"];++ # Define some environment variables that we will use in the build.+ setVariables = ''+ export HASHEDNAME=$(echo "$propagatedNativeBuildInputs $name" \+ | md5sum | awk '{print $1}')+ export UNIQNAME="''${HASHEDNAME:0:10}-${name}-${version}"+ export BUILD_DIR=$TMPDIR/$UNIQNAME-build+ export npmFlags="${npmFlags}"+ '';++ shellHook = ''+ runHook preShellHook+ runHook setVariables+ export PATH=${npm}/bin:${nodejs}/bin:$(pwd)/node_modules/.bin:$PATH+ mkdir -p $TMPDIR/$UNIQNAME+ (+ cd $TMPDIR/$UNIQNAME+ eval "$configurePhase"+ )+ echo "Installed dependencies in $TMPDIR/$UNIQNAME."+ export PATH=$TMPDIR/$UNIQNAME/node_modules/.bin:$PATH+ export NODE_PATH=$TMPDIR/$UNIQNAME/node_modules+ runHook postShellHook+ '';++ inherit dontStrip dontPatchELF;++ meta = {+ inherit platforms;+ maintainers = [ stdenv.lib.maintainers.offline ];+ };++ # Propagate pieces of information about the package so that downstream+ # packages can reflect on them.+ passthru = (passthru // {+ inherit uniqueName packageJsonName namespace version;+ # The basic name is the name without namespace or version.+ basicName = name;+ peerDependencies = _peerDependencies.requiredDeps;++ # Expose a list of recursive dependencies to upstream packages, so that+ # they can be shimmed out.+ recursiveDeps = let+ required = attrValues requiredDependencies;+ recursive = attrNames recursiveDependencies;+ in+ flatten (map (dep: remove name dep.recursiveDeps) required) +++ recursive;++ # The `env` attribute is meant to be used with `nix-shell` (although+ # that's not required). It will build the package with its dev+ # dependencies. This means that the package must have dev dependencies+ # defined, or it will error.+ env = buildNodePackage (args // {installDevDependencies = true;});++ # An 'overrideNodePackage' attribute, which will call+ # `buildNodePackage` with the given arguments overridden.+ # We don't use the name `override` because this will get stomped on+ # if the derivation is the result of a `callPackage` application.+ overrideNodePackage = newArgs: buildNodePackage (args // newArgs);+ });+ } // (removeAttrs args attrsToRemove) // {+ name = "${namePrefix}${name}-${version}${nameSuffix}";++ # Pass the required dependencies and+ propagatedBuildInputs = propagatedBuildInputs +++ attrValues propagatedDependencies +++ [nodejs pkgs.tree];+++ buildInputs = [npm] ++ buildInputs +++ attrValues (_devDependencies.requiredDeps) +++ neededNatives;++ # Expose list of recursive dependencies upstream, up to the package that+ # caused recursive dependency+ recursiveDeps =+ (flatten (+ map (dep: remove name dep.recursiveDeps) (attrValues requiredDependencies)+ )) +++ (attrNames recursiveDependencies);+ };++ in stdenv.mkDerivation mkDerivationArgs;++in self
+ nix-libs/nodeLib/checkPackageJson.js view
@@ -0,0 +1,18 @@+// Given an expected name of a package, checks that this matches the+// name declared in the package.json file.++// Pull the expected name out of the arguments. For example, the expected+// name might be 'foo' or '@foo/bar'.+var fs = require('fs');+var expectedName = process.argv[2];++// Load up the package object.+var packageObj = JSON.parse(fs.readFileSync('./package.json'));++// Ensure that they match.+if (expectedName !== packageObj['name']) {+ console.error("Package name declared in package.json ("+ + packageObj['name'] + ") does not match expected name ("+ + expectedName + ")");+ process.exit(1);+}
+ nix-libs/nodeLib/default.nix view
@@ -0,0 +1,152 @@+/*+A set of tools for generating node packages, such as to be imported by+default.nix files generated by nixfromnpm.+*/++{+ # Self-reference so that we can pass through to downstream libraries+ self+}:++{+ # Base set of packages, i.e. nixpkgs.+ pkgs,+ # nodejs derivation.+ nodejs ? pkgs.nodejs-4_x,+ # Whether to use npm3 (requires a prebuilt tarball of npm3).+ npm3 ? true+} @ args:++let+ # Function to replace dots with something+ replaceDots = c: replaceChars ["."] [c];+ inherit (builtins) readDir removeAttrs length getEnv elemAt hasAttr;+ inherit (pkgs.lib) attrNames attrValues filterAttrs flip foldl+ hasSuffix hasPrefix removeSuffix replaceChars+ optional optionals stringToCharacters+ concatStrings tail splitString;+ inherit (pkgs.stdenv) isLinux;++ # Function to remove the first character of a string.+ dropFirstChar = str: concatStrings (tail (stringToCharacters str));++ # Concatenate a list of sets.+ joinSets = foldl (a: b: a // b) {};++ # Builds the extracted nix file. Since of course it can't use npm3,+ # being that it hasn't been built yet, we disable npm3 for this.+ _npm3 = ((self (args // {npm3 = false;}))+ .generatePackages {nodePackagesPath = ../nodePackages;})+ .nodePackages.npm;++ # Parse the `NPM_AUTH_TOKENS` environment variable to discover+ # namespace-token associations and turn them into an attribute set+ # which we can use as an input to the fetchPrivateNpm function.+ # Split the variable on ':', then turn each k=v element in+ # the list into an attribute set and join all of those sets.+ namespaceTokens = joinSets (+ flip map (splitString ":" (getEnv "NPM_AUTH_TOKENS")) (kvPair:+ let kv = splitString "=" kvPair; in+ if length kv != 2 then {}+ else {"${elemAt kv 0}" = elemAt kv 1;}));++ # A function similar to fetchUrl but allows setting of custom headers.+ fetchUrlWithHeaders = pkgs.callPackage ./fetchUrlWithHeaders.nix {};++ # Uses the parsed namespace tokens to create a function that can+ # fetch a private package from an npm repo.+ fetchPrivateNpm = {namespace, headers ? {}, ...}@args:+ if !(hasAttr namespace namespaceTokens)+ then throw "NPM_AUTH_TOKENS does not contain namespace ${namespace}"+ else let+ Authorization = "Bearer ${namespaceTokens.${namespace}}";+ headers = {inherit Authorization;} // headers;+ in+ fetchUrlWithHeaders (removeAttrs args ["namespace"] // {inherit headers;});+in++rec {+ inherit nodejs;++ buildNodePackage = import ./buildNodePackage.nix ({+ inherit pkgs nodejs buildNodePackage;+ neededNatives = [pkgs.python] ++ optionals isLinux [pkgs.utillinux];+ } // (if npm3 then {npm = _npm3;} else {}));+ # A generic package that will fail to build. This is used to indicate+ # packages that are broken, without failing the entire generation of+ # a package expression.+ brokenPackage = {name, reason}:+ let+ deriv = pkgs.stdenv.mkDerivation {+ name = "BROKEN-${name}";+ buildCommand = ''+ echo "Package ${name} is broken: ${reason}"+ exit 1+ '';+ passthru.withoutTests = deriv;+ passthru.pkgName = name;+ passthru.basicName = "BROKEN";+ passthru.uniqueName = "BROKEN";+ passthru.overrideNodePackage = (_: deriv);+ passthru.namespace = null;+ passthru.version = "BROKEN";+ passthru.override = _: deriv;+ passthru.recursiveDeps = [];+ passthru.peerDependencies = {};+ };+ in+ deriv;++ # The function that a default.nix can call into which will scan its+ # directory for all of the package files and generate a big attribute set+ # for all of them. Re-exports the `callPackage` function and all of the+ # attribute sets, as well as the nodeLib.+ generatePackages = {+ # Path to find node packages in.+ nodePackagesPath,+ # Extensions are other node libraries which will be folded into the+ # generated one.+ extensions ? [],+ # If any additional arguments should be made available to callPackage+ # (for example for packages which require additional arguments), they+ # can be passed in here. Those packages can declare an `extras` argument+ # which will contain whatever is passed in here.+ extras ? {}+ }:+ let+ callPackageWith = pkgSet: path: overridingArgs: let+ inherit (builtins) intersectAttrs functionArgs;+ inherit (pkgs.lib) filterAttrs;+ # The path must be a function; import it here.+ func = import path;+ # Get the arguments to the function; e.g. "{a=false; b=true;}", where+ # a false value is an argument that has no default.+ funcArgs = functionArgs func;+ # Take only the arguments that don't have a default.+ noDefaults = filterAttrs (_: v: v == false) funcArgs;+ # Intersect this set with the package set to create the arguments to+ # the function.+ satisfyingArgs = intersectAttrs noDefaults pkgSet;+ # Override these arguments with whatever's passed in.+ actualArgs = satisfyingArgs // overridingArgs;+ # Call the function with these args to get a derivation.+ deriv = func actualArgs;+ in deriv;++ nodePackages = joinSets (map (e: e.nodePackages) extensions) //+ (import nodePackagesPath {inherit callPackage;});++ callPackage = callPackageWith {+ inherit fetchUrlWithHeaders namespaceTokens;+ inherit pkgs nodePackages buildNodePackage brokenPackage;+ inherit extras;+ inherit (nodePackages) namespaces;+ };+ in+ {+ inherit callPackage namespaceTokens pkgs;+ nodePackages = nodePackages // {inherit nodejs;};+ nodeLib = self args;+ npm3 = _npm3;+ };+}
+ nix-libs/nodeLib/fetch.py view
@@ -0,0 +1,21 @@+import os+import requests+out = os.environ['out']+url = os.environ['url']+headers = {"User-Agent": "nix-fetchurl"}+header_names = os.environ.get("headerNames", "")+for name in header_names.split():+ if "__HTTP_HEADER_{}".format(name) not in os.environ:+ exit("FATAL: no corresponding value set for header {}"+ .format(name))+ headers[name] = os.environ["__HTTP_HEADER_{}".format(name)]+print('GET {} with headers {}'.format(url, headers))+response = requests.get(url, headers=headers)+if response.status_code != 200:+ exit("Received a {} response. :(\nContent: {}"+ .format(response.status_code, response.content))+else:+ print('Response: {} ({} bytes)'+ .format(response.status_code, len(response.content)))+with open(out, 'wb') as f:+ f.write(response.content)
+ nix-libs/nodeLib/fetchUrlWithHeaders.nix view
@@ -0,0 +1,71 @@+# A python-based fetchurl function, allowing the passage of custom headers.+# Just calls into `requests` under the hood.+{+ pythonPackages, stdenv+}:+++{ # URL to fetch.+ url ? ""++, # Additional curl options needed for the download to succeed.+ curlOpts ? ""++, # Name of the file. If empty, use the basename of `url' (or of the+ # first element of `urls').+ name ? ""++ # Different ways of specifying the hash.+, outputHash ? ""+, outputHashAlgo ? ""+, md5 ? ""+, sha1 ? ""+, sha256 ? ""++, # Meta information, if any.+ meta ? {}++ # Headers to set, if any.+, headers ? {}+}:++let+ inherit (stdenv.lib) flip mapAttrs' nameValuePair;+ hasHash = (outputHash != "" && outputHashAlgo != "")+ || md5 != "" || sha1 != "" || sha256 != "";++ # Create an attribute set translating each header name and value into+ # the header name prefixed with __HTTP_HEADER. When the derivation is+ # evaluated, the script will pick up these environment variables and use+ # them to produce the actual headers.+ headerValues = flip mapAttrs' headers (headerName: headerValue:+ nameValuePair "__HTTP_HEADER_${headerName}" headerValue);+in++if !hasHash+then throw "You must specify the output hash for ${url}"+else++stdenv.mkDerivation ({+ inherit url;+ name = if name != "" then name else baseNameOf (toString url);++ outputHashAlgo = if outputHashAlgo != "" then outputHashAlgo else+ if sha256 != "" then "sha256" else if sha1 != "" then "sha1" else "md5";+ outputHash = if outputHash != "" then outputHash else+ if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;++ # Only flat hashing, which is the normal mode if you're fetching a file.+ outputHashMode = "flat";++ # Doing the download on a remote machine just duplicates network+ # traffic, so don't do that.+ preferLocalBuild = true;++ headerNames = builtins.attrNames headers;++ buildInputs = with pythonPackages; [python requests2];+ buildCommand = ''+ python ${./fetch.py}+ '';+} // headerValues)
+ nix-libs/nodeLib/installBinaries.py view
@@ -0,0 +1,41 @@+# This script will read the package.json file and create a symlink for each+# item under the `bin` key. The bin can be a string or a dictionary.+# See: https://docs.npmjs.com/files/package.json#bin+import json+import os+from os.path import join, isdir, normpath+import sys++with open('package.json', 'r') as f:+ package = json.load(f)++if 'bin' not in package:+ sys.exit(0)++_bin = package['bin']++if isinstance(_bin, basestring):+ # This is equivalent to a singleton dictionary where the key is the+ # name of the package.+ _bin = {package['name']: _bin}+elif not isinstance(_bin, dict):+ # Otherwise it must be a dictionary.+ sys.exit("Expected `bin` key in package.json to point to a string "+ "or a dict, but it is '{}', of type '{}'"+ .format(_bin, type(_bin).__name__))++out_dir = os.environ['out']++# Create the .bin folder+bin_folder = join(out_dir, 'bin')+if not isdir(bin_folder):+ os.makedirs(bin_folder)++print("Creating binaries in {}".format(bin_folder))++for bin_name, bin_path in _bin.items():+ # Get the absolute path of the script being pointed to.+ bin_abs_path = normpath(join(out_dir, 'lib', 'node_modules',+ package['name'], bin_path))+ print("Linking binary {} to {}".format(bin_name, bin_abs_path))+ os.symlink(bin_abs_path, join(bin_folder, bin_name))
+ nix-libs/nodeLib/npm3.tar.gz view
binary file changed (absent → 583680 bytes)
+ nix-libs/nodeLib/removeImpureDependencies.js view
@@ -0,0 +1,49 @@+// These packages come packaged with nodejs.+var fs = require('fs');+var url = require('url');++function versionSpecIsImpure(versionSpec) {+ // Returns true if a version spec is impure.+ return (versionSpec == "latest" || versionSpec == "unstable" ||+ // file path references+ versionSpec.substr(0, 2) == ".." ||+ versionSpec.substr(0, 2) == "./" ||+ versionSpec.substr(0, 2) == "~/" ||+ versionSpec.substr(0, 1) == '/' ||+ // github owner/repo references+ /^[^/]+\/[^/]+(#.*)?$/.test(versionSpec) ||+ // is a URL+ url.parse(versionSpec).protocol);+}++// Load up the package object.+var packageObj = JSON.parse(fs.readFileSync('./package.json'));++// Purify dependencies.+var depTypes = ['dependencies', 'devDependencies', 'optionalDependencies'];+for (var i in depTypes) {+ var depType = depTypes[i];+ var depSet = packageObj[depType];+ if (depSet !== undefined) {+ for (var depName in depSet) {+ var versionSpec = depSet[depName];+ if (versionSpecIsImpure(versionSpec)) {+ console.log("Replacing impure version spec " + versionSpec ++ " for dependency " + depName + " with '*'");+ depSet[depName] = '*';+ }+ }+ }+}++/* Remove peer dependencies */+if (process.env.removePeerDependencies && packageObj.peerDependencies) {+ console.log("WARNING: removing the following peer dependencies:");+ for (key in packageObj.peerDependencies) {+ console.log(" " + key + ": " + packageObj.peerDependencies[key]);+ }+ delete packageObj.peerDependencies;+}++/* Write the fixed JSON file */+fs.writeFileSync("package.json", JSON.stringify(packageObj));
nixfromnpm.cabal view
@@ -1,5 +1,5 @@ name: nixfromnpm-version: 0.2.1+version: 0.10.1 synopsis: Generate nix expressions from npm packages. description: Given an npm package name and one or more npm repositories, will dump out a@@ -9,62 +9,47 @@ using the same target directory will result in re-use of the existing files, to avoid unnecessary duplication. license: MIT+license-file: LICENSE author: Allen Nelson maintainer: anelson@narrativescience.com build-type: Simple cabal-version: >=1.10 bug-reports: https://github.com/adnelson/nixfromnpm/issues+Category: Tools, Nix +data-files: nix-libs/nodeLib/default.nix+ , nix-libs/nodeLib/buildNodePackage.nix+ , nix-libs/nodeLib/removeImpureDependencies.js+ , nix-libs/nodeLib/checkPackageJson.js+ , nix-libs/nodeLib/fetchUrlWithHeaders.nix+ , nix-libs/nodeLib/installBinaries.py+ , nix-libs/nodeLib/fetch.py+ , nix-libs/nodeLib/npm3.tar.gz+ , top_packages.txt+ source-repository head type: git location: git://github.com/adnelson/nixfromnpm.git -library- exposed-modules: NixFromNpm- other-modules: NixFromNpm.Common- , NixFromNpm.ConvertToNix- , NixFromNpm.NpmLookup- , NixFromNpm.NpmTypes- , NixFromNpm.NpmVersion- , NixFromNpm.Options- , NixFromNpm.SemVer- , NixFromNpm.Parsers.Common- , NixFromNpm.Parsers.NpmVersion- , NixFromNpm.Parsers.SemVer- build-depends: base >=4.8 && <4.9- , classy-prelude- , text- , mtl- , bytestring- , unordered-containers- , containers- , parsec- , aeson- , data-default- , shelly- , MissingH- , error-list- , text-render- , system-filepath- , filepath- , network-uri- , directory- , hnix- , optparse-applicative- hs-source-dirs: src- default-language: Haskell2010--- executable nixfromnpm main-is: Main.hs- -- other-modules:- other-extensions: NoImplicitPrelude, OverloadedStrings+ other-extensions: FlexibleContexts+ , FlexibleInstances+ , LambdaCase+ , NoImplicitPrelude+ , NoMonomorphismRestriction+ , OverloadedStrings+ , QuasiQuotes+ , RecordWildCards+ , ScopedTypeVariables+ , TypeFamilies+ , TypeSynonymInstances+ , ViewPatterns build-depends: base >=4.8 && <4.9 , classy-prelude , text- , mtl , bytestring+ , mtl , unordered-containers , containers , parsec@@ -72,47 +57,21 @@ , data-default , shelly , MissingH- , error-list , text-render , system-filepath- , filepath , network-uri , directory- , hnix- , nixfromnpm+ , hnix >=0.2.3 , optparse-applicative- -- extra-libraries: curl+ , curl+ , temporary+ , SHA+ , monad-control+ , lifted-base+ , transformers+ , unix+ , ansi-terminal+ , semver-range+ , data-fix hs-source-dirs: src default-language: Haskell2010--Test-Suite spec- Type: exitcode-stdio-1.0- Default-Language: Haskell2010- Hs-Source-Dirs: src- , test- Main-Is: Spec.hs- Build-Depends: base >=4.8 && <4.9- , classy-prelude- , mtl- , text- , bytestring- , unordered-containers- , containers- , parsec- , io-streams- , http-client-streams- , aeson- , data-default- , shelly- , MissingH- , github- , hspec- , hspec-expectations- , error-list- , text-render- , system-filepath- , filepath- , network-uri- , directory- , hnix- , optparse-applicative
src/Main.hs view
@@ -3,18 +3,23 @@ {-# LANGUAGE OverloadedStrings #-} module Main where +import qualified Data.Text.Encoding as T import Options.Applicative--import NixFromNpm hiding (getArgs, (<>))+import System.Exit +import NixFromNpm.Common hiding (getArgs, (<>))+import NixFromNpm.Options (NixFromNpmOptions, parseOptions,+ validateOptions)+import NixFromNpm.Conversion.ToDisk (dumpPkgFromOptions)+import NixFromNpm.Merge (mergeInto, MergeType(..), Source(..), Dest(..)) main :: IO () main = do- maybeToken <- getEnv "GITHUB_TOKEN"- let opts = info (helper <*> pOptions maybeToken)+ let opts = info (helper <*> parseOptions) (fullDesc <> progDesc description <> header headerText) parsedOpts <- execParser opts- dumpPkgFromOptions parsedOpts+ validatedOpts <- validateOptions parsedOpts+ exitWith =<< dumpPkgFromOptions validatedOpts where description = concat ["nixfromnpm allows you to generate nix expressions ", "automatically from npm packages. It provides ",
− src/NixFromNpm.hs
@@ -1,21 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE LambdaCase #-}-module NixFromNpm (module NixFromNpm.Common,- module NixFromNpm.Options,- module NixFromNpm.SemVer,- module NixFromNpm.Parsers.SemVer,- module NixFromNpm.NpmVersion,- module NixFromNpm.NpmTypes,- module NixFromNpm.Parsers.NpmVersion,- module NixFromNpm.NpmLookup,- module NixFromNpm.ConvertToNix) where--import NixFromNpm.Common-import NixFromNpm.Options-import NixFromNpm.SemVer-import NixFromNpm.Parsers.SemVer-import NixFromNpm.NpmVersion-import NixFromNpm.NpmTypes-import NixFromNpm.Parsers.NpmVersion-import NixFromNpm.NpmLookup-import NixFromNpm.ConvertToNix
− src/NixFromNpm/Common.hs
@@ -1,177 +0,0 @@-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE NoMonomorphismRestriction #-}-{-# LANGUAGE FlexibleContexts #-}-{-# LANGUAGE TypeSynonymInstances #-}-{-# LANGUAGE FlexibleInstances #-}-{-# LANGUAGE ViewPatterns #-}-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE TypeFamilies #-}-module NixFromNpm.Common (- module ClassyPrelude,- module Control.Applicative,- module Control.Exception,- module Control.Exception.ErrorList,- module Control.Monad,- module Control.Monad.Except,- module Control.Monad.Identity,- module Control.Monad.State.Strict,- module Control.Monad.Reader,- module Control.Monad.Trans,- module Data.Char,- module Data.Default,- module Data.HashMap.Strict,- module Data.Either,- module Data.Maybe,- module Data.List,- module Data.String.Utils,- module GHC.Exts,- module Filesystem.Path.CurrentOS,- module Network.URI,- module GHC.IO.Exception,- module System.Directory,- module Text.Render,- module System.FilePath.Posix,- Name, Record, Path,- tuple, tuple3, fromRight, cerror, cerror', uriToText, uriToString, slash,- putStrsLn, pathToText, putStrs, dropSuffix, maybeIf, grab, withDir,- pathToString, joinBy, mapJoinBy, getEnv, modifyMap- ) where--import ClassyPrelude hiding (assert, asList, find, FilePath, bracket,- maximum, maximumBy, try)-import qualified Prelude as P-import Control.Monad (when)-import Control.Monad.Trans (MonadIO(..), lift)-import Control.Monad.Reader (ReaderT(..), MonadReader(..), (<=<), (>=>), ask,- asks)-import Control.Monad.State.Strict (MonadState, StateT, State, get, gets,- modify, put, liftM, liftIO, runState,- runStateT, execState, execStateT,- evalState, evalStateT)-import Control.Monad.Except (ExceptT, MonadError(..), throwError, runExceptT)-import Control.Exception.ErrorList-import Control.Monad.Identity (Identity(..))-import Control.Applicative hiding (empty, optional)-import Data.Char (isDigit, isAlpha)-import Data.Default-import Data.List (maximum, maximumBy)-import Data.HashMap.Strict (HashMap, (!))-import qualified Data.HashMap.Strict as H-import Data.Maybe (fromJust, isJust, isNothing)-import Data.Either (isRight, isLeft)-import Data.String.Utils hiding (join)-import qualified Data.Text as T-import Filesystem.Path.CurrentOS (FilePath, fromText, toText, collapse)-import GHC.Exts (IsList)-import GHC.IO.Exception-import Control.Exception (bracket)-import Text.Render hiding (renderParens)-import Network.URI (URI(..), parseURI, parseAbsoluteURI,- parseRelativeReference, relativeTo)-import qualified Network.URI as NU-import Shelly hiding (get, relativeTo)-import System.Directory-import System.FilePath.Posix hiding (FilePath)---- | Indicates that the text is some identifier.-type Name = Text---- | Indicates that the text is some path.-type Path = Text---- | A record is a lookup table with string keys.-type Record = HashMap Name---- | Takes two applicative actions and returns their result as a 2-tuple.-tuple :: Applicative f => f a -> f b -> f (a, b)-tuple action1 action2 = (,) <$> action1 <*> action2---- | Takes three applicative actions and returns their result as a 3-tuple.-tuple3 :: Applicative f => f a -> f b -> f c -> f (a, b, c)-tuple3 action1 action2 action3 = (,,) <$> action1 <*> action2 <*> action3---- | Creates a new hashmap by applying a function to every key in it.-alterKeys :: (Eq k, Hashable k, Eq k', Hashable k') =>- (k -> k') -> HashMap k v -> HashMap k' v-alterKeys f mp = do- let pairs = H.toList mp- let newPairs = P.map (\(k, v) -> (f k, v)) pairs- let newMap = H.fromList newPairs- newMap---- | Create a hashmap by applying a test to everything in the existing--- map.-modifyMap :: (Eq k, Hashable k) => (a -> Maybe b) -> HashMap k a -> HashMap k b-modifyMap test inputMap = foldl' step mempty $ H.toList inputMap where- step result (k, elem) = case test elem of- Nothing -> result- Just newElem -> H.insert k newElem result--cerror :: [String] -> a-cerror = error . concat--cerror' :: [Text] -> a-cerror' = cerror . map unpack--fromRight :: Either a b -> b-fromRight (Right x) = x-fromRight (Left err) = error "Expected `Right` value"--uriToText :: URI -> Text-uriToText = pack . uriToString--uriToString :: URI -> String-uriToString uri = NU.uriToString id uri ""---- | Appends text to URI with a slash. Ex: foo.com `slash` bar == foo.com/bar.-slash :: URI -> Text -> URI-slash uri txt = case parseRelativeReference (unpack txt) of- Nothing -> error ("Invalid appending URI: " <> show txt)- Just uri' -> uri' `relativeTo` uri--infixl 6 `slash`--putStrsLn :: MonadIO m => [Text] -> m ()-putStrsLn = putStrLn . concat--putStrs :: MonadIO m => [Text] -> m ()-putStrs = putStr . concat--pathToText :: FilePath -> Text-pathToText pth = case toText pth of- Left p -> p- Right p -> p--dropSuffix :: String -> String -> String-dropSuffix suffix s | s == suffix = ""-dropSuffix suffix (c:cs) = c : dropSuffix suffix cs-dropSuffix suffix "" = ""--maybeIf :: Bool -> a -> Maybe a-maybeIf True x = Just x-maybeIf False _ = Nothing--grab :: (Hashable k, Eq k) => k -> HashMap k v -> v-grab k = fromJust . H.lookup k--withDir :: String -> IO a -> IO a-withDir d action = do- cur <- getCurrentDirectory- result <- bracket (setCurrentDirectory d)- (\_ -> setCurrentDirectory cur)- (\_ -> action)- return result--pathToString :: FilePath -> String-pathToString = unpack . pathToText--joinBy :: Text -> [Text] -> Text-joinBy = T.intercalate--mapJoinBy :: Text -> (a -> Text) -> [a] -> Text-mapJoinBy sep func = joinBy sep . map func---- | Reads an environment variable.-getEnv :: Text -> IO (Maybe Text)-getEnv = shelly . silently . get_env
− src/NixFromNpm/ConvertToNix.hs
@@ -1,295 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE ScopedTypeVariables #-}-module NixFromNpm.ConvertToNix where--import Data.HashMap.Strict (HashMap)-import qualified Data.HashMap.Strict as H-import Data.Map (Map)-import qualified Data.Map as M-import Data.Text (Text)-import qualified Data.Text as T--import NixFromNpm.Common-import Nix.Types-import Nix.Parser-import Nix.Pretty (prettyNix)-import NixFromNpm.Options-import NixFromNpm.NpmTypes-import NixFromNpm.SemVer-import NixFromNpm.Parsers.SemVer-import NixFromNpm.NpmLookup (getPkg, FullyDefinedPackage(..), concatDots,- PackageMap, mapPM, PreExistingPackage(..))--_startingSrc :: String-_startingSrc = "\- \{nixpkgs ? import <nixpkgs> {}}: \- \let \- \ inherit (nixpkgs.lib) attrValues foldl; \- \ joinSets = foldl (a: b: a // b) {}; \- \ joinedExtensions = joinSets (attrValues extensions); \- \ allPkgs = nixpkgs // nodePkgs // joinedExtensions // \- \ {inherit (nixpkgs.nodePackages)buildNodePackage;}; \- \ callPackage = nixpkgs.lib.callPackageWith allPkgs; \- \ nodePkgs = joinedExtensions // byVersion // defaults; \- \in \- \nodePkgs"--_startingExpr :: NExpr-_startingExpr = case parseNixString _startingSrc of- Success e -> e- Failure e -> error $ unlines ["FATAL: Starting source failed to parse:",- show e]--callPackage :: NExpr -> NExpr-callPackage = callPackageWith []--callPackageWith :: [Binding NExpr] -> NExpr -> NExpr-callPackageWith args e = mkApp (mkApp (mkSym "callPackage") e)- (mkNonRecSet args)--callPackageWithRec :: [Binding NExpr] -> NExpr -> NExpr-callPackageWithRec args e = mkApp (mkApp (mkSym "callPackage") e)- (mkRecSet args)---- | Turns a string into one that can be used as an identifier.-fixName :: Name -> Name-fixName = T.replace "." "-"---- | Converts a package name and semver into an identifier.-toDepName :: Name -> SemVer -> Name-toDepName name (a, b, c) = concat [fixName name, "_", pack $- intercalate "-" $ map show [a, b, c]]--- | Gets the .nix filename of a semver. E.g. (0, 1, 2) -> 0.1.2.nix-toDotNix :: SemVer -> Text-toDotNix v = concatDots v <> ".nix"---- | Creates a doublequoted string from some text.-str :: Text -> NExpr-str = mkStr DoubleQuoted---- | Converts distinfo into a nix fetchurl call.-distInfoToNix :: DistInfo -> NExpr-distInfoToNix DistInfo{..} = mkApp (mkSym "fetchurl") $ mkNonRecSet- [ "url" `bindTo` str diUrl,- "sha1" `bindTo` str diShasum ]---- | Tests if there is information in the package meta.-metaNotEmpty :: PackageMeta -> Bool-metaNotEmpty PackageMeta{..} = isJust pmDescription---- | Converts package meta to a nix expression.-metaToNix :: PackageMeta -> NExpr-metaToNix PackageMeta{..} = case pmDescription of- Nothing -> mkNonRecSet []- Just d -> mkNonRecSet ["description" `bindTo` str d]---- | Converts a resolved package object into a nix expression. The expresion--- will be a function where the arguments are its dependencies, and its result--- is a call to `buildNodePackage`.-resolvedPkgToNix :: ResolvedPkg -> NExpr-resolvedPkgToNix ResolvedPkg{..} = do- let -- Get a string representation of each dependency in name-version format.- deps = map (mkSym . uncurry toDepName) $ H.toList rpDependencies- -- Get the parameters of the package function (deps + utility functions).- _funcParams = map (uncurry toDepName) (H.toList rpDependencies)- <> ["buildNodePackage", "fetchurl"]- -- None of these have defaults, so put them into pairs with Nothing.- funcParams = mkFormalSet $ map (\x -> (x, Nothing)) _funcParams- let args = mkNonRecSet $ catMaybes [- Just $ "name" `bindTo` str rpName,- Just $ "version" `bindTo` (str $ renderSV rpVersion),- Just $ "src" `bindTo` distInfoToNix rpDistInfo,- maybeIf (length deps > 0) $ "deps" `bindTo` mkList deps,- maybeIf (metaNotEmpty rpMeta) $ "meta" `bindTo` metaToNix rpMeta- ]- mkFunction funcParams $ mkApp (mkSym "buildNodePackage") args---- | Creates the `default.nix` file that is the top-level expression we are--- generating.-mkDefaultNix :: Record [SemVer] -- ^ Map of names to versions of packages that- -- exist in this library.- -> Record Path -- ^ Map of extensions being included.- -> NExpr -- ^ A generated nix expression.-mkDefaultNix versionMap extensionMap = do- let mkPath' = mkPath False . unpack- toPath name ver = mkPath' $ concat ["./", name, "/", toDotNix ver]- -- Make a set of all of the extensions- extensionsSet = mkNonRecSet $- -- Map over the expression map, creating a binding for each pair.- flip map (H.toList extensionMap) $ \(name, path) ->- name `bindTo` (mkApp- (mkApp (mkSym "import") (mkPath False (unpack path)))- (mkNonRecSet [Inherit Nothing [mkSelector "nixpkgs"]]))- mkBinding name ver = toDepName name ver- `bindTo` callPackage (toPath name ver)- mkBindings name vers = map (mkBinding name) vers- mkDefVer name vers = case vers of- [] -> errorC ["FATAL: no versions generated for package ", name]- _ -> fixName name `bindTo` mkSym (toDepName name $ maximum vers)- -- This bit of map gymnastics will create a list of pairs of names- -- with all of the versions of that name that we have.- versOnly = sortOn fst $ H.toList versionMap- byVersion = mkNonRecSet $ concatMap (uncurry mkBindings) versOnly- defaults = mkWith (mkSym "byVersion") $- mkNonRecSet $ map (uncurry mkDefVer) versOnly- newBindings = ["extensions" `bindTo` extensionsSet,- "byVersion" `bindTo` byVersion,- "defaults" `bindTo` defaults]- modifyFunctionBody (appendBindings newBindings) _startingExpr---- | The npm lookup utilities will produce a bunch of fully defined packages.--- However, the only packages that we want to write are the new ones; that--- is, the ones that we've discovered and the ones that already exist. This--- will perform the appropriate filter.-takeNewPackages :: PackageMap FullyDefinedPackage- -> (PackageMap ResolvedPkg, PackageMap NExpr)-takeNewPackages startingRec = do- let isNew (NewPackage rpkg) = Just rpkg- isNew _ = Nothing- exists (FromExistingInOutput expr) = Just expr- exists _ = Nothing- newPkgs = H.map (modifyMap isNew) startingRec- existingPkgs = H.map (modifyMap exists) startingRec- removeEmpties = H.filter (not . H.null)- (removeEmpties newPkgs, removeEmpties existingPkgs)---- | Actually writes the packages to disk. Takes in the new packages to write,--- and the names/paths to the libraries being extended.-dumpPkgs :: MonadIO m- => String -- ^ Path to output directory.- -> PackageMap ResolvedPkg -- ^ New packages being written.- -> PackageMap NExpr -- ^ Existing packages to be included- -- in the generated default.nix.- -> Record Path -- ^ Libraries being extended.- -> m ()-dumpPkgs path newPackages existingPackages extensions = liftIO $ do- let _path = pack path- -- If there aren't any new packages, we can stop here.- if H.null newPackages- then putStrLn "No new packages created." >> return ()- else do- putStrsLn ["Creating new packages at ", _path]- createDirectoryIfMissing True path- withDir path $ do- -- Write the .nix file for each version of this package.- forM_ (H.toList newPackages) $ \(pkgName, pkgVers) -> do- let subdir = path </> unpack pkgName- createDirectoryIfMissing False subdir- withDir subdir $ forM_ (H.toList pkgVers) $ \(ver, rpkg) -> do- let expr = resolvedPkgToNix rpkg- fullPath = subdir </> unpack (toDotNix ver)- putStrsLn ["Writing package file at ", pack fullPath]- writeFile (unpack $ toDotNix ver) $ show $ prettyNix expr- -- Write the default.nix file for the library.- -- We need to build up a record mapping package names to the list of- -- versions being defined in this library.- let versionMap = map H.keys newPackages <> map H.keys existingPackages- defaultNix = mkDefaultNix versionMap extensions- writeFile "default.nix" $ show $ prettyNix defaultNix---- | Given the path to a package, finds all of the .nix files which parse--- correctly.-parseVersion :: Name -> Path -> IO (Maybe (SemVer, NExpr))-parseVersion pkgName path = do- let pth = unpack path- versionTxt = pack $ dropSuffix ".nix" $ takeBaseName pth- case parseSemVer versionTxt of- Left _ -> return Nothing -- not a version file- Right version -> parseNixString . pack <$> readFile pth >>= \case- Failure err -> do- putStrsLn ["Warning: expression for ", pkgName, " version ",- versionTxt, " failed to parse:\n", pack $ show err]- return Nothing -- invalid nix, should overwrite- Success expr -> return $ Just (version, expr)---- | Given the path to a file possibly containing nix expressions, finds all--- expressions findable at that path and returns a map of them.-findExisting :: Maybe Name -- ^ Is `Just` if this is an extension.- -> Path -- ^ The path to search.- -> IO (PackageMap PreExistingPackage) -- ^ Mapping of package- -- names to maps of- -- versions to nix- -- expressions.-findExisting maybeName path = do- doesDirectoryExist (unpack path) >>= \case- False -> case maybeName of- Just name -> errorC ["Extension ", name, " at path ", path,- " does not exist."]- Nothing -> return mempty- True -> withDir (unpack path) $ do- let wrapper :: NExpr -> PreExistingPackage- wrapper = case maybeName of Nothing -> FromOutput- Just name -> FromExtension name- putStrsLn ["Searching for existing expressions in ", path, "..."]- contents <- getDirectoryContents "."- verMaps <- forM contents $ \dir -> do- exprs <- doesDirectoryExist dir >>= \case- True -> withDir dir $ do- contents <- getDirectoryContents "."- let files = pack <$> filter (endswith ".nix") contents- catMaybes <$> mapM (parseVersion $ pack dir) files- False -> do- return mempty -- not a directory- case exprs of- [] -> return Nothing- vs -> return $ Just (pack dir, H.map wrapper $ H.fromList exprs)- let total = sum $ map (H.size . snd) $ catMaybes verMaps- putStrsLn ["Found ", render total, " existing expressions"]- return $ H.fromList $ catMaybes verMaps---- | Given the output directory and any number of extensions to load,--- finds any existing packages.-preloadPackages :: Bool -- ^ Whether to skip the existence check.- -> Path -- ^ Output path to search for existing packages.- -> Record Path -- ^ Mapping of names of libraries to extend,- -- and paths to those libraries.- -> IO (PackageMap PreExistingPackage)-preloadPackages noExistCheck path toExtend = do- existing <- if noExistCheck then pure mempty- else findExisting Nothing path- libraries <- fmap concat $ forM (H.toList toExtend) $ \(name, path) -> do- findExisting (Just name) path- return (existing <> libraries)---- | Given the name of a package and a place to dump expressions to, generates--- the expressions needed to build that package.-dumpPkgNamed :: Text -- ^ The name of the package to fetch.- -> Path -- ^ The path to output to.- -> PackageMap PreExistingPackage -- ^ Set of existing packages.- -> Record Path -- ^ Names -> paths of extensions.- -> Maybe Text -- ^ Optional github token.- -> IO () -- ^ Writes files to a folder.-dumpPkgNamed name path existing extensions token = do- pwd <- getCurrentDirectory- packages <- getPkg name existing token- let (new, existing) = takeNewPackages packages- dumpPkgs (pwd </> unpack path) new existing extensions---- | Parse the NAME=PATH extension directives.-getExtensions :: [Text] -> Record Path-getExtensions = foldl' step mempty where- step :: Record Path -> Text -> Record Path- step exts nameEqPath = case T.split (== '=') nameEqPath of- [name, path] -> append name path- [path] -> append (pack $ takeBaseName (unpack path)) path- _ -> errorC ["Extensions must be of the form NAME=PATH (in argument ",- nameEqPath, ")"]- where- append name path = case H.lookup name exts of- Nothing -> H.insert name path exts- Just path' -> errorC ["Extension ", name, " is mapped to both path ",- path, " and path ", path']---- displayExisting :: PackageMap -> IO ()--- displayExisting pmap = forM_--dumpPkgFromOptions :: NixFromNpmOptions -> IO ()-dumpPkgFromOptions NixFromNpmOptions{..} = do- forM_ nfnoPkgNames $ \name -> do- let extensions = getExtensions nfnoExtendPaths- existing <- preloadPackages nfnoNoCache nfnoOutputPath extensions- -- displayExisting existing- dumpPkgNamed name nfnoOutputPath existing extensions nfnoGithubToken
− src/NixFromNpm/NpmLookup.hs
@@ -1,478 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE ScopedTypeVariables #-}-module NixFromNpm.NpmLookup where-----------------------------------------------------------------------------import qualified Prelude as P-import qualified Data.List as L-import Data.Text (Text)-import qualified Data.Text as T-import qualified Data.Text.Encoding as T-import Data.HashMap.Strict (HashMap)-import qualified Data.HashMap.Strict as H-import Data.HashSet (HashSet)-import qualified Data.HashSet as HS--import qualified Data.ByteString.Lazy.Char8 as BL8-import Data.Aeson.Parser-import Data.Aeson-import Data.Aeson.Types (Parser, typeMismatch)-import qualified Text.Parsec as Parsec-import Shelly hiding (get)--import NixFromNpm.Common-import NixFromNpm.NpmTypes-import NixFromNpm.SemVer-import NixFromNpm.Parsers.Common hiding (Parser, Error, lines)-import NixFromNpm.Parsers.SemVer-import NixFromNpm.NpmVersion-import NixFromNpm.Parsers.NpmVersion-import Nix.Types------------------------------------------------------------------------------- | Things which can be converted into nix expressions: either they--- are actual nix expressions themselves (which can be either--- existing in the output, or existing in an extension), or they are--- new packages which we have discovered.-data FullyDefinedPackage- = NewPackage ResolvedPkg- | FromExistingInOutput NExpr- | FromExistingInExtension Name NExpr- deriving (Show, Eq)---- | The type of pre-existing packages, which can either come from the--- output path, or come from an extension-data PreExistingPackage- = FromOutput NExpr- | FromExtension Name NExpr- deriving (Show, Eq)--toFullyDefined :: PreExistingPackage -> FullyDefinedPackage-toFullyDefined (FromOutput expr) = FromExistingInOutput expr-toFullyDefined (FromExtension name expr) = FromExistingInExtension name expr---- | We use this data structure a lot: a mapping of package names to--- a mapping of versions to fully defined packages.-type PackageMap pkg = Record (HashMap SemVer pkg)---- | Map a function across a PackageMap.-mapPM :: (a -> b) -> PackageMap a -> PackageMap b-mapPM f = H.map (H.map f)---- | The state of the NPM fetcher.-data NpmFetcherState = NpmFetcherState {- registries :: [URI],- githubAuthToken :: Maybe Text,- resolved :: PackageMap FullyDefinedPackage,- pkgInfos :: Record PackageInfo,- -- For cycle detection.- currentlyResolving :: HashSet (Name, SemVer),- indentLevel :: Int,- knownProblematicPackages :: HashSet Name,- getDevDeps :: Bool-} deriving (Show, Eq)--type NpmFetcher = ExceptT EList (StateT NpmFetcherState IO)--concatDots :: SemVer -> Text-concatDots (a, b, c) = pack $ intercalate "." (map show [a,b,c])--indent :: Text -> NpmFetcher Text-indent txt = do- ind <- gets indentLevel- return $ concat [txt, " (depth of ", pack $ show ind, ")"]--putStrLnI :: Text -> NpmFetcher ()-putStrLnI = indent >=> putStrLn--putStrsLnI :: [Text] -> NpmFetcher ()-putStrsLnI = putStrLnI . concat--putStrI :: Text -> NpmFetcher ()-putStrI = indent >=> putStr--putStrsI :: [Text] -> NpmFetcher ()-putStrsI = putStrI . concat--addResolvedPkg :: Name -> SemVer -> ResolvedPkg -> NpmFetcher ()-addResolvedPkg name version _rpkg = do- let rpkg = NewPackage _rpkg- pkgSet <- H.lookupDefault mempty name <$> gets resolved- modify $ \s -> s {- resolved = H.insert name (H.insert version rpkg pkgSet) (resolved s)- }---- | Performs a curl query and returns whatever that query returns.-curl :: [Text] -> NpmFetcher Text-curl args = shell $ print_stdout False $ run "curl" (["-L", "--fail"] <> args)---- | Queries NPM for package information.-_getPackageInfo :: Name -> URI -> NpmFetcher PackageInfo-_getPackageInfo pkgName registryUri = do- let uri = uriToText $ registryUri `slash` pkgName- putStrsLn ["Querying ", uriToText registryUri,- " for package ", pkgName, "..."]- jsonStr <- curl [uri]- case eitherDecode $ BL8.fromChunks [T.encodeUtf8 jsonStr] of- Left err -> throwErrorC ["couldn't parse JSON from NPM: ", pack err]- Right info -> return info---- | Same as _getPackageInfo, but caches results for speed.-getPackageInfo :: Name -> NpmFetcher PackageInfo-getPackageInfo name = lookup name . pkgInfos <$> get >>= \case- Just info -> return info- Nothing -> inContext ctx $ do- regs <- gets registries- info <- firstSuccess "No repos contained package" $- map (_getPackageInfo name) regs- storePackageInfo name info- return info- where ctx = pack $ "When querying NPM registry for package " <> show name--storePackageInfo :: Name -> PackageInfo -> NpmFetcher ()-storePackageInfo name info = do- infos <- gets pkgInfos- let existingInfo = H.lookupDefault mempty name infos- newInfo = existingInfo <> info- modify $ \s -> s {pkgInfos = H.insert name newInfo (pkgInfos s)}---toSemVerList :: Record a -> NpmFetcher [(SemVer, a)]-toSemVerList rec = do- -- Pairings of parsed semvers (or errors) to values.- let parsePair (k, v) = (parseSemVer k, v)- pairs = map parsePair $ H.toList rec- case filter (\(k, _) -> isRight k) pairs of- [] -> throwError1 "No correctly-formatted versions strings found"- okPairs -> return $ map (\(Right k, v) -> (k, v)) okPairs--bestMatchFromRecord :: SemVerRange -> Record a -> NpmFetcher a-bestMatchFromRecord range rec = do- pairs <- toSemVerList rec- case filter (matches range . fst) pairs of- [] -> throwError1 "No versions satisfy given range"- matches -> return $ snd $ maximumBy (compare `on` fst) matches--shell :: Sh Text -> NpmFetcher Text-shell action = do- (code, out, err) <- shelly $ errExit False $ do- out <- action- code <- lastExitCode- err <- lastStderr- return (code, out, err)- case code of- 0 -> return out- n -> do- throwErrorC $ catMaybes [- Just "Shell command returned an error.",- maybeIf (out /= "") $ "\nstdout:\n" <> out,- Just $ "\nstderr:\n" <> err]--silentShell :: Sh Text -> NpmFetcher Text-silentShell = shell . silently---- | Returns the SHA1 hash of the result of fetching the URI, and the path--- in which the tarball is stored.-nixPrefetchSha1 :: URI -> NpmFetcher (Text, FilePath)-nixPrefetchSha1 uri = do- hashAndPath <- silentShell $ do- setenv "PRINT_PATH" "1"- run "nix-prefetch-url" ["--type", "sha1", uriToText uri]- if length (lines hashAndPath) /= 2- then error "Expected two lines from nix-prefetch-url"- else do- let [hashBase32, path] = lines hashAndPath- -- Convert the hash to base16, which is the format NPM uses.- hash <- silentShell $ do- run "nix-hash" ["--type", "sha1", "--to-base16", hashBase32]- return (T.strip hash, fromString $ unpack path)--extractVersionInfo :: FilePath -> Text -> NpmFetcher VersionInfo-extractVersionInfo tarballPath subpath = do- ind <- gets indentLevel- pkJson <- silentShell $ withTmpDir $ \dir -> do- chdir dir $ do- putStrs ["Extracting ", pathToText tarballPath, " to tempdir"]- putStrsLn [" (depth of ", pack $ show ind, ")"]- run_ "tar" ["-xf", pathToText tarballPath]- curdir <- pathToText <$> pwd- pth <- fmap T.strip $ run "find" [curdir, "-name", "package.json"]- -|- run "head" ["-n", "1"]- when (pth == "") $ error "No package.json found"- map decodeUtf8 $ readBinary $ fromString $ unpack $ pth- case eitherDecode $ BL8.fromChunks [encodeUtf8 pkJson] of- Left err -> error $ "couldn't parse JSON as VersionInfo: " <> err- Right info -> return info---- | Fetch a package over HTTP. Return the version of the fetched package,--- and store the hash.-fetchHttp :: Text -- ^ Subpath in which to find the package.json.- -> URI -- ^ The URI to fetch.- -> NpmFetcher SemVer -- ^ The version of the package at that URI.-fetchHttp subpath uri = do- -- Use nix-fetch to download and hash the tarball.- (hash, tarballPath) <- nixPrefetchSha1 uri- -- Extract the tarball to a temp directory and parse the package.json.- versionInfo <- extractVersionInfo tarballPath subpath- -- Create the DistInfo.- let dist = DistInfo {diUrl = uriToText uri, diShasum = hash}- -- Add the dist information to the version info and resolve it.- resolveVersionInfo $ versionInfo {viDist = Just dist}--githubCurl :: Text -> NpmFetcher Value-githubCurl uri = do- -- Add in the github auth token if it is provided.- extraCurlArgs <- gets githubAuthToken >>= \case- Nothing -> return []- Just token -> return ["-H", "Authorization: token " <> token]- let curlArgs = extraCurlArgs <> [- -- This accept header tells github to allow redirects.- "-H", "Accept: application/vnd.github.quicksilver-preview+json",- uri- ]- -- putStrsLn $ ["calling curl with args: ", T.intercalate " " curlArgs]- jsonStr <- curl curlArgs- case eitherDecode $ BL8.fromChunks [T.encodeUtf8 jsonStr] of- Left err -> throwErrorC ["couldn't parse JSON from github: ", pack err]- Right info -> return info---- | Queries NPM for package information.-getDefaultBranch :: Name -> Name -> NpmFetcher Name-getDefaultBranch owner repo = do- let rpath = "/" <> owner <> "/" <> repo- let uri = concat ["https://api.github.com/repos", rpath]- putStrs ["Querying github for default branch of ", rpath, "..."]- githubCurl uri >>= \case- Object o -> case H.lookup "default_branch" o of- Just (String b) -> putStr " OK. " >> return b- Nothing -> putStrLn "" >> error "No default branch, or not a string"- _ -> error "Expected an object back from github"---- | Given a github repo and a branch, gets the SHA of the head of that--- branch-getShaOfBranch :: Name -- ^ Repo owner- -> Name -- ^ Repo name- -> Name -- ^ Name of the branch to get- -> NpmFetcher Text -- ^ The hash of the branch-getShaOfBranch owner repo branchName = do- let rpath = "/" <> owner <> "/" <> repo- let uri = concat ["https://api.github.com/repos", rpath,- "/branches/", branchName]- putStrs ["Querying github for sha of ", rpath, "/", branchName, "..."]- githubCurl uri >>= \case- Object o -> case H.lookup "commit" o of- Just (Object o') -> case H.lookup "sha" o' of- Just (String sha) -> return sha- Nothing -> error "No sha in commit info"- Nothing -> error "No commit info"- _ -> error "Didn't get an object back"---- | Fetch a package from git.-fetchGithub :: URI -> NpmFetcher SemVer-fetchGithub uri = do- (owner, repo) <- case split "/" $ uriPath uri of- [_, owner, repo] -> return (pack owner, pack $ dropSuffix ".git" repo)- _ -> throwErrorC ["Invalid repo path: ", pack $ uriPath uri]- hash <- case uriFragment uri of- -- if there isn't a ref or a tag, use the default branch.- "" -> do- branch <- getDefaultBranch owner repo- putStrLn $ " Branch is " <> branch- sha <- getShaOfBranch owner repo branch- putStrLn $ " Hash is " <> sha- return sha- -- otherwise, use that as a tag.- '#':frag -> return $ pack frag- frag -> throwErrorC ["Invalid fragment '", pack frag, "'"]- -- Use the hash to pull down a zip.- let uri = concat ["https://github.com/", owner, "/", repo, "/archive/",- hash, ".tar.gz"]- fetchHttp (repo <> "-" <> hash) (fromJust $ parseURI $ unpack uri)--resolveNpmVersionRange :: Name -> NpmVersionRange -> NpmFetcher SemVer-resolveNpmVersionRange name range = case range of- SemVerRange svr -> resolveDep name svr- NpmUri uri -> case uriScheme uri of- "git:" -> fetchGithub uri- "git+https:" -> fetchGithub uri- "http:" -> fetchHttp "package" uri- "https:" -> fetchHttp "package" uri- scheme -> throwErrorC ["Unknown uri scheme ", pack scheme]- GitId src owner repo rev -> case src of- Github -> do- let frag = case rev of- Nothing -> ""- Just r -> "#" <> r- let uri = concat ["https://github.com/", owner, "/", repo, frag]- fetchGithub $ fromJust $ parseURI $ unpack uri- _ -> throwErrorC ["Can't handle git source ", pack $ show src]- Tag tag -> resolveByTag tag name- vr -> throwErrorC ["Don't know how to resolve dependency '",- pack $ show vr, "'"]---- | Uses the set of downloaded packages as a cache to avoid unnecessary--- duplication.-resolveDep :: Name -> SemVerRange -> NpmFetcher SemVer-resolveDep name range = H.lookup name <$> gets resolved >>= \case- -- We've alread defined some versions of this package.- Just versions -> case filter (matches range) (H.keys versions) of- [] -> _resolveDep name range -- No matching versions, need to fetch.- vs -> do- let bestVersion = maximum vs- versionDots = concatDots bestVersion- package = fromJust $ H.lookup bestVersion versions- putStrsLn ["Requirement ", name, " version ", pack $ show range,- " already satisfied:"]- putStrsLn $ case package of- NewPackage _ -> ["Fetched package version ", versionDots]- FromExistingInOutput _ -> ["Already had version ", versionDots,- " in output directory (use --no-cache",- " to override)"]- FromExistingInExtension name _ -> ["Version ", versionDots,- " provided by extension ", name]- return bestVersion- -- We haven't yet found any versions of this package.- Nothing -> _resolveDep name range--startResolving :: Name -> SemVer -> NpmFetcher ()-startResolving name ver = do- putStrsLn ["Resolving ", name, " version ", renderSV ver]- modify $ \s ->- s {currentlyResolving = HS.insert (name, ver) $ currentlyResolving s}--finishResolving :: Name -> SemVer -> NpmFetcher ()-finishResolving name ver = do- modify $ \s ->- s {currentlyResolving = HS.delete (name, ver) $ currentlyResolving s}- putStrsLn ["Finished resolving ", name, " ", renderSV ver]--resolveVersionInfo :: VersionInfo -> NpmFetcher SemVer-resolveVersionInfo versionInfo = do- let name = viName versionInfo- version = viVersion versionInfo- ctx = concat ["When resolving package ", name, ", version ", version]- inContext ctx $ do- version <- case parseSemVer $ viVersion versionInfo of- Left err -> throwErrorC ["Invalid semver in versionInfo object ",- viVersion versionInfo,- " Due to: ", pack $ show err]- Right v -> return v- HS.member (name, version) <$> gets currentlyResolving >>= \case- True -> do putStrsLn ["Warning: cycle detected"]- return version- False -> do- let recurOn deptype deps = map H.fromList $ do- let depList = H.toList $ deps versionInfo- when (length depList > 0) $- putStrsLn ["Found ", deptype, ": ", pack (show depList)]- res <- map catMaybes $ forM depList $ \(depName, depRange) -> do- HS.member depName <$> gets knownProblematicPackages >>= \case- True -> do- putStrsLn ["WARNING: ", name, " is a broken package"]- return Nothing- False -> do- depVersion <- resolveNpmVersionRange depName depRange- return $ Just (depName, depVersion)- return res- -- We need to recur into the package's dependencies.- -- To prevent the cycles, we store which packages we're currently- -- resolving.- startResolving name version- deps <- recurOn "Dependencies" viDependencies- devDeps <- gets getDevDeps >>= \case- True -> recurOn "Dev dependencies" viDevDependencies- False -> return mempty- finishResolving name version- let dist = case viDist versionInfo of- Nothing -> error "Version information did not include dist"- Just d -> d- -- Store this version's info.- addResolvedPkg name version $ ResolvedPkg {- rpName = name,- rpVersion = version,- rpDistInfo = dist,- rpMeta = viMeta versionInfo,- rpDependencies = deps,- rpDevDependencies = devDeps- }- return version---- | Resolves a dependency given a name and version range.-_resolveDep :: Name -> SemVerRange -> NpmFetcher SemVer-_resolveDep name range = do- let ctx = concat ["When resolving dependency ", name, " (",- pack $ show range, ")"]- inContext ctx $ do- pInfo <- getPackageInfo name- versionInfo <- bestMatchFromRecord range $ piVersions pInfo- resolveVersionInfo versionInfo--resolveByTag :: Name -> Name -> NpmFetcher SemVer-resolveByTag tag pkgName = do- pInfo <- getPackageInfo pkgName- case H.lookup tag $ piTags pInfo of- Nothing -> throwErrorC ["Package ", pkgName, " has no tag '", tag, "'"]- Just version -> case H.lookup version $ piVersions pInfo of- Nothing -> throwErrorC ["Tag '", tag, "' refers to version '", version,- "' but no such version exists for package ",- pack $ show pkgName]- Just versionInfo -> resolveVersionInfo versionInfo--parseURIs :: [Text] -> [URI]-parseURIs rawUris = map p $! rawUris where- p txt = case parseURI $ unpack txt of- Nothing -> errorC ["Invalid URI: ", txt]- Just uri -> uri--startState :: PackageMap PreExistingPackage- -> [Text]- -> Maybe Text- -> NpmFetcherState-startState existing registries token = do- NpmFetcherState {- registries = parseURIs registries,- githubAuthToken = token,- resolved = mapPM toFullyDefined existing,- pkgInfos = mempty,- currentlyResolving = mempty,- knownProblematicPackages = HS.fromList ["websocket-server"],- indentLevel = 0,- getDevDeps = False- }---- | Read NPM registry from env or use default.-getRegistries :: IO [Text]-getRegistries = do- let npmreg = "https://registry.npmjs.org/"- others <- shelly $ silently $ do- get_env "ADDITIONAL_NPM_REGISTRIES" >>= \case- Nothing -> return []- Just regs -> return $ T.words regs- return (others `snoc` npmreg)---- | Read github auth token from env or use none.-getToken :: IO (Maybe Text)-getToken = shelly $ silently $ get_env "GITHUB_TOKEN"--runIt :: NpmFetcher a -> IO (a, NpmFetcherState)-runIt x = do- state <- startState mempty <$> getRegistries <*> getToken- runItWith state x--runItWith :: NpmFetcherState -> NpmFetcher a -> IO (a, NpmFetcherState)-runItWith state x = do- runStateT (runExceptT x) state >>= \case- (Left elist, _) -> error $ "\n" <> (unpack $ render elist)- (Right x, state) -> return (x, state)--getPkg :: Name -- ^ Name of package to get.- -> PackageMap PreExistingPackage -- ^ Set of pre-existing packages.- -> Maybe Text -- ^ A possible github token.- -> IO (PackageMap FullyDefinedPackage) -- ^ Set of fully defined packages.-getPkg name existing token = do- let range = Gt (0, 0, 0)- state <- startState existing <$> getRegistries <*> pure token- (_, finalState) <- runItWith state (resolveDep name range)- return (resolved finalState)
− src/NixFromNpm/NpmTypes.hs
@@ -1,108 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE ScopedTypeVariables #-}-module NixFromNpm.NpmTypes where--import Data.Aeson-import Data.Aeson.Types (Parser, typeMismatch)-import Data.HashMap.Strict (HashMap)-import qualified Data.HashMap.Strict as H--import NixFromNpm.Common-import NixFromNpm.SemVer-import NixFromNpm.NpmVersion-import NixFromNpm.Parsers.NpmVersion-import NixFromNpm.Parsers.SemVer--data PackageInfo = PackageInfo {- piVersions :: Record VersionInfo,- piTags :: Record Name-} deriving (Show, Eq)--data PackageMeta = PackageMeta {- pmDescription :: Maybe Text-} deriving (Show, Eq)--data VersionInfo = VersionInfo {- viDependencies :: Record NpmVersionRange,- viDevDependencies :: Record NpmVersionRange,- viDist :: Maybe DistInfo, -- not present if in a package.json file.- viMain :: Maybe Text,- viName :: Text,- viHasTest :: Bool,- viMeta :: PackageMeta,- viVersion :: Text-} deriving (Show, Eq)---- | Distribution info from NPM. Tells us the URL and hash of a tarball.-data DistInfo = DistInfo {- diUrl :: Text,- diShasum :: Text-} deriving (Show, Eq)--data ResolvedPkg = ResolvedPkg {- rpName :: Name,- rpVersion :: SemVer,- rpDistInfo :: DistInfo,- rpMeta :: PackageMeta,- rpDependencies :: Record SemVer,- rpDevDependencies :: Record SemVer-} deriving (Show, Eq)--instance Semigroup PackageInfo where- PackageInfo vs ts <> PackageInfo vs' ts' =- PackageInfo (vs <> vs') (ts <> ts')--instance Monoid PackageInfo where- mempty = PackageInfo mempty mempty- mappend = (<>)--instance FromJSON VersionInfo where- parseJSON = getObject "version info" >=> \o -> do- dependencies <- getDict "dependencies" o- devDependencies <- getDict "devDependencies" o- dist <- o .:? "dist"- name <- o .: "name"- main <- o .:? "main"- version <- o .: "version"- packageMeta <- fmap PackageMeta $ o .:? "description"- scripts :: Record Value <- getDict "scripts" o- return $ VersionInfo {- viDependencies = dependencies,- viDevDependencies = devDependencies,- viDist = dist,- viMain = main,- viName = name,- viHasTest = H.member "test" scripts,- viMeta = packageMeta,- viVersion = version- }--instance FromJSON SemVerRange where- parseJSON v = case v of- String s -> case parseSemVerRange s of- Left err -> typeMismatch ("valid semantic version (got " <> show v <> ")") v- Right range -> return range- _ -> typeMismatch "string" v--instance FromJSON PackageInfo where- parseJSON = getObject "package info" >=> \o -> do- vs <- getDict "versions" o- tags <- getDict "dist-tags" o- return $ PackageInfo vs tags--instance FromJSON DistInfo where- parseJSON = getObject "dist info" >=> \o -> do- tarball <- o .: "tarball"- shasum <- o .: "shasum"- return $ DistInfo tarball shasum---- | Gets a hashmap from an object, or otherwise returns an empty hashmap.-getDict :: (FromJSON a) => Text -> Object -> Parser (HashMap Text a)-getDict key o = mapM parseJSON =<< (o .:? key .!= mempty)--getObject :: String -> Value -> Parser (HashMap Text Value)-getObject _ (Object o) = return o-getObject msg v =- typeMismatch ("object (got " <> show v <> ", message " <> msg <> ")") v
− src/NixFromNpm/NpmVersion.hs
@@ -1,17 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE OverloadedStrings #-}-module NixFromNpm.NpmVersion where--import NixFromNpm.Common-import NixFromNpm.SemVer--data GitSource = Github | Bitbucket | Gist | GitLab deriving (Show, Eq)--data NpmVersionRange- = SemVerRange SemVerRange- | Tag Name- | NpmUri URI- | GitId GitSource Name Name (Maybe Name)- | LocalPath FilePath- deriving (Show, Eq)
− src/NixFromNpm/Options.hs
@@ -1,73 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE OverloadedStrings #-}-module NixFromNpm.Options where--import Options.Applicative--import NixFromNpm.Common hiding ((<>))---- | Various options we have available for nixfromnpm. As of right now,--- most of these are unimplemented.-data NixFromNpmOptions = NixFromNpmOptions {- nfnoPkgNames :: [Name], -- ^ Names of packages to build.- nfnoPkgPaths :: [Text], -- ^ Paths to package.jsons to build.- nfnoOutputPath :: Text, -- ^ Path to output built expressions to.- nfnoNoCache :: Bool, -- ^ Build all expressions from scratch.- nfnoExtendPaths :: [Text], -- ^ Extend existing expressions.- nfnoTest :: Bool, -- ^ Fetch only; don't write expressions.- nfnoRegistries :: [Text], -- ^ List of registries to query.- nfnoTimeout :: Int, -- ^ Number of seconds after which to timeout.- nfnoGithubToken :: Maybe Text -- ^ Github authentication token.-} deriving (Show, Eq)--textOption :: Mod OptionFields String -> Parser Text-textOption opts = pack <$> strOption opts--pOptions :: Maybe Text -> Parser NixFromNpmOptions-pOptions githubToken = NixFromNpmOptions- <$> many (textOption packageName)- <*> many (textOption packageFile)- <*> textOption outputDir- <*> noCache- <*> extendPaths- <*> isTest- <*> liftA2 snoc registries (pure "https://registry.npmjs.org")- <*> timeout- <*> token- where- packageName = (short 'p'- <> long "package"- <> metavar "PACKAGENAME"- <> help "Package to generate expression for")- packageFile = (short 'f'- <> long "file"- <> metavar "PACKAGEFILE"- <> help "Path to package.json to generate expression for")- outputDir = (short 'o'- <> long "output"- <> metavar "DIRECTORY"- <> help "Directory to output expressions to")- noCache = switch (long "no-cache"- <> help "Build all expressions from scratch")- extendHelp = "Use expressions at PATH called NAME"- extendPaths = many (textOption (long "extend"- <> short 'E'- <> metavar "NAME=PATH"- <> help extendHelp))- isTest = switch (long "test"- <> help "Don't write expressions; just test")- timeout = option auto (long "timeout"- <> metavar "SECONDS"- <> help "Time requests out after SECONDS seconds"- <> value 10)- registries :: Parser [Text]- registries = many $ textOption (long "registry"- <> short 'R'- <> metavar "REGISTRY"- <> help "NPM registry to query")- tokenHelp = ("Token to use for github access (also can be set with " <>- "GITHUB_TOKEN environment variable)")- token = (Just <$> textOption (long "github-token"- <> metavar "TOKEN"- <> help tokenHelp))- <|> pure githubToken
− src/NixFromNpm/Parsers/Common.hs
@@ -1,49 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE OverloadedStrings #-}-module NixFromNpm.Parsers.Common (- module Text.Parsec,- module NixFromNpm.Common,- Parser,- parse, parseFull, spaces, spaces1, sstring, schar, lexeme, pInt- ) where--import qualified Prelude as P-import Text.Parsec hiding (many, (<|>), spaces, parse, State, uncons)-import qualified Text.Parsec as Parsec--import NixFromNpm.Common hiding (try)-import NixFromNpm.SemVer--type Parser = ParsecT String () Identity---- | Given a parser and a string, attempts to parse the string.-parse :: Parser a -> Text -> Either ParseError a-parse p = Parsec.parse p "" . unpack--parseFull :: Parser a -> Text -> Either ParseError a-parseFull p = Parsec.parse (p <* eof) "" . unpack---- | Consumes any spaces (not other whitespace).-spaces :: Parser String-spaces = many $ char ' '---- | Consumes at least one space (not other whitespace).-spaces1 :: Parser String-spaces1 = many1 $ char ' '---- | Parses the given string and any trailing spaces.-sstring :: String -> Parser String-sstring = lexeme . string---- | Parses the given character and any trailing spaces.-schar :: Char -> Parser Char-schar = lexeme . char---- | Parses `p` and any trailing spaces.-lexeme :: Parser a -> Parser a-lexeme p = p <* spaces---- | Parses an integer.-pInt :: Parser Int-pInt = lexeme $ P.read <$> many1 digit
− src/NixFromNpm/Parsers/NpmVersion.hs
@@ -1,79 +0,0 @@-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE OverloadedStrings #-}--- | Tools for parsing NPM version range indicators.-module NixFromNpm.Parsers.NpmVersion where--import Data.Aeson-import qualified Data.Aeson.Types as DAT--import NixFromNpm.SemVer-import NixFromNpm.Parsers.Common-import NixFromNpm.Parsers.SemVer-import NixFromNpm.NpmVersion--pUri :: Parser NpmVersionRange-pUri = try $ fmap NpmUri $ do- parseURI <$> many anyChar >>= \case- Nothing -> unexpected "Not a valid URI"- Just uri -> case uriScheme uri of- "git:" -> return uri- "git+http:" -> return uri- "git+https:" -> return uri- "http:" -> return uri- "https:" -> return uri- scheme -> unexpected ("Unknown URI scheme " <> scheme)--pGitId :: Parser NpmVersionRange-pGitId = try $ do- let sources = choice $ map (try . sstring) ["github", "gitlab", "gist",- "bitbucket"]- source <- optionMaybe (sources <* char ':') >>= \case- Just "github" -> return Github- Just "gitlab" -> return GitLab- Just "bitbucket" -> return Bitbucket- Just "gist" -> return Gist- Nothing -> return Github- account <- many1 $ noneOf ":/"- char '/'- repo <- many1 $ noneOf "#"- ref <- optionMaybe $ char '#' *> (pack <$> many1 anyChar)- return $ GitId source (pack account) (pack repo) ref--pLocalPath :: Parser NpmVersionRange-pLocalPath = LocalPath . fromText . pack <$> do- -- The string must start with one of these prefixes.- lookAhead $ choice $ map string ["/", "./", "../", "~/"]- many anyChar--pEmptyString :: Parser NpmVersionRange-pEmptyString = try $ do- filter (/= ' ') <$> many anyChar >>= \case- [] -> return $ SemVerRange $ Geq (0, 0, 0)- _ -> unexpected "Not an empty string"--pTag :: Parser NpmVersionRange-pTag = do- filter (/= ' ') <$> many anyChar >>= \case- [] -> unexpected "empty string, not a tag"- tag -> return $ Tag $ pack tag--pNpmVersionRange :: Parser NpmVersionRange-pNpmVersionRange = choice [pEmptyString,- SemVerRange <$> pSemVerRange,- pUri,- pGitId,- pLocalPath,- pTag]--parseNpmVersionRange :: Text -> Either ParseError NpmVersionRange-parseNpmVersionRange = parse pNpmVersionRange--instance FromJSON NpmVersionRange where- parseJSON v = case v of- String s -> case parseNpmVersionRange s of- Left err -> DAT.typeMismatch- ("valid NPM version (got " <> show v <> ")"- <> " Error: " <> show err) v- Right range -> return range- _ -> DAT.typeMismatch "string" v
− src/NixFromNpm/Parsers/SemVer.hs
@@ -1,87 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE LambdaCase #-}-module NixFromNpm.Parsers.SemVer (- parseSemVer, parseSemVerRange, pSemVerRange, pSemVer- ) where--import qualified Prelude as P-import NixFromNpm.Parsers.Common-import NixFromNpm.SemVer---- | Parse a string as a version range, or return an error.-parseSemVerRange :: Text -> Either ParseError SemVerRange-parseSemVerRange = parse pSemVerRange---- | Parse a string as an explicit version, or return an error.-parseSemVer :: Text -> Either ParseError SemVer-parseSemVer = parse pSemVer---- | Parses a semantic version.-pSemVer :: Parser SemVer-pSemVer = wildcardToSemver <$> pWildCard--pVersionComp :: Parser SemVerRange-pVersionComp = do- comparator <- cmp- ver <- pSemVer- let func = case comparator of {"=" -> Eq; ">" -> Gt; "<" -> Lt;- ">=" -> Geq; "<=" -> Leq; "==" -> Eq}- return $ func ver---- | Parses a comparison operator.-cmp :: Parser String-cmp = choice $ fmap (try . sstring) [">=", "<=", ">", "<", "==", "="]---- | Parses versions with an explicit range qualifier (gt, lt, etc).-pSemVerRangeSingle :: Parser SemVerRange-pSemVerRangeSingle = choice [- wildcardToRange <$> pWildCard,- tildeToRange <$> pTildeRange,- caratToRange <$> pCaratRange,- pVersionComp- ]---- | Parses semantic version ranges joined with Ands and Ors.-pJoinedSemVerRange :: Parser SemVerRange-pJoinedSemVerRange = do- first <- pSemVerRangeSingle- option first $ do- lookAhead (sstring "||" <|> cmp) >>= \case- "||" -> Or first <$> (sstring "||" *> pJoinedSemVerRange)- _ -> And first <$> pJoinedSemVerRange---- | Parses a hyphenated range.-pHyphen :: Parser SemVerRange-pHyphen = hyphenatedRange <$> pWildCard <*> (sstring "-" *> pWildCard)---- | Parses a "wildcard" (which is a possibly partial semantic version).-pWildCard :: Parser Wildcard-pWildCard = try $ do- let seps = choice $ map sstring ["x", "X", "*"]- let bound = choice [seps *> pure Nothing, Just <$> pInt]- let stripNothings [Nothing] = []- stripNothings (Just x:xs) = x : stripNothings xs- takeWhile isJust <$> sepBy1 bound (sstring ".") >>= \case- [] -> return Any- [Just n] -> return $ One n- [Just n, Just m] -> return $ Two n m- [Just n, Just m, Just o] -> return $ Three n m o- w -> unexpected ("Invalid version " ++ show w)---- | Parses a tilde range (~1.2.3).-pTildeRange :: Parser Wildcard-pTildeRange = do- sstring "~"- -- For some reason, including the following operators after- -- a tilde is valid, but seems to have no effect.- optional $ choice [try $ sstring ">=", sstring ">", sstring "="]- pWildCard---- | Parses a carat range (^1.2.3).-pCaratRange :: Parser Wildcard-pCaratRange = sstring "^" *> pWildCard---- | Top-level parser. Parses a semantic version range.-pSemVerRange :: Parser SemVerRange-pSemVerRange = try pHyphen <|> pJoinedSemVerRange
− src/NixFromNpm/SemVer.hs
@@ -1,117 +0,0 @@-{-# LANGUAGE NoImplicitPrelude #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE LambdaCase #-}-module NixFromNpm.SemVer where--import qualified Prelude as P-import Data.Text (Text)-import qualified Data.Text as T-import Data.Aeson.Parser-import Data.Aeson-import Data.Aeson.Types (typeMismatch)--import NixFromNpm.Common--type SemVer = (Int, Int, Int)---- | A partially specified semantic version. Implicitly defines--- a range of acceptable versions, as seen in @wildcardToRange@.-data Wildcard = Any- | One Int- | Two Int Int- | Three Int Int Int- deriving (Show, Eq)--data SemVerRange- = Eq SemVer- | Gt SemVer- | Lt SemVer- | Geq SemVer- | Leq SemVer- | And SemVerRange SemVerRange- | Or SemVerRange SemVerRange- deriving (Eq)--renderSV :: SemVer -> Text-renderSV (x, y, z) = pack (renderSV' (x, y, z))--renderSV' :: SemVer -> String-renderSV' (x, y, z) = show x <> "." <> show y <> "." <> show z--instance Show SemVerRange where- show = \case- Eq sv -> "=" <> renderSV' sv- Gt sv -> ">" <> renderSV' sv- Lt sv -> "<" <> renderSV' sv- Geq sv -> ">=" <> renderSV' sv- Leq sv -> "<=" <> renderSV' sv- And svr1 svr2 -> show svr1 <> " " <> show svr2- Or svr1 svr2 -> show svr1 <> " || " <> show svr2---- | Returns whether a given semantic version matches a range.-matches :: SemVerRange -> SemVer -> Bool-matches range ver = case range of- Eq sv -> ver == sv- Gt sv -> ver > sv- Lt sv -> ver < sv- Geq sv -> ver >= sv- Leq sv -> ver <= sv- And sv1 sv2 -> matches sv1 ver && matches sv2 ver- Or sv1 sv2 -> matches sv1 ver || matches sv2 ver---- | Gets the highest-matching semver in a range.-bestMatch :: SemVerRange -> [SemVer] -> Either String SemVer-bestMatch range vs = case filter (matches range) vs of- [] -> Left "No matching versions"- vs -> Right $ maximum vs---- | Fills in zeros in a wildcard.-wildcardToSemver :: Wildcard -> SemVer-wildcardToSemver Any = (0, 0, 0)-wildcardToSemver (One n) = (n, 0, 0)-wildcardToSemver (Two n m) = (n, m, 0)-wildcardToSemver (Three n m o) = (n, m, o)---- | Translates a wildcard (partially specified version) to a range.--- Ex: 2 := >=2.0.0 <3.0.0--- Ex: 1.2.x := 1.2 := >=1.2.0 <1.3.0-wildcardToRange :: Wildcard -> SemVerRange-wildcardToRange = \case- Any -> Geq (0, 0, 0)- One n -> Geq (n, 0, 0) `And` Lt (n+1, 0, 0)- Two n m -> Geq (n, m, 0) `And` Lt (n, m + 1, 0)- Three n m o -> Eq (n, m, o)---- | Translates a ~wildcard to a range.--- Ex: ~1.2.3 := >=1.2.3 <1.(2+1).0 := >=1.2.3 <1.3.0-tildeToRange :: Wildcard -> SemVerRange-tildeToRange = \case- Any -> tildeToRange (Three 0 0 0)- One n -> tildeToRange (Three n 0 0)- Two n m -> tildeToRange (Three n m 0)- Three n m o -> And (Geq (n, m, o)) (Lt (n, m + 1, 0))---- | Translates a ^wildcard to a range.--- Ex: ^1.2.x := >=1.2.0 <2.0.0-caratToRange :: Wildcard -> SemVerRange-caratToRange = \case- One n -> And (Geq (n, 0, 0)) (Lt (n+1, 0, 0))- Two n m -> And (Geq (n, m, 0)) (Lt (n+1, 0, 0))- Three 0 0 n -> Eq (0, 0, n)- Three 0 n m -> And (Geq (0, n, m)) (Lt (0, n + 1, 0))- Three n m o -> And (Geq (n, m, o)) (Lt (n+1, 0, 0))---- | Translates two hyphenated wildcards to an actual range.--- Ex: 1.2.3 - 2.3.4 := >=1.2.3 <=2.3.4--- Ex: 1.2 - 2.3.4 := >=1.2.0 <=2.3.4--- Ex: 1.2.3 - 2 := >=1.2.3 <3.0.0-hyphenatedRange :: Wildcard -> Wildcard -> SemVerRange-hyphenatedRange wc1 wc2 = And sv1 sv2 where- sv1 = case wc1 of Any -> Geq (0, 0, 0)- One n -> Geq (n, 0, 0)- Two n m -> Geq (n, m, 0)- Three n m o -> Geq (n, m, o)- sv2 = case wc2 of Any -> Geq (0, 0, 0) -- Refers to "any version"- One n -> Lt (n+1, 0, 0)- Two n m -> Lt (n, m + 1, 0)- Three n m o -> Leq (n, m, o)
− test/Spec.hs
@@ -1,1 +0,0 @@-{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
+ top_packages.txt view
@@ -0,0 +1,108 @@+lodash+async+request+underscore+express+commander+debug+chalk+q+bluebird+mkdirp+colors+through2+coffee-script+moment+yeoman-generator+glob+gulp-util+optimist+minimist+cheerio+node-uuid+fs-extra+body-parser+react+jade+jquery+socket.io+redis+winston+yosay+uglify-js+handlebars+rimraf+semver+gulp+yargs+extend+through+mongodb+mime+underscore.string+mongoose+xml2js+grunt+ejs+superagent+mocha+marked+js-yaml+connect+shelljs+object-assign+xtend+browserify+inquirer+ember-cli-babel+aws-sdk+promise+minimatch+ws+prompt+mysql+less+morgan+cookie-parser+uuid+chai+inherits+event-stream+babel-runtime+when+readable-stream+babel+concat-stream+esprima+es6-promise+jsdom+nan+co+qs+stylus+joi+open+gulp-rename+nodemailer+backbone+path+validator+chokidar+npm+wrench+cli-color+bunyan+nconf+socket.io-client+nopt+mustache+classnames+pkginfo+clone+postcss+express-session+hoek+ncp+pg+cli-table+iconv-lite