diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,10 @@ ## Unreleased - +## 0.1.1.0 -- 2020-03-19 + +* Support Nixpkgs 20.03 +* Fixes for macOS ## 0.1.0.0 -- 2019-10-04 diff --git a/README.asciidoc b/README.asciidoc --- a/README.asciidoc +++ b/README.asciidoc @@ -1,274 +1,7 @@ -== Introduction - Arion is a tool for building and running applications that consist of multiple docker containers using NixOS modules. It has special support for docker images that are built with Nix, for a smooth development experience and improved performance. -It is built on top of https://docs.docker.com/compose/overview/[Docker -Compose], which implements the container orchestration functionality. -Instead of configuring the compositions in YAML files like -`docker-compose.yaml`, Arion uses the https://nixos.org/nix/[Nix] -language to declare the compositions. Because of this, Arion gives you -the ability to declare your deployments, configuration and packaging -in the same language. By replacing multiple tools with a single -language, you decrease your mental load and you can more easily -refactor and maintain your configurations. - -Although Arion can be used as a Docker Compose with an improved -configuration front end, there is more to be gained from integrating -with Nix. In particular, the more structured approach of Nix compared -to Dockerfiles allows the following: - - * Build components of your image in *parallel, automatically* - * *Share packages between images*, regardless of the order they were - added - * Improve performance by *skipping container - image creation* - * Work with *structured data instead of strings*, - templates and a multitude of expression languages - * Refactor across deployments, configuration and packaging - -Arion allows to compose containers with different granularity: - - * <> - * <> - * <> - * <> - -== Installation - -=== Nix - -```bash -$ nix-env -iA arion -f https://github.com/hercules-ci/arion/tarball/master -``` - -=== NixOS - -Add this module to your NixOS configuration: - -```nix -{ ... }: { - environment.systemPackages = [ (import (builtins.fetchTarball https://github.com/hercules-ci/arion/tarball/master) {}).arion ]; - virtualisation.docker.enable = true; - users.extraUsers.myuser.extraGroups = ["docker"]; -} -``` - -//// - -== Not installing: use it in a project - -TODO: describe: using nix-shell or in a script, building images as - part of nix-build, pinning, see also todomvc-nix. - -TODO: exposed Nix functions: arion.build, arion.eval (a bit of IFD) - - -//// - - -== Usage - -Arion is configured declaratively with two files: - -=== arion-pkgs.nix - -Arion needs `arion-pkgs.nix` to import nixpkgs, it's contents can be as simple as: - -```nix -import {} -``` - -or more sophisticated (recommended) setup with https://github.com/nmattia/niv[Niv]. - -=== arion-compose.nix - -Describe containers using NixOS-style modules. There are a few options: - -==== Minimal: Plain command using nixpkgs - -`examples/minimal/arion-compose.nix`: - -```nix -{ pkgs, ... }: -{ - config.docker-compose.services = { - - webserver = { - service.useHostStore = true; - service.command = [ "sh" "-c" '' - cd "$$WEB_ROOT" - ${pkgs.python3}/bin/python -m http.server - '' ]; - service.ports = [ - "8000:8000" # host:container - ]; - service.environment.WEB_ROOT = "${pkgs.nix.doc}/share/doc/nix/manual"; - }; - }; -} - - -``` - -==== NixOS: run only one systemd service - -`examples/nixos-unit/arion-compose.nix`: - -```nix - -{ - docker-compose.services.webserver = { config, pkgs, ... }: { - - nixos.configuration = {config, pkgs, ...}: { - boot.isContainer = true; - services.nginx.enable = true; - services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual"; - system.build.run-nginx = pkgs.writeScript "run-nginx" '' - #!${pkgs.bash}/bin/bash - PATH='${config.systemd.services.nginx.environment.PATH}' - echo nginx:x:${toString config.users.users.nginx.uid}:${toString config.users.groups.nginx.gid}:nginx web server user:/var/empty:/bin/sh >>/etc/passwd - echo nginx:x:${toString config.users.groups.nginx.gid}:nginx >>/etc/group - ${config.systemd.services.nginx.runner} - ''; - }; - service.command = [ config.nixos.build.run-nginx ]; - service.useHostStore = true; - service.ports = [ - "8000:80" # host:container - ]; - }; -} - -``` - -==== NixOS: run full OS - -`examples/full-nixos/arion-compose.nix`: - -```nix -{ - docker-compose.services.webserver = { pkgs, ... }: { - nixos.useSystemd = true; - nixos.configuration.boot.tmpOnTmpfs = true; - nixos.configuration.services.nginx.enable = true; - nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual"; - service.useHostStore = true; - service.ports = [ - "8000:80" # host:container - ]; - }; -} -``` - -==== Docker image from DockerHub - -```nix -{ - docker-compose.services.postgres = { - service.image = "postgres:10"; - service.volumes = [ "${toString ./.}/postgres-data:/var/lib/postgresql/data" ]; - service.environment.POSTGRES_PASSWORD = "mydefaultpass"; - }; -} -``` - -=== Run - -Start containers and watch their logs: - -```bash -$ arion up -d -$ arion logs -f -``` - -You can go to `examples/*/` and run these commands to give it a quick try. - -== A full featured Nix command example - -To see how Arion can be used in a project, have a look at -https://github.com/nix-community/todomvc-nix/tree/master/deploy/arion[todomvc-nix]. - -```bash -$ git clone https://github.com/nix-community/todomvc-nix -$ cd todomvc-nix/deploy/arion -$ arion up -``` - -== Project Status - -This project was born out of a process supervision need for local -development environments while working on -https://www.hercules-ci.com[Hercules CI]. (It was also born out of -ancient Greek deities disguised as horses. More on that later.) - -If you do want to use Arion for production environments, you’ll probably -want to either build normal container images or manage garbage -collection roots if you control the deployment host. Neither scenario is -made easier by arion at this time. - -Arion has run successfully on Linux distributions other than NixOS, but we only perform CI for Arion on NixOS. - - -== How it works - -Arion is essentially a thin wrapper around Nix and docker-compose. When -it runs, it does the following: - -* Evaluate the configuration using Nix, producing a -`docker-compose.yaml` and a garbage collection root -* Invoke `docker-compose` -* Clean up the garbage collection root - -Most of the interesting stuff happens in Arion’s Nix expressions, where -it runs the module system (known from NixOS) and provides the -configuration that makes the Docker Compose file do the things it needs -to do. - -One of the more interesting built-in modules is the -link:src/nix/modules/service/host-store.nix[host-store.nix module] which -performs the bind mounts to make the host Nix store available in the -container. - -== FAQ - -=== Do I need to use Hercules CI? - -Nope, it’s just Nix and Docker Compose under the hood. - -=== What about garbage collection? - -Arion removes the need for garbage collecting docker images, delegating -this task to Nix. - -Arion creates a garbage collection root and cleans it up after -completing the command. This means that `arion up` without `-d` is safe -with respect to garbage collection. A deployment that is more serious -than local development must leave a GC root on the deployment host. This -use case is not supported as of now. - -=== Why is my container not running latest code? - -Restart it with `arion restart ` or if you've changed the image rebuild -them using `arion up -d --always-recreate-deps `. - -=== What is messing with my environment variables? - -Docker Compose performs its own environment variable substitution. This -can be a little annoying in `services.command` for example. Either -reference a script from `pkgs.writeScript` or escape the dollar sign as -`$$`. - -=== Why name it ``Arion``? - -Arion comes from Greek mythology. Poseidon, the god of ~Docker~ the seas -had his eye on Demeter. Demeter tried to trick him by disguising as a -horse, but Poseidon saw through the deception and they had Arion. - -So Arion is a super fast divine horse; the result of some weird mixing. -Also it talks. - -(And we feel morally obliged to name our stuff after Greek mythology) +# https://docs.hercules-ci.com/arion/[Intro and Documentation] diff --git a/arion-compose.cabal b/arion-compose.cabal --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: arion-compose -version: 0.1.0.0 +version: 0.1.1.0 synopsis: Run docker-compose with help from Nix/NixOS description: Arion is a tool for building and running applications that consist of multiple docker containers using NixOS modules. It has special support for docker images that are built with Nix, for a smooth development experience and improved performance. homepage: https://github.com/hercules-ci/arion#readme diff --git a/src/haskell/test/Arion/NixSpec.hs b/src/haskell/test/Arion/NixSpec.hs --- a/src/haskell/test/Arion/NixSpec.hs +++ b/src/haskell/test/Arion/NixSpec.hs @@ -18,7 +18,7 @@ { evalUid = 123 , evalModules = NEL.fromList ["src/haskell/testdata/Arion/NixSpec/arion-compose.nix"] - , evalPkgs = "import {}" + , evalPkgs = "import { system = \"x86_64-linux\"; }" , evalWorkDir = Nothing , evalMode = ReadOnly , evalUserArgs = ["--show-trace"] diff --git a/src/haskell/testdata/Arion/NixSpec/arion-compose.nix b/src/haskell/testdata/Arion/NixSpec/arion-compose.nix --- a/src/haskell/testdata/Arion/NixSpec/arion-compose.nix +++ b/src/haskell/testdata/Arion/NixSpec/arion-compose.nix @@ -1,5 +1,5 @@ { - docker-compose.services.webserver = { pkgs, ... }: { + services.webserver = { pkgs, ... }: { nixos.useSystemd = true; nixos.configuration.boot.tmpOnTmpfs = true; nixos.configuration.services.nginx.enable = true; diff --git a/src/nix/eval-composition.nix b/src/nix/eval-composition.nix --- a/src/nix/eval-composition.nix +++ b/src/nix/eval-composition.nix @@ -18,12 +18,7 @@ builtinModules = [ argsModule - ./modules/composition/docker-compose.nix - ./modules/composition/host-environment.nix - ./modules/composition/images.nix - ./modules/composition/service-info.nix - ./modules/composition/arion-base-image.nix - ]; + ] ++ import ./modules.nix; argsModule = { _file = ./eval-composition.nix; diff --git a/src/nix/modules.nix b/src/nix/modules.nix new file mode 100644 --- /dev/null +++ b/src/nix/modules.nix @@ -0,0 +1,7 @@ +[ + ./modules/composition/docker-compose.nix + ./modules/composition/host-environment.nix + ./modules/composition/images.nix + ./modules/composition/service-info.nix + ./modules/composition/arion-base-image.nix +] diff --git a/src/nix/modules/composition/arion-base-image.nix b/src/nix/modules/composition/arion-base-image.nix --- a/src/nix/modules/composition/arion-base-image.nix +++ b/src/nix/modules/composition/arion-base-image.nix @@ -10,7 +10,7 @@ tag = lib.head (lib.strings.splitString "-" (baseNameOf builtImage.outPath)); name = "arion-base"; - builtImage = pkgs.dockerTools.buildLayeredImage { + builtImage = pkgs.dockerTools.buildImage { inherit name; contents = pkgs.runCommand "minimal-contents" {} '' mkdir -p $out/bin $out/usr/bin diff --git a/src/nix/modules/service/default-exec.nix b/src/nix/modules/service/default-exec.nix --- a/src/nix/modules/service/default-exec.nix +++ b/src/nix/modules/service/default-exec.nix @@ -9,7 +9,7 @@ default = ["/bin/sh"]; description = '' Container program and arguments to invoke when calling - arion exec <service.name> without further arguments. + `arion exec ` without further arguments. ''; }; }; diff --git a/src/nix/modules/service/docker-compose-service.nix b/src/nix/modules/service/docker-compose-service.nix --- a/src/nix/modules/service/docker-compose-service.nix +++ b/src/nix/modules/service/docker-compose-service.nix @@ -11,11 +11,11 @@ inherit (types) listOf nullOr attrsOf str either int bool; link = url: text: - ''${text}''; + ''link:${url}[${text}]''; dockerComposeRef = fragment: - ''See Docker Compose#${fragment}''; + ''See ${link "https://docs.docker.com/compose/compose-file/#${fragment}" "Docker Compose#${fragment}"}''; dockerComposeKitchenSink = '' - Analogous to the docker run counterpart. + Analogous to the `docker run` counterpart. ${dockerComposeRef "domainname-hostname-ipc-mac_address-privileged-read_only-shm_size-stdin_open-tty-user-working_dir"} ''; @@ -33,7 +33,7 @@ out.service = mkOption { type = attrsOf types.unspecified; description = '' - Raw input for the service in docker-compose.yaml. + Raw input for the service in `docker-compose.yaml`. You should not need to use this option. If anything is missing, please contribute the missing option. @@ -47,7 +47,7 @@ service.name = mkOption { type = str; description = '' - The name of the service - <name> in the composition-level services.<name> + The name of the service - `` in the composition-level `services.` ''; readOnly = true; }; @@ -110,7 +110,7 @@ default = []; description = '' See ${link "https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities" - "docker run --device documentation"} + "`docker run --device` documentation"} ${dockerComposeRef "devices"} ''; @@ -201,11 +201,11 @@ description = '' Enable/disable linux capabilities, or pick Docker's default. - Setting a capability to true means that it will be - "added". Setting it to false means that it will be "dropped". + Setting a capability to `true` means that it will be + "added". Setting it to `false` means that it will be "dropped". ${dockerComposeRef "cap_add-cap_drop"} - Omitted and null capabilities will therefore be set + Omitted and `null` capabilities will therefore be set according to Docker's ${ link "https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities" "default list of capabilities." diff --git a/src/nix/modules/service/extended-info.nix b/src/nix/modules/service/extended-info.nix --- a/src/nix/modules/service/extended-info.nix +++ b/src/nix/modules/service/extended-info.nix @@ -12,10 +12,10 @@ type = attrsOf unspecified; description = '' Information about a service to include in the Docker Compose file, - but that will not be used by the docker-compose command + but that will not be used by the `docker-compose`> command itself. - It will be inserted in x-arion.serviceInfo.<service.name>. + It will be inserted in `x-arion.serviceInfo.`. ''; default = {}; }; diff --git a/src/nix/modules/service/host-store.nix b/src/nix/modules/service/host-store.nix --- a/src/nix/modules/service/host-store.nix +++ b/src/nix/modules/service/host-store.nix @@ -1,6 +1,6 @@ /* - This service-level bind mounts the host store into the container + This service-level module bind mounts the host store into the container when the service.useHostStore option is set to true. */ diff --git a/src/nix/modules/service/image.nix b/src/nix/modules/service/image.nix --- a/src/nix/modules/service/image.nix +++ b/src/nix/modules/service/image.nix @@ -40,7 +40,7 @@ build.image = mkOption { type = nullOr package; description = '' - Docker image derivation to be docker loaded. + Docker image derivation to be `docker load`ed. ''; internal = true; }; @@ -58,11 +58,11 @@ type = bool; description = '' Whether to build this image with Nixpkgs' - dockerTools.buildLayeredImage - and then load it with docker load. + `dockerTools.buildLayeredImage` + and then load it with `docker load`. - By default, an image will be built with Nix unless - is set. See also , which defaults to + By default, an image will be built with Nix unless `service.image` + is set. See also `image.name`, which defaults to the service name. ''; }; @@ -73,8 +73,8 @@ description = '' A human readable name for the docker image. - Shows up in the docker ps output in the - IMAGE column, among other places. + Shows up in the `docker ps` output in the + `IMAGE` column, among other places. ''; }; image.contents = mkOption { @@ -92,11 +92,11 @@ been modeled in the Arion module system. This attribute set does not have an appropriate merge function. - Please use the specific image options instead. + Please use the specific `image` options instead. Run-time configuration of the container. A full list of the - options are available at in the Docker Image Specification - v1.2.0. + options is available in the https://github.com/moby/moby/blob/master/image/spec/v1.2.md#image-json-field-descriptions[Docker Image Specification + v1.2.0]. ''; }; image.command = mkOption { diff --git a/src/nix/modules/service/nixos-init.nix b/src/nix/modules/service/nixos-init.nix --- a/src/nix/modules/service/nixos-init.nix +++ b/src/nix/modules/service/nixos-init.nix @@ -13,7 +13,7 @@ description = '' When enabled, call the NixOS systemd-based init system. - Configure NixOS with nixos.configuration. + Configure NixOS with the `nixos.configuration` option. ''; }; }; diff --git a/src/nix/modules/service/nixos.nix b/src/nix/modules/service/nixos.nix --- a/src/nix/modules/service/nixos.nix +++ b/src/nix/modules/service/nixos.nix @@ -18,7 +18,7 @@ This option is unused by default, because not all images use NixOS. - One way to use this is to enable nixos.useSystemd, but the + One way to use this is to enable `nixos.useSystemd`, but the NixOS configuration can be used in other ways. ''; }; @@ -27,11 +27,11 @@ type = attrs; readOnly = true; description = '' - NixOS build products from config.system.build, such as toplevel and etc. + NixOS build products from `config.system.build`, such as `toplevel` and `etc`. This option is unused by default, because not all images use NixOS. - One way to use this is to enable nixos.useSystemd, but the + One way to use this is to enable `nixos.useSystemd`, but the NixOS configuration can be used in other ways. ''; }; @@ -44,7 +44,7 @@ This option is unused by default, because not all images use NixOS. - One way to use this is to enable nixos.useSystemd, but the + One way to use this is to enable `nixos.useSystemd`, but the NixOS configuration can be used in other ways. ''; };