diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ # Revision history for Arion -## Unreleased +## 0.1.2.0 -- 2020-03-05 + +* Support use of prebuilt `docker-compose.yaml`. + Separates build and execution without duplicating evaluation. + +* Avoid storing tarballs (wasting store space) by using + `dockerTools.streamLayeredImage` if available. + +* Project name is now configurable via the `project.name` option + +* Support --no-ansi, --compatibility, --log-level options ## 0.1.1.1 -- 2020-03-20 diff --git a/arion-compose.cabal b/arion-compose.cabal --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -1,7 +1,7 @@ cabal-version: 2.4 name: arion-compose -version: 0.1.1.1 +version: 0.1.2.0 synopsis: Run docker-compose with help from Nix/NixOS description: Arion is a tool for building and running applications that consist of multiple docker containers using NixOS modules. It has special support for docker images that are built with Nix, for a smooth development experience and improved performance. homepage: https://github.com/hercules-ci/arion#readme @@ -25,7 +25,7 @@ data-dir: src common common - build-depends: base >=4.12.0.0 && <4.14 + build-depends: base >=4.12.0.0 && <4.15 , aeson , aeson-pretty , async @@ -49,6 +49,7 @@ exposed-modules: Arion.Nix Arion.Aeson Arion.DockerCompose + Arion.ExtendedInfo Arion.Images Arion.Services other-modules: Paths_arion_compose diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -10,6 +10,7 @@ import Arion.Images (loadImages) import qualified Arion.DockerCompose as DockerCompose import Arion.Services (getDefaultExec) +import Arion.ExtendedInfo (loadExtendedInfoFromPath, ExtendedInfo(images, projectName)) import Options.Applicative import Control.Monad.Fail @@ -17,6 +18,8 @@ import qualified Data.Text as T import qualified Data.Text.IO as T +import Data.Aeson(Value) + import System.Posix.User (getRealUserID) data CommonOptions = @@ -24,6 +27,10 @@ { files :: NonEmpty FilePath , pkgs :: Text , nixArgs :: [Text] + , prebuiltComposeFile :: Maybe FilePath + , noAnsi :: Bool + , compatibility :: Bool + , logLevel :: Maybe Text } deriving (Show) @@ -56,6 +63,15 @@ <> help "Causes Nix to print out a stack trace in case of Nix expression evaluation errors.") -- TODO --option support (https://github.com/pcapriotti/optparse-applicative/issues/284) userNixArgs <- many (T.pack <$> strOption (long "nix-arg" <> metavar "ARG" <> help "Pass an extra argument to nix. Example: --nix-arg --option --nix-arg substitute --nix-arg false")) + prebuiltComposeFile <- optional $ strOption + ( long "prebuilt-file" + <> metavar "JSONFILE" + <> help "Do not evaluate and use the prebuilt JSONFILE instead. Causes other evaluation-related options to be ignored." ) + noAnsi <- flag False True (long "no-ansi" + <> help "Avoid ANSI control sequences") + compatibility <- flag False True (long "no-ansi" + <> help "If set, Docker Compose will attempt to convert deploy keys in v3 files to their non-Swarm equivalent") + logLevel <- optional $ fmap T.pack $ strOption (long "log-level" <> metavar "LEVEL" <> help "Set log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)") pure $ let nixArgs = userNixArgs <|> "--show-trace" <$ guard showTrace in CommonOptions{..} @@ -135,23 +151,63 @@ runBuildAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () runBuildAndDC cmd dopts opts = do - ea <- defaultEvaluationArgs opts - Arion.Nix.withBuiltComposition ea $ \path -> do - loadImages path - DockerCompose.run DockerCompose.Args - { files = [path] - , otherArgs = [cmd] ++ unDockerComposeArgs dopts - } + withBuiltComposeFile opts $ callDC cmd dopts opts True runEvalAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () runEvalAndDC cmd dopts opts = do - ea <- defaultEvaluationArgs opts - Arion.Nix.withEvaluatedComposition ea $ \path -> - DockerCompose.run DockerCompose.Args - { files = [path] - , otherArgs = [cmd] ++ unDockerComposeArgs dopts - } + withComposeFile opts $ callDC cmd dopts opts False +callDC :: Text -> DockerComposeArgs -> CommonOptions -> Bool -> FilePath -> IO () +callDC cmd dopts opts shouldLoadImages path = do + extendedInfo <- loadExtendedInfoFromPath path + when shouldLoadImages $ loadImages (images extendedInfo) + let firstOpts = projectArgs extendedInfo <> commonArgs opts + DockerCompose.run DockerCompose.Args + { files = [path] + , otherArgs = firstOpts ++ [cmd] ++ unDockerComposeArgs dopts + } + +projectArgs :: ExtendedInfo -> [Text] +projectArgs extendedInfo = + do + n <- toList (projectName extendedInfo) + ["--project-name", n] + +commonArgs :: CommonOptions -> [Text] +commonArgs opts = do + guard (noAnsi opts) + ["--no-ansi"] + <> do + guard (compatibility opts) + ["--compatibility"] + <> do + l <- toList (logLevel opts) + ["--log-level", l] + +withBuiltComposeFile :: CommonOptions -> (FilePath -> IO r) -> IO r +withBuiltComposeFile opts cont = case prebuiltComposeFile opts of + Just prebuilt -> do + cont prebuilt + Nothing -> do + args <- defaultEvaluationArgs opts + Arion.Nix.withBuiltComposition args cont + +withComposeFile :: CommonOptions -> (FilePath -> IO r) -> IO r +withComposeFile opts cont = case prebuiltComposeFile opts of + Just prebuilt -> do + cont prebuilt + Nothing -> do + args <- defaultEvaluationArgs opts + Arion.Nix.withEvaluatedComposition args cont + +getComposeValue :: CommonOptions -> IO Value +getComposeValue opts = case prebuiltComposeFile opts of + Just prebuilt -> do + decodeFile prebuilt + Nothing -> do + args <- defaultEvaluationArgs opts + Arion.Nix.evaluateComposition args + defaultEvaluationArgs :: CommonOptions -> IO EvaluationArgs defaultEvaluationArgs co = do uid <- getRealUserID @@ -166,7 +222,7 @@ runCat :: CommonOptions -> IO () runCat co = do - v <- Arion.Nix.evaluateComposition =<< defaultEvaluationArgs co + v <- getComposeValue co T.hPutStrLn stdout (pretty v) runRepl :: CommonOptions -> IO () @@ -226,13 +282,18 @@ orEmpty' m = fromMaybe mempty <$> optional m runExec :: Bool -> Bool -> Maybe Text -> Bool -> Int -> [(Text, Text)] -> Maybe Text -> Text -> [Text] -> CommonOptions -> IO () -runExec detach privileged user noTTY index envs workDir service commandAndArgs opts = do - putErrText $ "Service: " <> service - - ea <- defaultEvaluationArgs opts - Arion.Nix.withEvaluatedComposition ea $ \path -> do +runExec detach privileged user noTTY index envs workDir service commandAndArgs opts = + withComposeFile opts $ \path -> do + extendedInfo <- loadExtendedInfoFromPath path commandAndArgs'' <- case commandAndArgs of - [] -> getDefaultExec path service + [] -> do + cmd <- getDefaultExec path service + case cmd of + [] -> do + putErrText "You must provide a command via service.defaultExec or on the command line." + exitFailure + _ -> + pure cmd x -> pure x let commandAndArgs' = case commandAndArgs'' of [] -> ["/bin/sh"] @@ -252,7 +313,7 @@ ] DockerCompose.run DockerCompose.Args { files = [path] - , otherArgs = args + , otherArgs = projectArgs extendedInfo <> commonArgs opts <> args } main :: IO () diff --git a/src/haskell/lib/Arion/ExtendedInfo.hs b/src/haskell/lib/Arion/ExtendedInfo.hs new file mode 100644 --- /dev/null +++ b/src/haskell/lib/Arion/ExtendedInfo.hs @@ -0,0 +1,37 @@ +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE OverloadedStrings #-} +{- + +Parses the x-arion field in the generated compose file. + +-} +module Arion.ExtendedInfo where + +import Prelude() +import Protolude +import Data.Aeson as Aeson +import Arion.Aeson +import Control.Lens +import Data.Aeson.Lens + +data Image = Image + { image :: Maybe Text -- ^ image tar.gz file path + , imageExe :: Maybe Text -- ^ path to exe producing image tar + , imageName :: Text + , imageTag :: Text + } deriving (Eq, Show, Generic, Aeson.ToJSON, Aeson.FromJSON) + +data ExtendedInfo = ExtendedInfo { + projectName :: Maybe Text, + images :: [Image] + } deriving (Eq, Show) + +loadExtendedInfoFromPath :: FilePath -> IO ExtendedInfo +loadExtendedInfoFromPath fp = do + v <- decodeFile fp + pure ExtendedInfo { + -- TODO: use aeson derived instance? + projectName = v ^? key "x-arion" . key "project" . key "name" . _String, + images = (v :: Aeson.Value) ^.. key "x-arion" . key "images" . _Array . traverse . _JSON + } diff --git a/src/haskell/lib/Arion/Images.hs b/src/haskell/lib/Arion/Images.hs --- a/src/haskell/lib/Arion/Images.hs +++ b/src/haskell/lib/Arion/Images.hs @@ -8,42 +8,27 @@ import Prelude() import Protolude hiding (to) -import qualified Data.Aeson as Aeson -import Arion.Aeson (decodeFile) import qualified System.Process as Process import qualified Data.Text as T -import Control.Lens -import Data.Aeson.Lens -import System.IO (withFile, IOMode(ReadMode)) - - -data Image = Image - { image :: Text -- ^ file path - , imageName :: Text - , imageTag :: Text - } deriving (Generic, Aeson.ToJSON, Aeson.FromJSON, Show) +import Arion.ExtendedInfo (Image(..)) type TaggedImage = Text -- | Subject to change -loadImages :: FilePath -> IO () -loadImages fp = do - - v <- decodeFile fp +loadImages :: [Image] -> IO () +loadImages requestedImages = do - loaded <- dockerImages + loaded <- getDockerImages let - images :: [Image] - images = (v :: Aeson.Value) ^.. key "x-arion" . key "images" . _Array . traverse . _JSON - isNew i = (imageName i <> ":" <> imageTag i) `notElem` loaded - traverse_ loadImage . map (toS . image) . filter isNew $ images + traverse_ loadImage . filter isNew $ requestedImages -loadImage :: FilePath -> IO () -loadImage imgPath = withFile (imgPath) ReadMode $ \fileHandle -> do +loadImage :: Image -> IO () +loadImage (Image { image = Just imgPath, imageName = name }) = + withFile (toS imgPath) ReadMode $ \fileHandle -> do let procSpec = (Process.proc "docker" [ "load" ]) { Process.std_in = Process.UseHandle fileHandle } @@ -51,10 +36,32 @@ e <- Process.waitForProcess procHandle case e of ExitSuccess -> pass - ExitFailure code -> panic $ "docker load (" <> show code <> ") failed for " <> toS imgPath + ExitFailure code -> + panic $ "docker load failed with exit code " <> show code <> " for image " <> name <> " from path " <> imgPath +loadImage (Image { imageExe = Just imgExe, imageName = name }) = do + let loadSpec = (Process.proc "docker" [ "load" ]) { Process.std_in = Process.CreatePipe } + Process.withCreateProcess loadSpec $ \(Just inHandle) _out _err loadProcHandle -> do + let streamSpec = Process.proc (toS imgExe) [] + Process.withCreateProcess streamSpec { Process.std_out = Process.UseHandle inHandle } $ \_ _ _ streamProcHandle -> + withAsync (Process.waitForProcess loadProcHandle) $ \loadExitAsync -> + withAsync (Process.waitForProcess streamProcHandle) $ \streamExitAsync -> do + r <- waitEither loadExitAsync streamExitAsync + case r of + Right (ExitFailure code) -> panic $ "image producer for image " <> name <> " failed with exit code " <> show code <> " from executable " <> imgExe + Right ExitSuccess -> pass + Left _ -> pass + loadExit <- wait loadExitAsync + case loadExit of + ExitFailure code -> panic $ "docker load failed with exit code " <> show code <> " for image " <> name <> " produced by executable " <> imgExe + _ -> pass + pass -dockerImages :: IO [TaggedImage] -dockerImages = do +loadImage (Image { imageName = name }) = do + panic $ "image " <> name <> " doesn't specify an image file or imageExe executable" + + +getDockerImages :: IO [TaggedImage] +getDockerImages = do let procSpec = Process.proc "docker" [ "images", "--filter", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}" ] (map toS . T.lines . toS) <$> Process.readCreateProcess procSpec "" diff --git a/src/haskell/testdata/Arion/NixSpec/arion-compose.json b/src/haskell/testdata/Arion/NixSpec/arion-compose.json --- a/src/haskell/testdata/Arion/NixSpec/arion-compose.json +++ b/src/haskell/testdata/Arion/NixSpec/arion-compose.json @@ -37,6 +37,9 @@ "imageTag": "" } ], + "project": { + "name": null + }, "serviceInfo": { "webserver": { "defaultExec": [ diff --git a/src/haskell/testdata/docker-compose-example.json b/src/haskell/testdata/docker-compose-example.json --- a/src/haskell/testdata/docker-compose-example.json +++ b/src/haskell/testdata/docker-compose-example.json @@ -30,6 +30,9 @@ "imageTag": "xr4ljmz3qfcwlq9rl4mr4qdrzw93rl70" } ], + "project": { + "name": null + }, "serviceInfo": { "webserver": { "defaultExec": [ diff --git a/src/nix/modules.nix b/src/nix/modules.nix --- a/src/nix/modules.nix +++ b/src/nix/modules.nix @@ -4,4 +4,5 @@ ./modules/composition/images.nix ./modules/composition/service-info.nix ./modules/composition/arion-base-image.nix + ./modules/composition/composition.nix ] diff --git a/src/nix/modules/composition/composition.nix b/src/nix/modules/composition/composition.nix new file mode 100644 --- /dev/null +++ b/src/nix/modules/composition/composition.nix @@ -0,0 +1,24 @@ +{ config, lib, ... }: +let + inherit (lib) types mkOption; + + link = url: text: + ''link:${url}[${text}]''; + +in +{ + options = { + project.name = mkOption { + description = '' + Name of the project. + + See ${link "https://docs.docker.com/compose/reference/envvars/#compose_project_name" "COMPOSE_PROJECT_NAME"} + ''; + type = types.nullOr types.str; + default = null; + }; + }; + config = { + docker-compose.extended.project.name = config.project.name; + }; +} diff --git a/src/nix/modules/composition/images.nix b/src/nix/modules/composition/images.nix --- a/src/nix/modules/composition/images.nix +++ b/src/nix/modules/composition/images.nix @@ -16,13 +16,20 @@ (let inherit (service) build; in { - image = build.image.outPath; imageName = build.imageName or service.image.name; imageTag = if build.image.imageTag != "" then build.image.imageTag else lib.head (lib.strings.splitString "-" (baseNameOf build.image.outPath)); - }); + } // (if build.image.isExe or false + then { + imageExe = build.image.outPath; + } + else { + image = build.image.outPath; + } + ) + ); in { options = { diff --git a/src/nix/modules/service/image.nix b/src/nix/modules/service/image.nix --- a/src/nix/modules/service/image.nix +++ b/src/nix/modules/service/image.nix @@ -9,7 +9,12 @@ (pkgs.writeText "dummy-config.json" (builtins.toJSON config.image.rawConfig)) ]; - builtImage = pkgs.dockerTools.buildLayeredImage { + buildOrStreamLayeredImage = args: + if pkgs.dockerTools?streamLayeredImage + then pkgs.dockerTools.streamLayeredImage args // { isExe = true; } + else pkgs.dockerTools.buildLayeredImage args; + + builtImage = buildOrStreamLayeredImage { inherit (config.image) name contents