diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +## [Unreleased][] + +## [0.1.0.0][] (2017-02-28) + +* Initial release + +[Unreleased]: https://github.com/IxpertaSolutions/tasty-jenkins-xml/compare/v0.1.0.0...HEAD +[0.1.0.0]: https://github.com/IxpertaSolutions/tasty-jenkins-xml/compare/v0.0.0.0...v0.1.0.0 diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright 2017 Ixperta Solutions s.r.o. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the author nor the names of his contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# tasty-jenkins-xml + +[![Haskell Programming Language](https://img.shields.io/badge/language-Haskell-blue.svg)](http://www.haskell.org) +[![BSD3 License](http://img.shields.io/badge/license-BSD3-brightgreen.svg)](https://tldrlegal.com/license/bsd-3-clause-license-%28revised%29) + +[![Hackage](http://img.shields.io/hackage/v/tasty-jenkins-xml.svg)](https://hackage.haskell.org/package/tasty-jenkins-xml) +[![Hackage Dependencies](https://img.shields.io/hackage-deps/v/tasty-jenkins-xml.svg)](http://packdeps.haskellers.com/reverse/tasty-jenkins-xml) +[![Build](https://travis-ci.org/IxpertaSolutions/tasty-jenkins-xml.svg?branch=master)](https://travis-ci.org/IxpertaSolutions/tasty-jenkins-xml) + +## Description + +**An ingredient transformer version of [tasty-ant-xml][tasty-ant-xml].** + +This package implements a [tasty][] ingredient transformer that makes it +possible to output test results as JUnit XML **in addition to** other output +ingredient, e.g. a `consoleTestReporter`. Internally it invokes the +[tasty-ant-xml][] ingredient. + +To be practically useful, it implements two additions: + + * `--jxml` alias for `--xml` for [test-framework][] compatibility, + + * `--exit-success` to distinguish between _failed_ and _unstable_ builds in + Jenkins CI. + +[tasty]: https://hackage.haskell.org/package/tasty +[tasty-ant-xml]: https://hackage.haskell.org/package/tasty-ant-xml +[test-framework]: https://hackage.haskell.org/package/test-framework + +## Usage + +Example: + +```haskell +import Test.Tasty +import Test.Tasty.Runners.JenkinsXML (jenkinsXMLTransformer) + +main :: IO () +main = defaultMainWithIngredients ingredients tests + where + ingredients = [listingTests, jenkinsXMLTransformer [consoleTestReporter]] +``` + +Alternatively `jenkinsXMLTransformer` may be applied directly to +`defaultIngredients`. + +For comparison, here's a `main` that uses [tasty-ant-xml][] instead (and can't +output to console and XML at the same time): + +```haskell +import Test.Tasty +import Test.Tasty.Runners.AntXML (antXMLRunner) + +main :: IO () +main = defaultMainWithIngredients ingredients tests + where + ingredients = [listingTests, antXMLRunner, consoleTestReporter] +``` + +## Contributing + +Contributions are welcome! Documentation, examples, code, and feedback - they +all help. + +## License + +The BSD 3-Clause License, see [LICENSE][] file for details. + +[LICENSE]: https://github.com/IxpertaSolutions/tasty-jenkins-xml/blob/master/LICENSE diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/src/Test/Tasty/Runners/JenkinsXML.hs b/src/Test/Tasty/Runners/JenkinsXML.hs new file mode 100644 --- /dev/null +++ b/src/Test/Tasty/Runners/JenkinsXML.hs @@ -0,0 +1,147 @@ +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE NoImplicitPrelude #-} + +module Test.Tasty.Runners.JenkinsXML + ( jenkinsXMLTransformer + ) + where + +import Control.Applicative (pure) +import Control.Monad ((>>=), msum, join) +import Data.Bool (Bool(True, False), (||)) +import Data.Foldable (concatMap) +import Data.Function ((.), ($), flip) +import Data.Functor (fmap) +import Data.Maybe (Maybe(Nothing, Just), fromMaybe) +import Data.Monoid ((<>)) +import Data.Proxy (Proxy(Proxy)) +import Data.Traversable (mapM) +import Data.Typeable (Typeable) +import System.IO (IO, FilePath) + +import Test.Tasty.Options + ( IsOption(defaultValue, parseValue, optionName, optionHelp, optionCLParser) + , OptionSet + , OptionDescription(Option) + , flagCLParser + , lookupOption + , safeRead + , setOption + ) +import Test.Tasty.Runners + ( Ingredient(TestReporter, TestManager) + , StatusMap + , TestTree + , Time + , launchTestTree + ) +import Test.Tasty.Runners.AntXML + ( antXMLRunner + , AntXMLPath(AntXMLPath) + ) + + +newtype CompatAntXMLPath = CompatAntXMLPath FilePath deriving Typeable + +instance IsOption (Maybe CompatAntXMLPath) where + defaultValue = Nothing + parseValue = Just . Just . CompatAntXMLPath + optionName = pure "jxml" + optionHelp = pure "An alias for --xml" + +newtype ExitSuccess = ExitSuccess { isExitSuccess :: Bool } deriving Typeable + +instance IsOption ExitSuccess where + defaultValue = ExitSuccess False + parseValue = fmap ExitSuccess . safeRead + optionName = pure "exit-success" + optionHelp = pure "Exit with status 0 even if some tests failed" + optionCLParser = flagCLParser Nothing (ExitSuccess True) + +type ReportFn = StatusMap -> IO (Time -> IO Bool) + +antXmlOptions :: [OptionDescription] +antXmlReport :: OptionSet -> TestTree -> Maybe ReportFn +TestReporter antXmlOptions antXmlReport = antXMLRunner + +-- | This 'Ingredient' transformer adds the possibility to produce a JUnit XML +-- file __in addition to__ the output producer by another 'Ingredient'. +-- Internally it invokes 'Test.Tasty.Runners.AntXML.antXMLRunner'. +-- +-- To be practically useful, it implements two additions: +-- +-- * @--jxml@ alias for @--xml@ for @test-framework@ compatibility, +-- +-- * @--exit-success@ to distinguish between /failed/ and /unstable/ builds +-- in Jenkins CI. +jenkinsXMLTransformer :: [Ingredient] -> Ingredient +jenkinsXMLTransformer = + testReporterTransformer (exitOption : compatOption : antXmlOptions) $ + reportTransform antXmlTransform . applyCompatOpt + where + exitOption = Option (Proxy :: Proxy ExitSuccess) + compatOption = Option (Proxy :: Proxy (Maybe CompatAntXMLPath)) + + applyCompatOpt opts = case lookupOption opts of + Nothing -> opts + Just (CompatAntXMLPath path) -> + setOption (Just (AntXMLPath path)) opts + + antXmlTransform opts testTree smap totalTime = fmap exit . antXml + where + exit retVal = retVal || isExitSuccess (lookupOption opts) + + antXml retVal = fromMaybe retVal `fmap` tryReport antXmlReport + + tryReport r = + (join . fmap ($ totalTime) . ($ smap)) `mapM` r opts testTree + +-- Combinator for transforming the final report/record callback of a +-- 'TestReporter'. +reportTransform + :: (OptionSet -> TestTree -> StatusMap -> Time -> Bool -> IO Bool) + -> OptionSet -> TestTree -> ReportFn -> ReportFn +reportTransform f opts testTree reportFn smap = + reportFn smap >>= \k -> pure $ \totalTime -> + k totalTime >>= f opts testTree smap totalTime + +-- Combinator for building ingredient transformers that change the behaviour +-- of an existing 'TestReporter' ingredient. +testReporterTransformer + :: [OptionDescription] + -- ^ Additional command-line options + -> (OptionSet -> TestTree -> ReportFn -> ReportFn) + -- ^ Function to transform the 'ReportFn' of a 'TestReporter', see + -- 'tryIngredient'' and 'launchTestTree' for details. + -> [Ingredient] + -- ^ Ingredients to transform and run. + -> Ingredient +testReporterTransformer options transform ingredients = + TestManager (options <> existingOptions) $ + tryIngredients' transform ingredients + where + existingOptions = flip concatMap ingredients $ \ingredient -> + case ingredient of + TestReporter opts _ -> opts + TestManager opts _ -> opts + +-- | Modified version of 'Test.Tasty.Ingredients.tryIngredient' that +-- transforms the 'ReportFn' by a given function, if the 'Ingredient' happens +-- to be a 'TestReporter'. 'TestManager' is left unmodified. +tryIngredient' + :: (OptionSet -> TestTree -> ReportFn -> ReportFn) + -> Ingredient -> OptionSet -> TestTree -> Maybe (IO Bool) +tryIngredient' f (TestReporter _ report) opts testTree = do -- Maybe monad + reportFn <- report opts testTree + pure $ launchTestTree opts testTree $ f opts testTree reportFn +tryIngredient' _ (TestManager _ manage) opts testTree = + manage opts testTree + +-- | Modified version of 'Test.Tasty.Ingredients.tryIngredients' that +-- transforms the 'ReportFn' by a given function, if the 'Ingredient' happens +-- to be a 'TestReporter'. 'TestManager' is left unmodified. +tryIngredients' + :: (OptionSet -> TestTree -> ReportFn -> ReportFn) + -> [Ingredient] -> OptionSet -> TestTree -> Maybe (IO Bool) +tryIngredients' f ins opts tree = + msum $ fmap (\i -> tryIngredient' f i opts tree) ins diff --git a/tasty-jenkins-xml.cabal b/tasty-jenkins-xml.cabal new file mode 100644 --- /dev/null +++ b/tasty-jenkins-xml.cabal @@ -0,0 +1,89 @@ +-- This file has been generated from package.yaml by hpack version 0.15.0. +-- +-- see: https://github.com/sol/hpack + +name: tasty-jenkins-xml +version: 0.1.0.0 +synopsis: Render tasty output to XML for Jenkins (ingredient transformer) +description: A tasty ingredient transformer (meaning consoleTestReporter or any other ingredient can run as well) to output test results in XML, using the Ant schema. This XML can be consumed by the Jenkins continuous integration framework. +category: Testing +homepage: https://github.com/IxpertaSolutions/tasty-jenkins-xml#readme +bug-reports: https://github.com/IxpertaSolutions/tasty-jenkins-xml/issues +author: Ixcom Core Team +maintainer: ixcom-core@ixperta.com +copyright: (c) 2017 Ixperta Solutions s.r.o. +license: BSD3 +license-file: LICENSE +tested-with: GHC==7.10.3, GHC==8.0.2 +build-type: Simple +cabal-version: >= 1.10 + +extra-source-files: + CHANGELOG.md + README.md + +source-repository head + type: git + location: https://github.com/IxpertaSolutions/tasty-jenkins-xml + +flag pedantic + description: Pass additional warning flags and -Werror to GHC. + manual: True + default: False + +library + hs-source-dirs: + src + ghc-options: -Wall -fwarn-tabs -fwarn-implicit-prelude + build-depends: + base >=4 && <5 + , tasty >= 0.10 && < 0.12 + , tasty-ant-xml == 1.0.* + if flag(pedantic) + ghc-options: -Werror + if impl(ghc >=8) + ghc-options: -Wredundant-constraints + exposed-modules: + Test.Tasty.Runners.JenkinsXML + other-modules: + Paths_tasty_jenkins_xml + default-language: Haskell2010 + +test-suite hlint + type: exitcode-stdio-1.0 + main-is: hlint.hs + hs-source-dirs: + test + ghc-options: -Wall -fwarn-tabs -fwarn-implicit-prelude + build-depends: + base >=4 && <5 + , tasty >= 0.10 && < 0.12 + , hlint >=1.9 + if flag(pedantic) + ghc-options: -Werror + if impl(ghc >=8) + ghc-options: -Wredundant-constraints + default-language: Haskell2010 + +test-suite spec + type: exitcode-stdio-1.0 + main-is: spec.hs + hs-source-dirs: + test + ghc-options: -Wall -fwarn-tabs -fwarn-implicit-prelude + build-depends: + base >=4 && <5 + , tasty >= 0.10 && < 0.12 + , bytestring >=0.9 + , directory >=1.2.3.0 + , hspec >=2.2 + , io-capture >=1.0 + , mockery >=0.3 + , tasty-hunit >=0.9 + , tasty-jenkins-xml + , unix >=2 + if flag(pedantic) + ghc-options: -Werror + if impl(ghc >=8) + ghc-options: -Wredundant-constraints + default-language: Haskell2010 diff --git a/test/hlint.hs b/test/hlint.hs new file mode 100644 --- /dev/null +++ b/test/hlint.hs @@ -0,0 +1,23 @@ +{-# LANGUAGE NoImplicitPrelude #-} + +module Main (main) where + +import Control.Monad (unless) +import Data.Function (($)) +import Data.List (null) +import Data.Monoid ((<>)) +import System.IO (IO, putStrLn) +import System.Exit (exitFailure) + +import Language.Haskell.HLint (hlint) + + +main :: IO () +main = do + putStrLn "" -- less confusing output, test-framework does this too + hints <- hlint $ hlintOpts <> ["src", "test"] + unless (null hints) exitFailure + where + hlintOpts = + [ "-XNoPatternSynonyms" + ] diff --git a/test/spec.hs b/test/spec.hs new file mode 100644 --- /dev/null +++ b/test/spec.hs @@ -0,0 +1,110 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE NoImplicitPrelude #-} +{-# LANGUAGE OverloadedStrings #-} + +module Main (main) where + +import Control.Monad (forM_) +import Data.Bool (Bool(True, False)) +import Data.Eq ((==)) +import Data.Function (($)) +import Data.Int (Int) +import Data.Maybe (Maybe(Just), maybe) +import Data.Monoid ((<>)) +import Data.String (String) +import System.Directory (doesFileExist) +import System.Environment (withArgs) +import System.Exit (ExitCode(ExitSuccess, ExitFailure)) +import System.IO (IO, readFile) + +import Data.ByteString.Lazy.Char8 (unpack) +import System.IO.Capture (capture) +import System.Posix.Process (ProcessStatus(Exited)) +import Test.Hspec + ( Selector + , around_ + , describe + , hspec + , it + , shouldBe + , shouldContain + , shouldReturn + , shouldThrow + ) +import Test.Mockery.Directory (inTempDirectory) +import Test.Tasty + ( TestTree + , defaultMainWithIngredients + , defaultIngredients + , testGroup + ) +import Test.Tasty.HUnit (assert, testCase) +import Test.Tasty.Runners.JenkinsXML (jenkinsXMLTransformer) + + +{-# ANN module ("HLint: ignore Reduce duplication" :: String) #-} +{-# ANN module ("HLint: ignore Redundant do" :: String) #-} + +main :: IO () +main = hspec $ do + describe "antXMLTransformer" $ around_ inTempDirectory $ do + it "lists options in --help" $ do + (out, err, exc, status) <- capture $ + withArgs ["--help"] $ + tastyMain `shouldThrow` exitSuccess + status `shouldBe` Just (Exited ExitSuccess) + (err, exc) `shouldBe` ("", "") + unpack out `shouldContain` "Usage:" + unpack out `shouldContain` "--xml" + unpack out `shouldContain` "--jxml" + unpack out `shouldContain` "--exit-success" + it "reports failure when some test fails" $ do + (out, err, exc, status) <- capture $ + withArgs [] $ + tastyMain `shouldThrow` exitFailure (Just 1) + status `shouldBe` Just (Exited ExitSuccess) + (err, exc) `shouldBe` ("", "") + unpack out `shouldContain` "\n1 out of 2 tests failed" + doesFileExist "tasty.xml" `shouldReturn` False + it "exits successfully even when some test fails (--exit-success)" $ do + (out, err, exc, status) <- capture $ + withArgs ["--exit-success"] $ + tastyMain `shouldThrow` exitSuccess + status `shouldBe` Just (Exited ExitSuccess) + (err, exc) `shouldBe` ("", "") + unpack out `shouldContain` "\n1 out of 2 tests failed" + doesFileExist "tasty.xml" `shouldReturn` False + forM_ ["--xml", "--jxml"] $ \flag -> + it ("writes xml in addition to console output (" <> flag <> ")") $ do + (out, err, exc, status) <- capture $ + withArgs [flag, "tasty.xml"] $ + tastyMain `shouldThrow` exitFailure (Just 1) + status `shouldBe` Just (Exited ExitSuccess) + (err, exc) `shouldBe` ("", "") + unpack out `shouldContain` "\n1 out of 2 tests failed" + xml <- readFile "tasty.xml" + xml `shouldContain` " True + ExitFailure _ -> False + +exitFailure :: Maybe Int -> Selector ExitCode +exitFailure code = \case + ExitSuccess -> False + ExitFailure code' -> maybe True (code' ==) code