diff --git a/CHANGES.markdown b/CHANGES.markdown
--- a/CHANGES.markdown
+++ b/CHANGES.markdown
@@ -1,3 +1,7 @@
+# 0.2.3
+  * Conditionals in Cabal files are now solved ([#35](https://github.com/martijnbastiaan/doctest-parallel/pull/37)). Thanks to @philderbeast for the report and contributions.
+  * Unexpected outputs in `$setup` blocks are no longer ignored ([#39](https://github.com/martijnbastiaan/doctest-parallel/pull/39))
+
 # 0.2.2
   * Command line arguments (such as `--randomize-order`) can now be overridden on a per-module basis ([#25](https://github.com/martijnbastiaan/doctest-parallel/pull/25))
   * Implicit pre-test module imports can now be disabled using `--no-implicit-module-import`. This can help to test functions from non-exposed modules ([#26](https://github.com/martijnbastiaan/doctest-parallel/pull/26))
diff --git a/README.markdown b/README.markdown
--- a/README.markdown
+++ b/README.markdown
@@ -377,6 +377,7 @@
  * Michael Orlitzky
  * Michael Snoyman
  * Nick Smallbone
+ * Phil de Joux
  * Sakari Jokinen
  * Simon Hengel
  * Sönke Hahn
diff --git a/doctest-parallel.cabal b/doctest-parallel.cabal
--- a/doctest-parallel.cabal
+++ b/doctest-parallel.cabal
@@ -1,7 +1,7 @@
 cabal-version: 2.0
 
 name:           doctest-parallel
-version:        0.2.2
+version:        0.2.3
 synopsis:       Test interactive Haskell examples
 description:    The doctest program checks examples in source code comments.  It is modeled
                 after doctest for Python (<https://docs.python.org/3/library/doctest.html>).
@@ -12,12 +12,13 @@
 homepage:       https://github.com/martijnbastiaan/doctest-parallel#readme
 license:        MIT
 license-file:   LICENSE
-copyright:      (c) 2009-2018 Simon Hengel, 2021 Martijn Bastiaan
+copyright:      (c) 2009-2018 Simon Hengel, 2021-2022 Martijn Bastiaan
 author:         Martijn Bastiaan <martijn@hmbastiaan.nl>
 maintainer:     Martijn Bastiaan <martijn@hmbastiaan.nl>
 build-type:     Simple
 tested-with:
-    GHC == 8.4.4
+    GHC == 8.2.2
+  , GHC == 8.4.4
   , GHC == 8.6.5
   , GHC == 8.8.4
   , GHC == 8.10.7
@@ -86,7 +87,7 @@
   other-modules:
       Paths_doctest_parallel
   build-depends:
-      Cabal
+      Cabal >= 3.4
     , Glob
     , base >=4.10 && <5
     , base-compat >=0.7.0
@@ -97,7 +98,7 @@
     , exceptions
     , extra
     , filepath
-    , ghc >=8.4 && <9.3
+    , ghc >=8.2 && <9.3
     , ghc-paths >=0.1.0.9
     , pretty
     , process
diff --git a/src/Test/DocTest/Helpers.hs b/src/Test/DocTest/Helpers.hs
--- a/src/Test/DocTest/Helpers.hs
+++ b/src/Test/DocTest/Helpers.hs
@@ -1,6 +1,7 @@
 {-# LANGUAGE CPP #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE MultiWayIf #-}
+{-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE ViewPatterns #-}
 
@@ -12,6 +13,7 @@
   ( canonicalizePath, doesFileExist )
 import System.FilePath ((</>), isDrive, takeDirectory)
 import System.FilePath.Glob (glob)
+import System.Info (compilerVersion)
 
 #if __GLASGOW_HASKELL__ < 804
 import Data.Monoid ((<>))
@@ -25,19 +27,25 @@
   ( Extension (DisableExtension, EnableExtension, UnknownExtension) )
 import Distribution.Types.UnqualComponentName ( unUnqualComponentName )
 import Distribution.PackageDescription
-  ( CondTree(CondNode, condTreeData), GenericPackageDescription (condLibrary)
+  ( GenericPackageDescription (condLibrary)
   , exposedModules, libBuildInfo, hsSourceDirs, defaultExtensions, package
-  , packageDescription, condSubLibraries, includeDirs, autogenModules )
+  , packageDescription, condSubLibraries, includeDirs, autogenModules, ConfVar )
 
+import Distribution.Compiler (CompilerFlavor(GHC))
+import Distribution.PackageDescription.Parsec (readGenericPackageDescription)
 import Distribution.Pretty (prettyShow)
+import Distribution.System (buildArch, buildOS)
+import Distribution.Types.Condition (Condition(..))
+import Distribution.Types.CondTree
+import Distribution.Types.ConfVar (ConfVar(..))
+import Distribution.Types.Version (Version, mkVersion')
+import Distribution.Types.VersionRange (withinRange)
 import Distribution.Verbosity (silent)
 
 #if MIN_VERSION_Cabal(3,6,0)
 import Distribution.Utils.Path (SourceDir, PackageDir, SymbolicPath)
 #endif
 
--- cabal-install-parsers
-import Distribution.PackageDescription.Parsec (readGenericPackageDescription)
 
 -- | Efficient implementation of set like deletion on lists
 --
@@ -60,6 +68,15 @@
   }
   deriving (Show)
 
+-- | Merge multiple libraries into one, by concatenating all their fields.
+mergeLibraries :: [Library] -> Library
+mergeLibraries libs = Library
+  { libSourceDirectories = concatMap libSourceDirectories libs
+  , libCSourceDirectories = concatMap libCSourceDirectories libs
+  , libModules = concatMap libModules libs
+  , libDefaultExtensions = concatMap libDefaultExtensions libs
+  }
+
 -- | Convert a "Library" to arguments suitable to be passed to GHCi.
 libraryToGhciArgs :: Library -> ([String], [String], [String])
 libraryToGhciArgs Library{..} = (hsSrcArgs <> cSrcArgs, modArgs, extArgs)
@@ -132,6 +149,43 @@
 compatPrettyShow = id
 #endif
 
+-- | Traverse the given tree, solve predicates in branches, and return its
+-- contents.
+--
+-- XXX: Branches guarded by Cabal flags are ignored. I'm not sure where we should
+--      get this info from.
+--
+solveCondTree :: CondTree ConfVar c a -> [(c, a)]
+solveCondTree CondNode{condTreeData, condTreeConstraints, condTreeComponents} =
+  (condTreeConstraints, condTreeData) : concatMap goBranch condTreeComponents
+ where
+  goBranch :: CondBranch ConfVar c a -> [(c, a)]
+  goBranch (CondBranch condBranchCondition condBranchIfTrue condBranchIfFalse) =
+    if   goCondition condBranchCondition
+    then solveCondTree condBranchIfTrue
+    else maybe mempty solveCondTree condBranchIfFalse
+
+  goCondition :: Condition ConfVar -> Bool
+  goCondition = \case
+    Var cv ->
+      case cv of
+        OS os -> os == buildOS
+        Arch ar -> ar == buildArch
+        Impl cf versionRange ->
+          case cf of
+            GHC -> withinRange buildGhc versionRange
+            _   -> error ("Unrecognized compiler: " <> show cf)
+        -- XXX: We currently ignore any flags passed to Cabal
+        PackageFlag _fn -> False
+    Lit b -> b
+    CNot con -> not (goCondition con)
+    COr con0 con1 -> goCondition con0 || goCondition con1
+    CAnd con0 con1 -> goCondition con0 && goCondition con1
+
+-- | GHC version as Cabal's 'Version' data structure
+buildGhc :: Version
+buildGhc = mkVersion' compilerVersion
+
 -- Given a filepath to a @package.cabal@, parse it, and yield a "Library". Yields
 -- the default Library if first argument is Nothing, otherwise it will look for
 -- a specific sublibrary.
@@ -145,10 +199,10 @@
           let pkgDescription = package (packageDescription pkg) in
           error ("Could not find main library in: " <> show pkgDescription)
         Just lib ->
-          go lib
+          pure (go lib)
 
     Just libName ->
-      go (findSubLib pkg libName (condSubLibraries pkg))
+      pure (go (findSubLib pkg libName (condSubLibraries pkg)))
 
  where
   findSubLib pkg targetLibName [] =
@@ -158,19 +212,22 @@
     | unUnqualComponentName libName == targetLibName = lib
     | otherwise = findSubLib pkg targetLibName libs
 
-  go CondNode{condTreeData=lib} =
-    let
-      buildInfo = libBuildInfo lib
-      sourceDirs = hsSourceDirs buildInfo
-      cSourceDirs = includeDirs buildInfo
-      root = takeDirectory pkgPath
-    in
-      pure Library
-        { libSourceDirectories = map ((root </>) . compatPrettyShow) sourceDirs
-        , libCSourceDirectories = map (root </>) cSourceDirs
-        , libModules = exposedModules lib `rmList` autogenModules buildInfo
-        , libDefaultExtensions = defaultExtensions buildInfo
-        }
+  go condNode = mergeLibraries libs1
+   where
+    libs0 = map snd (solveCondTree condNode)
+    libs1 = map goLib libs0
+
+  goLib lib = Library
+    { libSourceDirectories = map ((root </>) . compatPrettyShow) sourceDirs
+    , libCSourceDirectories = map (root </>) cSourceDirs
+    , libModules = exposedModules lib `rmList` autogenModules buildInfo
+    , libDefaultExtensions = defaultExtensions buildInfo
+    }
+   where
+    buildInfo = libBuildInfo lib
+    sourceDirs = hsSourceDirs buildInfo
+    cSourceDirs = includeDirs buildInfo
+    root = takeDirectory pkgPath
 
 
 -- Given a filepath to a @package.cabal@, parse it, and yield a "Library". Returns
diff --git a/src/Test/DocTest/Internal/Runner.hs b/src/Test/DocTest/Internal/Runner.hs
--- a/src/Test/DocTest/Internal/Runner.hs
+++ b/src/Test/DocTest/Internal/Runner.hs
@@ -379,8 +379,8 @@
 
 updateSummary :: FromSetup -> Summary -> Report ()
 updateSummary FromSetup summary =
-  -- Suppress counts, except for errors
-  updateSummary NotFromSetup summary{sExamples=0, sTried=0, sFailures=0}
+  -- Suppress counts, except for errors and unexpected outputs
+  updateSummary NotFromSetup summary{sExamples=0, sTried=0}
 updateSummary NotFromSetup summary = do
   ReportState n f v q s <- get
   put (ReportState n f v q $ s `mappend` summary)
diff --git a/test/MainSpec.hs b/test/MainSpec.hs
--- a/test/MainSpec.hs
+++ b/test/MainSpec.hs
@@ -111,7 +111,7 @@
     it "skips subsequent tests from a module, if $setup fails" $ do
       doctest ["SetupSkipOnFailure.Foo"]
         -- TODO: Introduce "skipped"
-        (cases 2) {sTried = 0, sFailures = 0}
+        (cases 2) {sTried = 0, sFailures = 1}
 
     it "works with additional object files" $ do
       doctest ["WithCbits.Bar"]