diff --git a/CHANGES.md b/CHANGES.md --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,13 @@ Unreleased ========== +0.1.2 +======= + +* [#44](https://github.com/serokell/xrefcheck/pull/44) + + Decide whether to show progress bar by default depending on `CI` env variable. + + Added `--progress` option. + 0.1.1.2 ======= diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -45,22 +45,16 @@ * [url-checker](https://github.com/paramt/url-checker) - GitHub action which checks links in specified files. * [broken-link-checker](https://github.com/stevenvachon/broken-link-checker) - advanced checker for `HTML` files. -## Build instructions [↑](#xrefcheck) -Run `stack install` to build everything and install the executable. - -### CI and nix [↑](#xrefcheck) +## Usage [↑](#xrefcheck) -To build only the executables, run `nix-build`. You can use this line on your CI to use xrefcheck: -``` -nix run -f https://github.com/serokell/xrefcheck/archive/master.tar.gz -c xrefcheck -``` +We provide the following ways for you to use xrefcheck: -Our CI uses `nix-build xrefcheck.nix` to build the whole project, including tests and Haddock. -It is based on the [`haskell.nix`](https://input-output-hk.github.io/haskell.nix/) project. -You can do that too if you wish. +- [statically linked binaries](https://github.com/serokell/xrefcheck/releases) +- [Docker image](https://hub.docker.com/r/serokell/xrefcheck) +- [building from source](#build-instructions-) -## Usage [↑](#xrefcheck) +If none of those are suitable for you, please open an issue! To find all broken links in a repository, run from within its folder: @@ -91,6 +85,21 @@ Currently supported options include: * Timeout for checking external references; * List of ignored folders. + +## Build instructions [↑](#xrefcheck) + +Run `stack install` to build everything and install the executable. + +### CI and nix [↑](#xrefcheck) + +To build only the executables, run `nix-build`. You can use this line on your CI to use xrefcheck: +``` +nix run -f https://github.com/serokell/xrefcheck/archive/master.tar.gz -c xrefcheck +``` + +Our CI uses `nix-build xrefcheck.nix` to build the whole project, including tests and Haddock. +It is based on the [`haskell.nix`](https://input-output-hk.github.io/haskell.nix/) project. +You can do that too if you wish. ## For further work [↑](#xrefcheck) diff --git a/exec/Main.hs b/exec/Main.hs --- a/exec/Main.hs +++ b/exec/Main.hs @@ -16,6 +16,7 @@ import Xrefcheck.Progress import Xrefcheck.Scan import Xrefcheck.Scanners +import Xrefcheck.System import Xrefcheck.Verify formats :: FormatsSupport @@ -39,14 +40,17 @@ Just configPath -> do readConfig configPath - repoInfo <- allowRewrite oShowProgressBar $ \rw -> do + withinCI <- askWithinCI + let showProgressBar = oShowProgressBar ?: not withinCI + + repoInfo <- allowRewrite showProgressBar $ \rw -> do let fullConfig = addTraversalOptions (cTraversal config) oTraversalOptions gatherRepoInfo rw formats fullConfig root when oVerbose $ fmtLn $ "=== Repository data ===\n\n" <> indentF 2 (build repoInfo) - verifyRes <- allowRewrite oShowProgressBar $ \rw -> + verifyRes <- allowRewrite showProgressBar $ \rw -> verifyRepo rw (cVerification config) oMode root repoInfo case verifyErrors verifyRes of Nothing -> diff --git a/src/Xrefcheck/CLI.hs b/src/Xrefcheck/CLI.hs --- a/src/Xrefcheck/CLI.hs +++ b/src/Xrefcheck/CLI.hs @@ -18,7 +18,7 @@ ) where import Data.Version (showVersion) -import Options.Applicative (Parser, ReadM, command, eitherReader, execParser, fullDesc, help, +import Options.Applicative (Parser, ReadM, command, eitherReader, execParser, flag', fullDesc, help, helper, hsubparser, info, infoOption, long, metavar, option, progDesc, short, strOption, switch, value) import Paths_xrefcheck (version) @@ -50,7 +50,7 @@ , oRoot :: FilePath , oMode :: VerifyMode , oVerbose :: Bool - , oShowProgressBar :: Bool + , oShowProgressBar :: Maybe Bool , oTraversalOptions :: TraversalOptions } @@ -99,9 +99,16 @@ short 'v' <> long "verbose" <> help "Report repository scan and verification details." - oShowProgressBar <- fmap not . switch $ - long "no-progress" <> - help "Do not display progress bar during verification." + oShowProgressBar <- asum + [ flag' (Just True) $ + long "progress" <> + help "Display progress bar during verification. \ + \This is enabled by default unless `CI` env var is set to true." + , flag' (Just False) $ + long "no-progress" <> + help "Do not display progress bar during verification." + , pure Nothing + ] oTraversalOptions <- traversalOptionsParser return Options{..} diff --git a/src/Xrefcheck/Core.hs b/src/Xrefcheck/Core.hs --- a/src/Xrefcheck/Core.hs +++ b/src/Xrefcheck/Core.hs @@ -1,4 +1,4 @@ -{- SPDX-FileCopyrightText: 2018-2019 Serokell +{- SPDX-FileCopyrightText: 2018-2020 Serokell - - SPDX-License-Identifier: MPL-2.0 -} @@ -9,7 +9,6 @@ module Xrefcheck.Core where -import Control.DeepSeq (NFData) import Control.Lens (makeLenses, (%=)) import Data.Char (isAlphaNum) import qualified Data.Char as C diff --git a/src/Xrefcheck/System.hs b/src/Xrefcheck/System.hs --- a/src/Xrefcheck/System.hs +++ b/src/Xrefcheck/System.hs @@ -5,13 +5,16 @@ module Xrefcheck.System ( readingSystem + , askWithinCI , RelGlobPattern (..) , bindGlobPattern ) where import Data.Aeson (FromJSON (..), withText) +import qualified Data.Char as C import GHC.IO.Unsafe (unsafePerformIO) import System.Directory (canonicalizePath) +import System.Environment (lookupEnv) import System.FilePath (()) import qualified System.FilePath.Glob as Glob @@ -19,6 +22,14 @@ -- so IO reading operations can be turned into pure values. readingSystem :: IO a -> a readingSystem = unsafePerformIO + +-- | Heuristics to check whether we are running within CI. +-- Check the respective env variable which is usually set in all CIs. +askWithinCI :: IO Bool +askWithinCI = lookupEnv "CI" <&> \case + Just "1" -> True + Just (map C.toLower -> "true") -> True + _ -> False -- | Glob pattern relative to repository root. newtype RelGlobPattern = RelGlobPattern FilePath diff --git a/src/Xrefcheck/Verify.hs b/src/Xrefcheck/Verify.hs --- a/src/Xrefcheck/Verify.hs +++ b/src/Xrefcheck/Verify.hs @@ -23,7 +23,7 @@ ) where import Control.Concurrent.Async (forConcurrently, withAsync) -import Control.Monad.Except (ExceptT, MonadError (..)) +import Control.Monad.Except (MonadError (..)) import qualified Data.Map as M import qualified Data.Text as T import Data.Text.Metrics (damerauLevenshteinNorm) @@ -31,12 +31,13 @@ import qualified GHC.Exts as Exts import Network.HTTP.Client (HttpException (..), HttpExceptionContent (..), responseStatus) import Network.HTTP.Req (GET (..), HEAD (..), HttpException (..), NoReqBody (..), defaultHttpConfig, - ignoreResponse, parseUrl, req, runReq) + ignoreResponse, req, runReq, useURI) import Network.HTTP.Types.Status (Status, statusCode, statusMessage) import System.Console.Pretty (Style (..), style) import System.Directory (canonicalizePath, doesDirectoryExist, doesFileExist) import System.FilePath (takeDirectory, ()) import qualified System.FilePath.Glob as Glob +import Text.URI (mkURI) import Time (RatioNat, Second, Time (..), ms, threadDelay, timeout) import Xrefcheck.Config @@ -94,6 +95,7 @@ | AnchorDoesNotExist Text [Anchor] | AmbiguousAnchorRef FilePath Text (NonEmpty Anchor) | ExternalResourceInvalidUri + | ExternalResourceUnknownProtocol | ExternalResourceUnavailable Status | ExternalResourceSomeError Text deriving (Show) @@ -113,6 +115,8 @@ " Use of such anchors is discouraged because referenced object\n\ \ can change silently whereas the document containing it evolves.\n" ExternalResourceInvalidUri -> + "⛂ Invalid url\n" + ExternalResourceUnknownProtocol -> "⛂ Bad url (expected 'http' or 'https')\n" ExternalResourceUnavailable status -> "⛂ Resource unavailable (" +| statusCode status |+ " " +| @@ -257,8 +261,10 @@ makeRequest :: _ => method -> RatioNat -> IO (Either VerifyError ()) makeRequest method timeoutFrac = runExceptT $ do - parsedUrl <- parseUrl (encodeUtf8 link) - & maybe (throwError ExternalResourceInvalidUri) pure + uri <- mkURI link + & maybe (throwError ExternalResourceInvalidUri) pure + parsedUrl <- useURI uri + & maybe (throwError ExternalResourceUnknownProtocol) pure let reqLink = case parsedUrl of Left (url, option) -> runReq defaultHttpConfig $ diff --git a/xrefcheck.cabal b/xrefcheck.cabal --- a/xrefcheck.cabal +++ b/xrefcheck.cabal @@ -1,201 +1,195 @@ cabal-version: 2.0 -name: xrefcheck -version: 0.1.1.2 -license: MPL-2.0 -license-file: LICENSE -copyright: 2018-2019 Serokell -maintainer: Serokell -author: Kostya Ivanov, Serokell -homepage: https://github.com/serokell/xrefcheck#readme -bug-reports: https://github.com/serokell/xrefcheck/issues -description: - Please see the README on GitHub at -build-type: Simple + +-- This file has been generated from package.yaml by hpack version 0.33.0. +-- +-- see: https://github.com/sol/hpack +-- +-- hash: ab6af365f22152ef223e8a0ab2213ac72da2aa58d5211dc9d389c66e8515674f + +name: xrefcheck +version: 0.1.2 +description: Please see the README on GitHub at +homepage: https://github.com/serokell/xrefcheck#readme +bug-reports: https://github.com/serokell/xrefcheck/issues +author: Kostya Ivanov, Serokell +maintainer: Serokell +copyright: 2018-2019 Serokell +license: MPL-2.0 +license-file: LICENSE +build-type: Simple extra-source-files: README.md CHANGES.md src-files/def-config.yaml source-repository head - type: git - location: https://github.com/serokell/xrefcheck + type: git + location: https://github.com/serokell/xrefcheck library - exposed-modules: - Xrefcheck.CLI - Xrefcheck.Config - Xrefcheck.Core - Xrefcheck.Progress - Xrefcheck.Scan - Xrefcheck.Scanners - Xrefcheck.Scanners.Markdown - Xrefcheck.System - Xrefcheck.Util - Xrefcheck.Verify - hs-source-dirs: src - other-modules: - Paths_xrefcheck - autogen-modules: - Paths_xrefcheck - default-language: Haskell2010 - default-extensions: AllowAmbiguousTypes BangPatterns - ConstraintKinds DataKinds DefaultSignatures DeriveDataTypeable - DeriveGeneric FlexibleContexts FlexibleInstances - FunctionalDependencies GeneralizedNewtypeDeriving LambdaCase - MonadFailDesugaring MultiParamTypeClasses MultiWayIf NamedFieldPuns - OverloadedStrings RankNTypes RecordWildCards ScopedTypeVariables - StandaloneDeriving TemplateHaskell TupleSections TypeFamilies - UndecidableInstances ViewPatterns TypeApplications TypeOperators - ghc-options: -Wall -Wincomplete-record-updates - build-depends: - Glob >=0.10.0 && <0.11, - aeson >=1.4.6.0 && <1.5, - aeson-options >=0.1.0 && <0.2, - async >=2.2.2 && <2.3, - base >=4.12.0.0 && <4.13, - bytestring >=0.10.8.2 && <0.11, - cmark-gfm >=0.2.1 && <0.3, - containers >=0.6.0.1 && <0.7, - data-default >=0.7.1.1 && <0.8, - deepseq >=1.4.4.0 && <1.5, - directory >=1.3.3.0 && <1.4, - directory-tree >=0.12.1 && <0.13, - filepath >=1.4.2.1 && <1.5, - fmt >=0.6.1.2 && <0.7, - http-client >=0.6.4 && <0.7, - http-types >=0.12.3 && <0.13, - lens >=4.17.1 && <4.18, - mtl >=2.2.2 && <2.3, - o-clock >=1.1.0 && <1.2, - optparse-applicative >=0.14.3.0 && <0.15, - pretty-terminal >=0.1.0.0 && <0.2, - req >=2.1.0 && <2.2, - roman-numerals >=0.5.1.5 && <0.6, - template-haskell >=2.14.0.0 && <2.15, - text >=1.2.3.1 && <1.3, - text-metrics >=0.3.0 && <0.4, - th-lift-instances >=0.1.14 && <0.2, - th-utilities >=0.2.3.1 && <0.3, - universum >=1.5.0 && <1.6, - with-utf8 >=1.0.0.0 && <1.1, - yaml >=0.11.2.0 && <0.12 - mixins: base hiding (Prelude), - universum (Universum as Prelude), - universum (Universum.Unsafe as Unsafe) + exposed-modules: + Xrefcheck.CLI + Xrefcheck.Config + Xrefcheck.Core + Xrefcheck.Progress + Xrefcheck.Scan + Xrefcheck.Scanners + Xrefcheck.Scanners.Markdown + Xrefcheck.System + Xrefcheck.Util + Xrefcheck.Verify + other-modules: + Paths_xrefcheck + autogen-modules: + Paths_xrefcheck + hs-source-dirs: + src + default-extensions: AllowAmbiguousTypes BangPatterns ConstraintKinds DataKinds DefaultSignatures DeriveDataTypeable DeriveGeneric FlexibleContexts FlexibleInstances FunctionalDependencies GeneralizedNewtypeDeriving LambdaCase MultiParamTypeClasses MultiWayIf NamedFieldPuns OverloadedStrings RankNTypes RecordWildCards ScopedTypeVariables StandaloneDeriving TemplateHaskell TupleSections TypeFamilies UndecidableInstances ViewPatterns TypeApplications TypeOperators + ghc-options: -Wall -Wincomplete-record-updates + build-depends: + Glob + , aeson + , aeson-options + , async + , base <4.15 + , bytestring + , cmark-gfm + , containers + , data-default + , deepseq + , directory + , directory-tree + , filepath + , fmt + , http-client + , http-types + , lens + , modern-uri + , mtl + , o-clock + , optparse-applicative + , pretty-terminal + , req + , roman-numerals + , template-haskell + , text + , text-metrics + , th-lift-instances + , th-utilities + , universum + , with-utf8 + , yaml + mixins: + base hiding (Prelude) + , universum (Universum as Prelude) + , universum (Universum.Unsafe as Unsafe) + default-language: Haskell2010 executable xrefcheck - main-is: Main.hs - hs-source-dirs: exec - other-modules: - Paths_xrefcheck - autogen-modules: - Paths_xrefcheck - default-language: Haskell2010 - default-extensions: AllowAmbiguousTypes BangPatterns - ConstraintKinds DataKinds DefaultSignatures DeriveDataTypeable - DeriveGeneric FlexibleContexts FlexibleInstances - FunctionalDependencies GeneralizedNewtypeDeriving LambdaCase - MonadFailDesugaring MultiParamTypeClasses MultiWayIf NamedFieldPuns - OverloadedStrings RankNTypes RecordWildCards ScopedTypeVariables - StandaloneDeriving TemplateHaskell TupleSections TypeFamilies - UndecidableInstances ViewPatterns TypeApplications TypeOperators - ghc-options: -Wall -Wincomplete-record-updates -threaded -rtsopts - -with-rtsopts=-N -O2 - build-depends: - Glob >=0.10.0 && <0.11, - aeson >=1.4.6.0 && <1.5, - aeson-options >=0.1.0 && <0.2, - async >=2.2.2 && <2.3, - base >=4.12.0.0 && <4.13, - bytestring >=0.10.8.2 && <0.11, - cmark-gfm >=0.2.1 && <0.3, - containers >=0.6.0.1 && <0.7, - data-default >=0.7.1.1 && <0.8, - deepseq >=1.4.4.0 && <1.5, - directory >=1.3.3.0 && <1.4, - directory-tree >=0.12.1 && <0.13, - filepath >=1.4.2.1 && <1.5, - fmt >=0.6.1.2 && <0.7, - http-client >=0.6.4 && <0.7, - http-types >=0.12.3 && <0.13, - lens >=4.17.1 && <4.18, - mtl >=2.2.2 && <2.3, - o-clock >=1.1.0 && <1.2, - optparse-applicative >=0.14.3.0 && <0.15, - pretty-terminal >=0.1.0.0 && <0.2, - req >=2.1.0 && <2.2, - roman-numerals >=0.5.1.5 && <0.6, - template-haskell >=2.14.0.0 && <2.15, - text >=1.2.3.1 && <1.3, - text-metrics >=0.3.0 && <0.4, - th-lift-instances >=0.1.14 && <0.2, - th-utilities >=0.2.3.1 && <0.3, - universum >=1.5.0 && <1.6, - with-utf8 >=1.0.0.0 && <1.1, - xrefcheck -any, - yaml >=0.11.2.0 && <0.12 - mixins: base hiding (Prelude), - universum (Universum as Prelude), - universum (Universum.Unsafe as Unsafe) + main-is: Main.hs + other-modules: + Paths_xrefcheck + autogen-modules: + Paths_xrefcheck + hs-source-dirs: + exec + default-extensions: AllowAmbiguousTypes BangPatterns ConstraintKinds DataKinds DefaultSignatures DeriveDataTypeable DeriveGeneric FlexibleContexts FlexibleInstances FunctionalDependencies GeneralizedNewtypeDeriving LambdaCase MultiParamTypeClasses MultiWayIf NamedFieldPuns OverloadedStrings RankNTypes RecordWildCards ScopedTypeVariables StandaloneDeriving TemplateHaskell TupleSections TypeFamilies UndecidableInstances ViewPatterns TypeApplications TypeOperators + ghc-options: -Wall -Wincomplete-record-updates -threaded -rtsopts -with-rtsopts=-N -O2 + build-depends: + Glob + , aeson + , aeson-options + , async + , base <4.15 + , bytestring + , cmark-gfm + , containers + , data-default + , deepseq + , directory + , directory-tree + , filepath + , fmt + , http-client + , http-types + , lens + , modern-uri + , mtl + , o-clock + , optparse-applicative + , pretty-terminal + , req + , roman-numerals + , template-haskell + , text + , text-metrics + , th-lift-instances + , th-utilities + , universum + , with-utf8 + , xrefcheck + , yaml + mixins: + base hiding (Prelude) + , universum (Universum as Prelude) + , universum (Universum.Unsafe as Unsafe) + default-language: Haskell2010 test-suite xrefcheck-tests - type: exitcode-stdio-1.0 - main-is: Main.hs - build-tool-depends: hspec-discover:hspec-discover -any - hs-source-dirs: tests - other-modules: - Spec - Test.Xrefcheck.AnchorsSpec - Test.Xrefcheck.ConfigSpec - Test.Xrefcheck.LocalSpec - Paths_xrefcheck - autogen-modules: - Paths_xrefcheck - default-language: Haskell2010 - default-extensions: AllowAmbiguousTypes BangPatterns - ConstraintKinds DataKinds DefaultSignatures DeriveDataTypeable - DeriveGeneric FlexibleContexts FlexibleInstances - FunctionalDependencies GeneralizedNewtypeDeriving LambdaCase - MonadFailDesugaring MultiParamTypeClasses MultiWayIf NamedFieldPuns - OverloadedStrings RankNTypes RecordWildCards ScopedTypeVariables - StandaloneDeriving TemplateHaskell TupleSections TypeFamilies - UndecidableInstances ViewPatterns TypeApplications TypeOperators - ghc-options: -Wall -Wincomplete-record-updates - build-depends: - Glob >=0.10.0 && <0.11, - QuickCheck >=2.13.2 && <2.14, - aeson >=1.4.6.0 && <1.5, - aeson-options >=0.1.0 && <0.2, - async >=2.2.2 && <2.3, - base >=4.12.0.0 && <4.13, - bytestring >=0.10.8.2 && <0.11, - cmark-gfm >=0.2.1 && <0.3, - containers >=0.6.0.1 && <0.7, - data-default >=0.7.1.1 && <0.8, - deepseq >=1.4.4.0 && <1.5, - directory >=1.3.3.0 && <1.4, - directory-tree >=0.12.1 && <0.13, - filepath >=1.4.2.1 && <1.5, - fmt >=0.6.1.2 && <0.7, - hspec >=2.7.1 && <2.8, - http-client >=0.6.4 && <0.7, - http-types >=0.12.3 && <0.13, - lens >=4.17.1 && <4.18, - mtl >=2.2.2 && <2.3, - o-clock >=1.1.0 && <1.2, - optparse-applicative >=0.14.3.0 && <0.15, - pretty-terminal >=0.1.0.0 && <0.2, - req >=2.1.0 && <2.2, - roman-numerals >=0.5.1.5 && <0.6, - template-haskell >=2.14.0.0 && <2.15, - text >=1.2.3.1 && <1.3, - text-metrics >=0.3.0 && <0.4, - th-lift-instances >=0.1.14 && <0.2, - th-utilities >=0.2.3.1 && <0.3, - universum >=1.5.0 && <1.6, - with-utf8 >=1.0.0.0 && <1.1, - xrefcheck -any, - yaml >=0.11.2.0 && <0.12 - mixins: base hiding (Prelude), - universum (Universum as Prelude), - universum (Universum.Unsafe as Unsafe) + type: exitcode-stdio-1.0 + main-is: Main.hs + other-modules: + Spec + Test.Xrefcheck.AnchorsSpec + Test.Xrefcheck.ConfigSpec + Test.Xrefcheck.LocalSpec + Paths_xrefcheck + autogen-modules: + Paths_xrefcheck + hs-source-dirs: + tests + default-extensions: AllowAmbiguousTypes BangPatterns ConstraintKinds DataKinds DefaultSignatures DeriveDataTypeable DeriveGeneric FlexibleContexts FlexibleInstances FunctionalDependencies GeneralizedNewtypeDeriving LambdaCase MultiParamTypeClasses MultiWayIf NamedFieldPuns OverloadedStrings RankNTypes RecordWildCards ScopedTypeVariables StandaloneDeriving TemplateHaskell TupleSections TypeFamilies UndecidableInstances ViewPatterns TypeApplications TypeOperators + ghc-options: -Wall -Wincomplete-record-updates + build-tool-depends: + hspec-discover:hspec-discover + build-depends: + Glob + , QuickCheck + , aeson + , aeson-options + , async + , base <4.15 + , bytestring + , cmark-gfm + , containers + , data-default + , deepseq + , directory + , directory-tree + , filepath + , fmt + , hspec + , http-client + , http-types + , lens + , modern-uri + , mtl + , o-clock + , optparse-applicative + , pretty-terminal + , req + , roman-numerals + , template-haskell + , text + , text-metrics + , th-lift-instances + , th-utilities + , universum + , with-utf8 + , xrefcheck + , yaml + mixins: + base hiding (Prelude) + , universum (Universum as Prelude) + , universum (Universum.Unsafe as Unsafe) + default-language: Haskell2010