diff --git a/changes.txt b/changes.txt --- a/changes.txt +++ b/changes.txt @@ -1,3 +1,11 @@ +1.0.0.0 +------- + +- Released Fri 02 Sep 2016 16:03:29 CEST +- Fixed command-line documentation +- Fixed "crash on wrong encoding" error +- Added progress bar for big (>100 source files) projects + 0.2.0.1 ------- diff --git a/issues.txt b/issues.txt --- a/issues.txt +++ b/issues.txt @@ -8,36 +8,34 @@ 14 symlinks? [feature:intermediate] 15 check --output pretty, there are trailing white spaces [bug] [u:1] 16 test con 7.6.3 [feature:basic] [test] - 17 linux kernel e darcs, lentil si inceppa su dei caratteri [bug] - [2016] - 18 autoignore non-text files? (binary, images, bloom filter) [2016] - 19 da cabal a hpack [u:1] [feature:advanced] - 20 sm: issue numbers is valuable [design] [request] - 21 sm: have a way to link to discussion/history of issue [design] + 17 autoignore non-text files? (binary, images, bloom filter) [2016] + 18 da cabal a hpack [u:1] [feature:advanced] + 19 sm: issue numbers is valuable [design] [request] + 20 sm: have a way to link to discussion/history of issue [design] [request] - 22 sm: 1. lentil autogenerates numbers where missing adding tags, 2. + 21 sm: 1. lentil autogenerates numbers where missing adding tags, 2. creates an associated numbered file in .issues/, displays associated notes written in those files is asked, 3. even update them when there is a source change. (4) these files could be webserved when they want something linkable. Vedi comments in doc/dev/issues.txt [design] [request] - 28 chickpea, lentil's brother. Two commands, add number to numberless + 27 chickpea, lentil's brother. Two commands, add number to numberless tag (#1212 or something), create a mini db with those issues, which automatically updated and you can even write stuff in there. O maybe you can even add lentil, chickpea and quinoa. Vedi more in doc/dev/issues.txt [design] - 34 make it lazy in ouputting todos [refactor] - 35 way to 'group' (section) output by tag [feature:advanced] - 36 win (and mac?) builds [package] [u:3] [2016] - 37 Henning: don't output nothing on lentil xxx.cabal (maybe add 19 + 33 make it lazy in ouputting todos [refactor] + 34 way to 'group' (section) output by tag [feature:advanced] + 35 win (and mac?) builds [package] [u:3] [2016] + 36 Henning: don't output nothing on lentil xxx.cabal (maybe add 19 skipped?) [request] [2016] - 38 code coverage [test] - 39 add .lentilignore (or even .lentilconf) Michał Antkiewicz + 37 code coverage [test] + 38 add .lentilignore (or even .lentilconf) Michał Antkiewicz [feature:intermediate] [request] - 41 + 40 https://blog.steve.org.uk/If_line_noise_is_a_program__all_fuzzers_are_developers.html in lentil? [debug] [u:2] - 42 parsing with http://awgn.github.io/cgrep/ [design] + 41 parsing with http://awgn.github.io/cgrep/ [design] src/Lentil/Args.hs 26 disambiguation optparse-applicative [feature:intermediate] @@ -51,7 +49,9 @@ 41 not using canonised paths because it is extremely slow on big repositories (like darcs). Explore different possibilities (conduit, new Filepath, unix program). [u:2] [duct] - 61 combine funziona su windows? [feature:intermediate] + 52 apparently toListM is not idiomatic [refactor] [duct] + 66 use pipes-files when released [feature:intermediate] + 77 combine funziona su windows? [feature:intermediate] src/Lentil/Parse/Issue.hs 30 "feature" flagword could (in theory) clash with a free-text comment @@ -60,16 +60,27 @@ .lentil dotfile [design] [u:2] [bug] src/Lentil/Parse/Run.hs - 27 hGetContents: invalid argument (invalid byte sequence) [fixme] - [u:3] [2016] - 33 a function String -> [Issue] - 70 change '\n' trick [refactor] [duct] + 58 a function String -> [Issue] + 80 personally i don't like when the leading character of the progress + bar is different [feeback] [request] + 97 sicuramente c'è un modo più elegante e breve [refactor] [duct] src/Lentil/Parse/Syntaxes.hs - 31 add langparsers che sia estensibile e leggibile a compilazione + 33 add langparsers che sia estensibile e leggibile a compilazione [u:2] [feature:intermediate] - 56 multiline signature? [lint] + 60 multiline signature? [lint] src/Main.hs 41 leave sort out as now? [design] 58 o ord? [fixme] + +test/Lentil/ArgsSpec.hs + 37 real parsing testing options [test] + +test/Lentil/FileSpec.hs + 38 excluding files fails on MS Windows! Check why! [u:3] [bug] + +test/Tests.hs + 4 + http://hackage.haskell.org/package/mockery-0.3.0/docs/Test-Mockery-Directory.html + instead of folder use this? [refactor] diff --git a/lentil.cabal b/lentil.cabal --- a/lentil.cabal +++ b/lentil.cabal @@ -1,5 +1,5 @@ name: lentil -version: 0.2.0.1 +version: 1.0.0.0 synopsis: frugal issue tracker description: minumum effort, cohesive issue tracker based on ubiquitous @TODO@s and @FIXME@s conventions. @@ -48,7 +48,8 @@ build-depends: base==4.*, optparse-applicative==0.13.*, regex-tdfa==1.2.*, natural-sort==0.1.*, parsec==3.1.*, filepath==1.4.*, directory==1.2.*, filemanip==0.3.*, - ansi-wl-pprint==0.6.*, csv==0.1.* + ansi-wl-pprint==0.6.*, csv==0.1.*, pipes==4.2.*, + terminal-progress-bar==0.0.*, text==1.2.* other-modules: Lentil.Types, Lentil.Args, Lentil.File, Lentil.Print, Lentil.Query, Lentil.Export, Lentil.Parse.Source, Lentil.Parse.Issue Lentil.Parse.Syntaxes, @@ -64,7 +65,8 @@ build-depends: base==4.*, optparse-applicative==0.13.*, regex-tdfa==1.2.*, natural-sort==0.1.*, parsec==3.1.*, filepath==1.4.*, directory==1.2.*, filemanip==0.3.*, - ansi-wl-pprint==0.6.*, csv==0.1.* + ansi-wl-pprint==0.6.*, csv==0.1.*, pipes==4.2.*, + terminal-progress-bar==0.0.*, text==1.2.* -- same as above, plus hspec , hspec==2.2.* other-modules: Lentil.Types, Lentil.Args, Lentil.File, Lentil.Print, diff --git a/src/Lentil/Args.hs b/src/Lentil/Args.hs --- a/src/Lentil/Args.hs +++ b/src/Lentil/Args.hs @@ -80,8 +80,8 @@ alias = option (str >>= parseAlias) ( short 'a' <> metavar "ALIAS" <> - help ("extension alias (e.g.: -a cpp" ++ aliasSign ++ - "d)") ) + help ("extension alias (e.g.: -a d" ++ aliasSign ++ + "cpp)") ) where parseAlias :: String -> ReadM (Maybe Alias) parseAlias s = return (aliasp s) diff --git a/src/Lentil/File.hs b/src/Lentil/File.hs --- a/src/Lentil/File.hs +++ b/src/Lentil/File.hs @@ -10,20 +10,19 @@ module Lentil.File where +import Lentil.Helpers import Lentil.Types import Lentil.Parse.Run +import Lentil.Parse.Syntaxes import System.FilePath import System.FilePath.Find import Data.Monoid import Control.Applicative +import Pipes import qualified Data.List as L - --- import Data.Either --- import System.Directory --- import Control.Exception.Base - +import qualified Pipes.Prelude as PP --------------- -- INSTANCES -- @@ -43,22 +42,42 @@ -- (conduit, new Filepath, unix program). [u:2] [duct] findIssues :: [Alias] -> [FilePath] -> [FilePath] -> IO [Issue] -findIssues as is xs = mapM fc is >>= (issueFinder as) . concat +findIssues as fps xs = + (fmap concat $ PP.toListM + (prodFilePaths fps xs as >-> issLoop)) >>= \r -> + perrEph "\r" >> -- clean progress bar + return r + where issLoop = issueFinder as >> issLoop + -- todo: [refactor] [duct] apparently toListM is not idiomatic + +-- Filepaths to be parsed, as a Producer +prodFilePaths :: [FilePath] -> [FilePath] -> + [Alias] -> Producer FilePathC IO () +prodFilePaths fps xs as = liftIO (fmap concat (mapM fc fps)) >>= \is -> + let is' = zip3 is [1..] (repeat $ length is) in + each is' where - fc i = find recPred (findClause (xs' i)) i + fc i = find recPred (findClause (xs' i) as) i -- search pattern xs' "." = map (combine ".") xs -- trick to exclude on '.' xs' _ = xs --- fp to exclude, clause -findClause :: [FilePath] -> FindClause Bool -findClause x = let xc = mconcat $ map fp2fc x - in fileType ==? RegularFile &&? - (not <$> fmap getAny xc) + -- TODO: use pipes-files when released [feature:intermediate] + +-- fp to exclude, clause (+ aliases, to include) +findClause :: [FilePath] -> [Alias] -> FindClause Bool +findClause xs as = let xc = mconcat $ map fp2fc xs + in fileType ==? RegularFile &&? + extCheck &&? + (not <$> fmap getAny xc) where fp2fc :: FilePath -> FindClause Any fp2fc f = Any . L.isPrefixOf f <$> filePath -- TODO: combine funziona su windows? [feature:intermediate] + + extCheck = (fmap getAny . mconcat) + (map (fmap Any . (extension ==?)) (extensionList as)) + -- recursion predicate: excludes dot ('.') or _ folders diff --git a/src/Lentil/Helpers.hs b/src/Lentil/Helpers.hs --- a/src/Lentil/Helpers.hs +++ b/src/Lentil/Helpers.hs @@ -18,9 +18,14 @@ import Control.Applicative hiding ( (<|>), optional, many ) import Prelude -- 7.8 hack --- output errors +-- output errors (to stderr) perr :: String -> IO () perr cs = I.hPutStrLn I.stderr cs + +-- output errors (to stderr), but without newline *and* calling /r +-- (erase what's before) first +perrEph :: String -> IO () +perrEph cs = I.hPutStr I.stderr ("\r" ++ cs) ------------- diff --git a/src/Lentil/Parse/Run.hs b/src/Lentil/Parse/Run.hs --- a/src/Lentil/Parse/Run.hs +++ b/src/Lentil/Parse/Run.hs @@ -7,6 +7,7 @@ -- Parsing functions interface ----------------------------------------------------------------------------- +{-# LANGUAGE FlexibleContexts #-} module Lentil.Parse.Run where @@ -17,63 +18,119 @@ import Lentil.Parse.Syntaxes import Text.Parsec +import Pipes import Control.Applicative hiding (many, (<|>)) + import qualified System.Directory as D +import qualified Data.Functor.Identity as I +import qualified System.ProgressBar as PB +import qualified Control.Monad as CM +import qualified Data.Text as T +import qualified Data.Text.IO as TI +import qualified Control.Exception as E import Prelude -- 7.8 hack +----------- +-- TYPES -- +----------- --- fixme: hGetContents: invalid argument (invalid byte sequence) [u:3] [2016] +type Cur = Int +type Tot = Int +type FilePathC = (FilePath, Cur, Tot) --- main function with wich to parse issues; errors on stderr -issueFinder :: [Alias] -> [FilePath] -> IO [Issue] -issueFinder as fps = fmap concat (mapM (fileParser as) fps) + + +issueFinder :: [Alias] -> Pipe FilePathC [Issue] IO () +issueFinder as = fpExist >-> + fp2par as >-> + fp2comm >-> + cms2iss + where + -- comm2Issues accepts Comment, we need [Comment] + cms2iss = await >>= \(fp, cs) -> + let tra = zip (repeat fp) cs in + each tra >-> + comm2Issues + -- todo a function String -> [Issue] (w/o IO) [debug] [refactor] ------------------ --- ANCILLARIES -- ------------------ +----------- +-- PIPES -- +----------- --- errors to stderr -fileParser :: [Alias] -> FilePath -> IO [Issue] -fileParser as fp = D.doesFileExist fp >>= \fb -> +-- file exist check +fpExist :: Pipe FilePathC FilePath IO () +fpExist = await >>= \(fp, k, t) -> + pbe k t >> + liftIO (D.doesFileExist fp) >>= \fb -> + if fb == False + then liftIO (perr $ fp ++ " : no such file") + else yield fp + where + fi i = fromIntegral i + lbl t = show t ++ " source files" + pb k t = PB.mkProgressBar (PB.msg $ lbl t) + PB.percentage 40 (fi k) (fi t) + pbe k t = CM.when (mod k 30 == 0 && t > 100) + (liftIO (perrEph $ pb k t)) - -- file exists - if fb == False - then perr (fp ++ " : no such file") >> return [] - else + -- todo: personally i don't like when the leading + -- character of the progress bar is different [feeback] + -- [request] - -- parser exists - case langParserAlias as fp of - Nothing -> return [] -- no error just empy list - Just p -> readFile fp >>= - runParIO p fp >>= - parIssues fp . comms2Tuple +-- pick appropriate parser (if exists) +fp2par :: [Alias] -> Pipe FilePath + (FilePath, ParSource [Comment]) IO () +fp2par as = await >>= \fp -> + case langParserAlias as fp of + Nothing -> return () + Just p -> yield (fp, fmap comms2Tuple p) +-- Parse raw comments +fp2comm :: Pipe (FilePath, ParSource [Comment]) (FilePath, [Comment]) IO () +fp2comm = await >>= \(fp, p) -> + liftIO (safeRead fp) >>= \t -> + (runParPipe p fp t >-> -- todo: sicuramente c'è un modo più elegante + (await >>= \r -> -- e breve. Ah! quando mettono le + -- tuples section! [refactor] [duct] + yield (fp, r))) --- generic parsing -- +comm2Issues :: Pipe (FilePath, Comment) [Issue] IO () +comm2Issues = await >>= \(fp, (r, i)) -> + runParPipe (setRow r >> issues) + fp ('\n':i) >> + comm2Issues -- needed or will pick just head issue -runParIO :: Parsec String () [a] -> FilePath -> String -> IO [a] -runParIO p fp t = case runParser p () fp t of - Left l -> perr (fp ++ " : parse error " ++ - show l) >> return [] - Right r -> return r --- issue parsing -- +----------------- +-- ANCILLARIES -- +----------------- -parIssues :: FilePath -> [Comment] -> IO [Issue] -parIssues fp cs = concat <$> mapM (parIssue fp) cs +-- generic parsing -- --- todo change '\n' trick [refactor] [duct] --- why '\n'? see setRow comment -parIssue :: FilePath -> Comment -> IO [Issue] -parIssue fp (r, t) = runParIO (setRow r >> issues) fp ('\n':t) +runParPipe :: (Stream i I.Identity t) => Parsec i () o -> + FilePath -> i -> + Pipe ip o IO () +runParPipe p fp i = case runParser p () fp i of + Left l -> liftIO (perr $ fp ++ " : parse error " ++ + show l) + Right r -> yield r +-- issue parsing -- + -- why (r-1)? Every TODO must start on a newline, so we have to add a top -- '\n' in case there immediately is one setRow :: Row -> ParIssue () setRow r = getPosition >>= setPosition . flip setSourceLine (r-1) +safeRead :: FilePath -> IO String +safeRead fp = E.try (TI.readFile fp) >>= \e -> + case e of + Right t -> return (T.unpack t) + Left x -> perr (fp ++ " : " ++ show (x :: E.IOException) ++ + " -- ignoring file") >> + return "" diff --git a/src/Lentil/Parse/Syntaxes.hs b/src/Lentil/Parse/Syntaxes.hs --- a/src/Lentil/Parse/Syntaxes.hs +++ b/src/Lentil/Parse/Syntaxes.hs @@ -18,41 +18,45 @@ import Prelude import qualified System.FilePath as SF - -import Data.Char (toLower) +import qualified Data.Char as C +type MaybePar = Maybe (ParSource [CommentString]) -- as langParser, with alias added -langParserAlias :: [Alias] -> String -> Maybe (ParSource [CommentString]) +langParserAlias :: [Alias] -> String -> MaybePar langParserAlias as fp = maybe (langParser fp) langParser (lookup ext as) - where ext = map toLower (SF.takeExtension fp) + where ext = map C.toLower (SF.takeExtension fp) +extensionList :: [Alias] -> [String] +extensionList as = concatMap fst languages ++ map fst as -- TODO: add langparsers che sia estensibile e leggibile -- a compilazione [u:2] [feature:intermediate] -langParser :: String -> Maybe (ParSource [CommentString]) -langParser fp | ext `elem` [".hs", ".lhs", - ".hsc", ".chs" ] = Just haskell - | ext `elem` [".cabal"] = Just haskell -- cabal - | ext `elem` [".elm"] = Just haskell -- Elm - | ext `elem` [".c", ".h"] = Just c - | ext `elem` [".cpp", ".hpp"] = Just c -- C++ - | ext `elem` [".java"] = Just c -- Java - | ext `elem` [".js"] = Just javascript - | ext `elem` [".ts"] = Just javascript -- TypeScript - | ext `elem` [".pas", ".pp", ".inc"] = Just pascal - | ext `elem` [".py"] = Just python - | ext `elem` [".coffee"] = Just python -- CoffeeScript - | ext `elem` [".rb"] = Just ruby - | ext `elem` [".pl", ".pm", ".t"] = Just perl - | ext `elem` [".sh"] = Just perl -- shell - | ext `elem` [".nix"] = Just nix - | ext `elem` [".xml", ".html"] = Just xml - | ext `elem` [".txt"] = Just text - | otherwise = Nothing - where ext = map toLower (SF.takeExtension fp) +langParser :: String -> MaybePar +langParser fp = lookupExt languages ext + where ext = map C.toLower (SF.takeExtension fp) +languages :: [([String], MaybePar)] +languages = [ ([".hs", ".lhs", + ".hsc", ".chs" ], Just haskell), + ([".cabal"], Just haskell), -- cabal + ([".elm"], Just haskell), -- Elm + ([".c", ".h"], Just c), + ([".cpp", ".hpp"], Just c), -- C++ + ([".java"], Just c), -- Java + ([".js"], Just javascript), + ([".ts"], Just javascript), -- TypeScript + ([".pas", ".pp", ".inc"], Just pascal), + ([".py"], Just python), + ([".coffee"], Just python), -- CoffeeScript + ([".rb"], Just ruby), + ([".pl", ".pm", ".t"], Just perl), + ([".sh"], Just perl), -- shell + ([".nix"], Just nix), + ([".xml", ".html"], Just xml), + ([".txt"], Just text) ] + -- todo multiline signature? [lint] haskell, c, javascript, pascal, python, ruby :: ParSource [CommentString] perl, nix :: ParSource [CommentString] @@ -77,4 +81,11 @@ xml = source $ ParSyntax [] [("")] ClangLike ['"', '\''] [] text = (:[]) . MultiLine 1 <$> many anyChar + +-- ANCILLARIES -- + +lookupExt :: [([String], MaybePar)] -> String -> MaybePar +lookupExt [] _ = Nothing +lookupExt ((ss,p):ds) e | elem e ss = p + | otherwise = lookupExt ds e diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -11,9 +11,9 @@ import Data.Monoid -- 7.8 import qualified System.IO as I - main :: IO () -main = execParser opts >>= runLentil +main = I.hSetBuffering I.stderr I.NoBuffering >> -- b/c progress bar + execParser opts >>= runLentil where opts = info (helpOvert <*> lOpts <* version) ( fullDesc <> header "lentil - frugal issue tracker" ) @@ -28,7 +28,7 @@ short 'v' <> help "show version and copyright info" ) where - versionCopy = "\nlentil - frugal issue tracker, version 0.2.0.1\n\ + versionCopy = "\nlentil - frugal issue tracker, version 1.0.0.0\n\ \(C) 2015-2016 Francesco Ariis - http://www.ariis.it\n\ \released under the GNU General Public License v3\n" diff --git a/test/Lentil/Parse/RunSpec.hs b/test/Lentil/Parse/RunSpec.hs --- a/test/Lentil/Parse/RunSpec.hs +++ b/test/Lentil/Parse/RunSpec.hs @@ -1,12 +1,15 @@ module Lentil.Parse.RunSpec where import Test.Hspec +import Pipes import Text.Parsec ( runParser ) import Lentil.Types import Lentil.Parse.Run import Lentil.Parse.Source +import qualified Pipes.Prelude as PP + import Prelude -- 7.8 hack -- Parsing tests @@ -17,6 +20,9 @@ (runParser p () fp cs) where fp = "" +fileParser :: [Alias] -> FilePath -> IO [Issue] +fileParser as fp = fmap concat (PP.toListM $ yield (fp, 0, 0) >-> issLoop) + where issLoop = issueFinder as >> issLoop main :: IO () main = hspec spec @@ -31,7 +37,7 @@ Issue fp 8 (Just "block1") [], Issue fp 9 (Just "block2") [Tag "tog"]] - describe "commentParser - specific" $ do + describe "issueFinder - specific" $ do it "doesn't parse contiguous line/block as a single issue" $ fileParser [] "test/test-files/specific/contiguous.c" `shouldReturn` [Issue "test/test-files/specific/contiguous.c" 1 @@ -77,13 +83,4 @@ it "parses a .xyz file as if it were a .c file" $ fileParser [(".xyz", ".c")] "test/test-files/lang-comm/xyz.xyz" `shouldReturn` spt "test/test-files/lang-comm/xyz.xyz" - - describe "issueFinder" $ do - it "reads a file (code or txt) for issues" $ - let fp = "test/test-files/lang-comm/test.txt" in - issueFinder [] [fp] >>= \i -> - i `shouldBe` [Issue fp 1 (Just "palla") [], - Issue fp 3 (Just "beta") [Tag "fixme"], - Issue fp 4 (Just "gamma") [Tag "xxx"], - Issue fp 6 (Just "qqq") [Tag "fixme", Tag "bug"]]