setdown 0.1.0.4 → 0.1.1.0
raw patch · 3 files changed
+62/−36 lines, 3 filesdep +table-layout
Dependencies added: table-layout
Files
- Main.hs +53/−23
- README.markdown +7/−12
- setdown.cabal +2/−1
Main.hs view
@@ -3,11 +3,15 @@ import qualified Data.ByteString.Lazy as B import qualified Data.Set as S++import qualified Data.Text.Lazy as LT import qualified Data.Text.Lazy.IO as T+import qualified Text.Layout.Table as Tab import System.Console.CmdArgs import System.Exit import Control.Monad (filterM, forM_, unless)+import Control.Arrow (first) import Data.List (intersperse, isSuffixOf, partition) import Data.Maybe (fromMaybe) @@ -37,6 +41,7 @@ data Options = Options { outputDirectory :: Maybe FilePath , setdownFile :: Maybe FilePath+ , showTransient :: Bool } deriving (Show, Data, Typeable) options :: Options@@ -48,10 +53,15 @@ &= help "The directory in which to place the output contents. Relative to your .setdown file." &= opt "output" , setdownFile = def- &= explicit &= name "input"+ &= explicit &= help "The setdown definition file that contains all of the set operations that should be performed." &= typ "definitions.setdown"+ , showTransient = def+ &= explicit+ &= name "show-transient"+ &= help "Show the simple and transient definitions that are generated for your setdown file and their intermediate results."+ &= typ "false" } &= program "setdown" &= summary "setdown allows you to perform set operations on multiple files efficiently using an intuitive language."@@ -105,6 +115,7 @@ inputFilePath <- getInputFileOrFail (setdownFile opts) putStrLn $ "==> Using setdown file: " ++ inputFilePath+ printNewline -- Todo work out the parent directory of the setdown file putStrLn "==> Creating the environment..."@@ -115,28 +126,28 @@ , cOutputDir = baseDir </> fromMaybe "output" (outputDirectory opts) } - putStrLn $ "Base Directory: " ++ cBaseDir context- putStrLn $ "Output Directory: " ++ cOutputDir context+ putStrLn $ " Base Directory: " ++ cBaseDir context+ putStrLn $ " Output Directory: " ++ cOutputDir context prepareContext context printNewline setData <- parse <$> (B.readFile inputFilePath) putStrLn "==> Parsed original definitions..." printDefinitions setData- -- Step 0: Verify that the definitions are well defined and that the referenced files exist- -- relative to the file that we pass in. printNewline + -- Step 0: Verify that the definitions are well defined and that the referenced files exist+ -- relative to the file that we pass in. putStrLn "==> Verification (Ensuring correctness in the set definitions file)" case duplicateDefinitionName setData of- [] -> putStrLn "OK: No duplicate definitions found."+ [] -> putStrLn " OK: No duplicate definitions found." xs -> do putStrLn "[Error 11] Duplicate definitions found:" mapM_ T.putStrLn xs exitWith (ExitFailure 11) case unknownIdentifier setData of- [] -> putStrLn "OK: No unknown identifiers found."+ [] -> putStrLn " OK: No unknown identifiers found." xs -> do putStrLn "[Error 12] Unknown identifiers used in the set descriptor file:" mapM_ T.putStrLn xs@@ -147,14 +158,19 @@ putStrLn "[Error 13] the following files could not be found:" forM_ allFiles (\fp -> putStrLn $ " - " ++ fp) exitWith (ExitFailure 13)- putStrLn "OK: All files in the definitions could be found."+ putStrLn " OK: All files in the definitions could be found." printNewline putStr "==> Simplifying and eliminating duplicates from set definitions..." let simpleSetData = eliminateDuplicates . orderDefinitions . complexToSimpleDefinitions $ setData- putStrLn "DONE:"- printSimpleDefinitions simpleSetData- printNewline+ if showTransient opts+ then do+ putStrLn "DONE:"+ printSimpleDefinitions simpleSetData+ printNewline+ else do+ putStrLn "DONE!"+ printNewline putStr "==> Checking for cycles in the simplified definitions..." let cycles = getCyclesInSimpleDefinitions simpleSetData@@ -165,19 +181,19 @@ putStrLn "We found the following cycles:" printCycles cycles exitWith (ExitFailure 20)- putStrLn "OK: No cycles were found in the definitions."+ putStrLn " OK: No cycles were found in the definitions." printNewline - putStrLn "==> Copying and Sorting all input files from the definitions..."+ putStrLn "==> Sorting and de-duplicated input files from the definitions..." -- Step 1: For every unique file, sort it (Use external sort for this purpose: -- https://hackage.haskell.org/package/external-sort-0.2/docs/Algorithms-ExternalSort.html add -- docs to that library if at all possible) -- TODO use file timestamps to not sort these big files more than once if possible sortedFiles <- extractAndSortFiles context (S.toList . extractFilenamesFromDefinitions $ setData) -- TODO use the simple set data here- printSortResults sortedFiles+ printTabularResults sortedFiles printNewline - putStrLn "==> Computing set operations between the files..."+ putStrLn "==> Setdown results" -- Step 2: Calculate the graph of everything that needs to be computed and compute things one at -- a time. Even make sure that you store the temporary results along the way. That way we can -- refer to them later if the same computation is made twice. We should certainly memoize with@@ -186,7 +202,7 @@ computedFiles <- runSimpleDefinitions context simpleSetData sortedFiles -- Step 3: Print out the final statistics with the defitions pointing to how many elements that -- each contained and where to find their output files.- printComputedResults computedFiles+ printComputedResults opts computedFiles filesNotFound :: [FilePath] -> IO [FilePath] filesNotFound = filterM (\x -> not <$> doesFileExist x)@@ -217,18 +233,32 @@ where wrapInQuotes x = "\"" ++ x ++ "\"" -printComputedResults :: [(SimpleDefinition, FilePath)] -> IO ()-printComputedResults results = do- unless (null tempResults) $ do+printTabularResults :: [(FilePath, FilePath)] -> IO ()+printTabularResults fileMapping = sequence_ . fmap putStrLn $ Tab.tableLines columns Tab.unicodeBoldHeaderS headers rows+ where+ headers = Tab.titlesH ["From", "To"]++ columns =+ [ Tab.column Tab.expand Tab.left Tab.noAlign Tab.noCutMark+ , Tab.column Tab.expand Tab.left Tab.noAlign Tab.noCutMark+ ]++ rows = [Tab.rowsG $ fmap (\(from, to) -> [from, to]) fileMapping]++printComputedResults :: Options -> [(SimpleDefinition, FilePath)] -> IO ()+printComputedResults opts results = do+ unless (null tempResults || not (showTransient opts)) $ do putStrLn "Transient results:"- printResults tempResults+ printTabularResults . fmap (first (LT.unpack . getIdentifier)) $ tempResults printNewline unless (null retainResults) $ do- putStrLn "Required results:"- printResults retainResults+ unless (not (showTransient opts)) $ putStrLn "Required results:"+ printTabularResults . fmap (first (LT.unpack . getIdentifier)) $ retainResults where (retainResults, tempResults) = partition (sdRetain . fst) results- printResults = sequence_ . intersperse printNewline . fmap printComputedResult+ --printResults = sequence_ . intersperse printNewline . fmap printComputedResult+ getIdentifier (SimpleDefinition ident _ _) = ident+ printComputedResult :: (SimpleDefinition, FilePath) -> IO () printComputedResult (SimpleDefinition ident _ _, fp) = do
README.markdown view
@@ -34,11 +34,8 @@ ### Input Files In setdown *each file is treated as a list of elements where each line-is an element*. You may have thought that each file would be treated as a Set of elements (no-duplicate lines). But I decided not to make this a requirement, this is because I don't assume that-you will give us sorted input with no duplicates or that it will be easy for you to do so. Instead-you can give us any file that you like and the first thing that setdown will do to those files is-sort them and remove duplicates; essentially turning them into sets.+is an element*. Input files do not need to begin as sets; they can contain duplicate and unsorted+elements. Setdown will automatically sort and de-duplicate all input files, turning them into sets. Another important point is that of relativity: specifically, if have a **.setdown** file that references the input file "some-elements.txt" and I run the setdown executable from a directory that@@ -65,7 +62,7 @@ You may be wondering what [operator precidence][1] the setdown language uses and the answer is: there is no operator precidence at all, instead *you must clearly specify the precidence of nested expressions with brackets*. This is very important because it will result in parsing errors-otherwise. To show you why I made this decision lets show you an example:+otherwise. To explain the reasoning for explicit operator precedence: -- Here is a simple expression def: A /\ B \/ C@@ -79,8 +76,7 @@ defV2-bempty: A /\ C So as you can see, order of operations really matters for set operations. Because it is so critical-I decided to make the use of brackets mandatory. Sorry for the extra brackets but you will thank me-when you expressions come out exactly the way that you expect them to.+the use of brackets is mandatory. ### Comments @@ -130,13 +126,12 @@ To build the code for this project just have [Haskell installed][4] and [cabal][5] and then: - cabal sandbox init- cabal install+ cabal new-install And that should have the code built on your machine. Then, if you modify the code, just use cabal run to run setdown: - cabal run -- --help- cabal run mydefinitions.setdown+ cabal run setdown -- --help+ cabal run setdown -- mydefinitions.setdown That is all that there is to it!
setdown.cabal view
@@ -10,7 +10,7 @@ -- PVP summary: +-+------- breaking API changes -- | | +----- non-breaking API additions -- | | | +--- code changes with no API change-version: 0.1.0.4+version: 0.1.1.0 -- A short (one-line) description of the package. synopsis: Treating files as sets to perform rapid set manipulation.@@ -95,6 +95,7 @@ , split == 0.2.* , mtl == 2.2.* , cmdargs >= 0.10+ , table-layout >= 0.8 build-tools: alex, happy