unicoder 0.4.1 → 0.5.0
raw patch · 7 files changed
+372/−297 lines, 7 filesdep +data-defaultdep +filepathdep +twitchdep ~basedep ~directorydep ~text
Dependencies added: data-default, filepath, twitch, unicoder
Dependency ranges changed: base, directory, text
Files
- README.md +12/−3
- Text/Unicoder.hs +0/−159
- app/unicoder.hs +151/−0
- src/Text/Unicoder.hs +162/−0
- test/test.hs +12/−15
- unicoder.cabal +35/−22
- unicoder.hs +0/−98
README.md view
@@ -3,12 +3,12 @@ [](https://travis-ci.org/Zankoku-Okuno/unicoder) -The unicoder reads in a source file and makes replacements in-place. The goal+Unicoder reads in a source file and makes replacements in-place. The goal is to allow ascii interfaces to be able to insert unicode without taking your hands off the keyboard. This can allow for unicode to be entered into source code or any other text document you're editing. -Entering unicode is not as easy as typing a special string (default backslash)+Entering unicode is as easy as typing a special string (default backslash) followed by an identifier. There's also syntax for wrapping content inside a pair of unicode strings. For example, with the default configuration, unicoder turns@@ -18,11 +18,19 @@ well as the identifier character set, so Unicoder can be relevant to any type of text data. +By default, unicoder takes input on stdin and puts the unicodized verision on+stdout: `unicoder < file.in > file.out`.+Unicoder can also operate in-place on multiple files (`unicoder -i src/*.c`)+and in file-watch mode (`unicoder -w 'src/**/*.c' &`; note that the glob pattern+is quoted so that it is passed into unicoder instead of being expanded+immediately).+ There's more documentation on our [Viewdocs](http://zankoku-okuno.viewdocs.io/unicoder/). If you're learning to use Unicoder, I would especially recommend our [examples](http://zankoku-okuno.viewdocs.io/unicoder/examples.md). + Examples -------- @@ -59,7 +67,8 @@ Will remain unchanged, as `x` and `n` are not in the config file. -There are also two-part replacements. These take a single (non-nested) argument, transforming+There are also two-part replacements. These take a single (non-nested) argument,+transforming ``` \bag{black}
− Text/Unicoder.hs
@@ -1,159 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}-module Text.Unicoder (- unicodize- , unicodizeStr- , Config- , locateConfig- , loadConfig- , parseConfig- ) where--import System.IO-import Paths_unicoder--import Data.Text (Text)-import qualified Data.Text as T-import qualified Data.Text.IO as T-import Data.Attoparsec.Text-import Data.Attoparsec.Combinator--import Data.Either-import Data.Maybe-import Data.List-import Data.Monoid-import Control.Applicative-import Control.Monad---{-| Perform the unicoder transformation on a 'Text' value. -}-unicodize :: Config -> Text -> Text-unicodize config input = case parseOnly (xform config) input of- Left err -> error "unicoder: internal error"- Right val -> val--{-| Perform the unicoder transformation on a 'String' value. -}-unicodizeStr :: Config -> String -> String-unicodizeStr config = T.unpack . unicodize config . T.pack---{-| Aggregate all settings needed to unicodize. -}-data Config = Config { _fromFile :: FilePath- , _idChars :: Char -> Bool- , _beginMark :: Text- , _endMark :: Maybe Text- , _betweenMarks :: Maybe (Text, Text)- , _macros0 :: [(Text, Text)]- , _macros1 :: [(Text, (Text, Text))]- }--{-| Determine the filesystem location of a config file path.- If the path does not include a slash, then it is resolved using- the unicoder built-in locations. If it does include a slash, then- it is resolved relative to the passed working directory.--}-locateConfig :: FilePath -- ^ the working directory- -> FilePath -- ^ the config path to resolve- -> IO FilePath -- ^ resolved, absolute path to the file-locateConfig cwd path | "/" `isPrefixOf` path = return path- | '/' `elem` path = return $- if "/" `isSuffixOf` cwd- then cwd ++ path- else cwd ++ "/" ++ path- | otherwise = getDataFileName (path ++ ".conf")--{-| Parse a config file, possibly failing. -}-parseConfig :: FilePath -> Text -> Maybe Config-parseConfig path contents = case removeBlanks $ T.lines contents of- [] -> Nothing- (lexer:raw_macros) -> do- let emptyConfig = Config { _fromFile = path- , _idChars = undefined- , _beginMark = "\\"- , _endMark = Nothing- , _betweenMarks = Nothing- , _macros0 = [], _macros1 = []- }- lexerConfig <- case removeBlanks $ T.splitOn " " lexer of- [idChars] -> return $ emptyConfig { _idChars = inClass (T.unpack idChars) }- [begin, idChars] -> return $- emptyConfig { _idChars = inClass (T.unpack idChars)- , _beginMark = begin- }- [begin, end, idChars] -> return $- emptyConfig { _idChars = inClass (T.unpack idChars)- , _beginMark = begin- , _endMark = Just end- }- [begin, end, open, close, idChars] -> return $- emptyConfig { _idChars = inClass (T.unpack idChars)- , _beginMark = begin- , _endMark = Just end- , _betweenMarks = Just (open, close)- }- _ -> Nothing- let (macros0, macros1) = partitionEithers . catMaybes $ parseMacro <$> raw_macros- Just $ lexerConfig { _macros0 = macros0, _macros1 = macros1 }- where- parseMacro :: T.Text -> Maybe (Either (Text, Text) (Text, (Text, Text)))- parseMacro input = case T.words input of- [k, v] -> Just $ Left (k, v)- [k, v1, v2] -> Just $ Right (k, (v1, v2))- _ -> Nothing--{-| Parse the contents of the passed file as unicoder `Config` -}-loadConfig :: FilePath -> IO (Maybe Config)-loadConfig path = do- contents <- T.readFile path- return $ parseConfig path contents---{-| The unicoder algorithm, implemented as an Attoparsec combinator. -}-xform :: Config -> Parser Text-xform config = mconcat <$> many (passthrough <|> macro <|> strayBegin) <* endOfInput- where- (beginStr, beginChr) = (_beginMark config, T.head beginStr)- passthrough = takeWhile1 (/= beginChr)- macro = do- string beginStr- full <|> half- strayBegin = T.singleton <$> char beginChr- full = do- name <- takeWhile1 (_idChars config)- mono name <|> di name- where- mono name = do- replace <- name `lookupM` _macros0 config- endMark- return replace- di name = do- (open, close) <- betweenMarks- (rOpen, rClose) <- name `lookupM` _macros1 config- string open- inner <- T.pack <$> anyChar `manyTill` string close- return $ rOpen <> recurse inner <> rClose- half = do- (open, close) <- betweenMarks- which <- (const fst <$> string open) <|> (const snd <$> string close)- name <- takeWhile1 (_idChars config)- replace <- which <$> name `lookupM` _macros1 config- endMark- return replace- endMark = case _endMark config of- Nothing -> return ()- Just end -> option () $ void (string end)- betweenMarks = case _betweenMarks config of- Nothing -> fail "" - Just x -> return x- lookupM k v = maybe (fail "") return (k `lookup` v)- recurse = unicodize config--removeBlanks :: [Text] -> [Text]-removeBlanks = filter (not . T.null)--{-TODO-a config lint - characters don't appear twice in the lexer- open and end are distinguishable- macros are defined only using idChars- macro lines are word-length 2 or 3--}
+ app/unicoder.hs view
@@ -0,0 +1,151 @@+import Data.String (IsString(..))++import Data.Default+import Control.Monad++import System.FilePath+import System.Directory+import System.IO+import System.Exit+import System.Console.GetOpt+import System.Environment++import Twitch (Dep, (|>))+import qualified Twitch+import qualified Data.Text.IO as T++import Text.Unicoder+import Paths_unicoder+import Data.Version+++main :: IO ()+main = do+ opts <- getOptions+ m_config <- loadConfig =<< locateConfig (optConfig opts)+ config <- case m_config of+ Nothing -> die "bad configuration file"+ Just config -> return config+ case optMode opts of+ StdPipes -> runHandle config (stdin, stdout)+ InPlace files -> mapM_ (runFile config) files+ FileWatch globs ->+ Twitch.defaultMainWithOptions def $+ forM_ (fromString <$> globs) $ \glob ->+ glob |> runFile config+ exitSuccess++runFile :: Config -> FilePath -> IO ()+runFile config filename = do+ source <- T.readFile filename+ let result = unicodize config source+ T.writeFile filename result++runHandle :: Config -> (Handle, Handle) -> IO ()+runHandle config (i, o) = do+ source <- T.hGetContents i+ let result = unicodize config source+ T.hPutStr o result++{-| Determine the filesystem location of a config file path.+ If the path does not include a slash, then it is resolved using+ the unicoder built-in locations. If it does include a slash, then+ it is resolved relative to the passed working directory.+-}+locateConfig :: FilePath -- ^ the config path to resolve+ -> IO FilePath -- ^ resolved, absolute path to the file+locateConfig path+ | pathSeparator `elem` path = do+ cwd <- getCurrentDirectory+ return $ normalise (cwd </> path)+ | otherwise = getDataFileName (path <.> "conf")+++putErrLn = hPutStrLn stderr+++data Mode+ = StdPipes+ | InPlace [FilePath]+ | FileWatch [String]+ deriving (Show)+data Options = Options + { optConfig :: FilePath+ , optOutput :: Maybe FilePath+ , optMode :: Mode+ }+ deriving (Show)+instance Default Options where+ def = Options+ { optConfig = "default"+ , optOutput = Nothing+ , optMode = StdPipes+ }+++getOptions :: IO Options+getOptions = do+ (actions, args, errors) <- getOpt Permute options <$> getArgs+ if null errors+ then do+ opts <- foldl (>>=) (return def) actions+ case optMode opts of+ InPlace _ -> return $ opts { optMode = InPlace args }+ FileWatch _ -> return $ opts { optMode = FileWatch args }+ _ | null args -> return opts+ | otherwise -> die $ "unrecognized option: " ++ show (head args)+ else do+ mapM_ putErrLn errors+ exitFailure++options :: [OptDescr (Options -> IO Options)]+options =+ [ Option "i" ["in-place"]+ (NoArg+ (\opt -> return opt {optMode = InPlace (error "uninit'd in-place") }))+ "Run once on listed files."+ , Option "w" ["file-watch"]+ (NoArg+ (\opt -> return opt { optMode = FileWatch (error "uninit'd file-watch") }))+ (unlines [ "Run on files whenever they are changed."+ , "Glob syntax is supported (including `**`) so that we can watch for new files."+ , "NOTE: Quote glob patterns whenever your shell already supports glob."+ ])+ , Option "c" ["config"]+ (ReqArg+ (\arg opt -> return opt { optConfig = arg })+ "file")+ "Specify a configuration (usually a programming language)."++ , Option "V" ["version"]+ (NoArg+ (\_ -> do+ putStrLn (showVersion version)+ exitSuccess))+ "Print version."+ , Option "" ["show-config-dir"]+ (NoArg+ (\_ -> do+ putStrLn =<< getDataFileName ""+ exitSuccess))+ "Print directory where configuration files are stored."+ , Option "h" ["help"]+ (NoArg+ (\_ -> do+ prg <- getProgName+ putStrLn $ concat ["Unicoder v", showVersion version, "-beta"]+ putStrLn "Ease input of unicode glyphs."+ putStrLn "This program is lisenced under the 3-clause BSD lisence."+ putStrLn (usageInfo (prg ++ " [options] [-i files... | -w globs...]") options)+ putStrLn "Replace special sequences of characters (as defined in a config file) with otherwise hard-to-type replacements."+ putStrLn "The idea is to make it easy to insert unicode symbols: type an escape sequence and run this tool over the source."+ putStrLn "--Config Files"+ putStrLn " Configuration files end in `.conf`. Use `--show-config-dir` to see where they are stored."+ putStrLn " Passing a file to `-c` that contains a slash will use exactly the file you specify, instead of looking one up."+ putStrLn " Check out an installed config file to see the syntax."+ putStrLn "--More Info"+ putStrLn " For more documentation, go to: http://zankoku-okuno.viewdocs.io/unicoder/"+ exitSuccess))+ "Show help"+ ]+
+ src/Text/Unicoder.hs view
@@ -0,0 +1,162 @@+{-|+Load unicoder configurations and oerform unicoder transformations on text and strings.++Unicoder replaces simple macros with configured strings, e.g.++> \E.x. \A.y. x \-> y+> \l.x,y. x \of x \of y++becomes++> ∃x ∀y x → y+> λx,y. x ∘ x ∘ y++For more information, see [the documentation](http://zankoku-okuno.viewdocs.io/unicoder/).+-}+module Text.Unicoder (+ -- * Unicoder Algorithm+ unicodize+ , unicodizeLazy+ , unicodizeStr+ -- * Configuration+ , Config+ , loadConfig+ , parseConfig+ ) where++import System.IO+import System.FilePath+import System.Directory++import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.Lazy as TL+import qualified Data.Text.IO as T+import Data.Attoparsec.Text+import Data.Attoparsec.Combinator++import Data.Either+import Data.Maybe+import Data.List+import Data.Monoid+import Control.Applicative+import Control.Monad+++{-| Perform the unicoder transformation on a 'Text' value. -}+unicodize :: Config -> Text -> Text+unicodize config input = case parseOnly (xform config) input of+ Left err -> error "unicoder: internal error"+ Right val -> val++{-| Perform the unicoder transformation on a lazy 'TL.Text' value. -}+unicodizeLazy :: Config -> TL.Text -> TL.Text+unicodizeLazy config = TL.fromStrict . unicodize config . TL.toStrict++{-| Perform the unicoder transformation on a 'String' value. -}+unicodizeStr :: Config -> String -> String+unicodizeStr config = T.unpack . unicodize config . T.pack+++{-| Aggregate all settings needed to unicodize. -}+data Config = Config+ { _fromFile :: FilePath+ , _idChars :: Char -> Bool+ , _beginMark :: Text+ , _endMark :: Maybe Text+ , _betweenMarks :: Maybe (Text, Text)+ , _macros0 :: [(Text, Text)]+ , _macros1 :: [(Text, (Text, Text))]+ }++{-| Parse a config file, possibly failing. -}+parseConfig :: FilePath -> Text -> Maybe Config+parseConfig path contents = case removeBlanks $ T.lines contents of+ [] -> Nothing+ (lexer:raw_macros) -> do+ let empty = Config+ { _fromFile = path+ , _idChars = undefined+ , _beginMark = "\\"+ , _endMark = Nothing+ , _betweenMarks = Nothing+ , _macros0 = [], _macros1 = []+ }+ lexerConfig <- case removeBlanks $ T.splitOn " " lexer of+ [idChars] -> return $+ empty { _idChars = inClass (T.unpack idChars)+ }+ [begin, idChars] -> return $+ empty { _idChars = inClass (T.unpack idChars)+ , _beginMark = begin+ }+ [begin, end, idChars] -> return $+ empty { _idChars = inClass (T.unpack idChars)+ , _beginMark = begin+ , _endMark = Just end+ }+ [begin, end, open, close, idChars] -> return $+ empty { _idChars = inClass (T.unpack idChars)+ , _beginMark = begin+ , _endMark = Just end+ , _betweenMarks = Just (open, close)+ }+ _ -> Nothing+ let (macros0, macros1) = partitionEithers . catMaybes $ parseMacro <$> raw_macros+ Just $ lexerConfig { _macros0 = macros0, _macros1 = macros1 }+ where+ parseMacro :: T.Text -> Maybe (Either (Text, Text) (Text, (Text, Text)))+ parseMacro input = case T.words input of+ [k, v] -> Just $ Left (k, v)+ [k, v1, v2] -> Just $ Right (k, (v1, v2))+ _ -> Nothing++{-| Parse the contents of the passed file as unicoder `Config` -}+loadConfig :: FilePath -> IO (Maybe Config)+loadConfig path = do+ contents <- T.readFile path+ return $ parseConfig path contents+++{-| The unicoder algorithm, implemented as an Attoparsec combinator. -}+xform :: Config -> Parser Text+xform config = mconcat <$> many (passthrough <|> macro <|> strayBegin) <* endOfInput+ where+ (beginStr, beginChr) = (_beginMark config, T.head beginStr)+ passthrough = takeWhile1 (/= beginChr)+ macro = do+ string beginStr+ full <|> half+ strayBegin = T.singleton <$> char beginChr+ full = do+ name <- takeWhile1 (_idChars config)+ mono name <|> di name+ where+ mono name = do+ replace <- name `lookupM` _macros0 config+ endMark+ return replace+ di name = do+ (open, close) <- betweenMarks+ (rOpen, rClose) <- name `lookupM` _macros1 config+ string open+ inner <- T.pack <$> anyChar `manyTill` string close+ return $ rOpen <> recurse inner <> rClose+ half = do+ (open, close) <- betweenMarks+ which <- (const fst <$> string open) <|> (const snd <$> string close)+ name <- takeWhile1 (_idChars config)+ replace <- which <$> name `lookupM` _macros1 config+ endMark+ return replace+ endMark = case _endMark config of+ Nothing -> return ()+ Just end -> option () $ void (string end)+ betweenMarks = case _betweenMarks config of+ Nothing -> fail "" + Just x -> return x+ lookupM k v = maybe (fail "") return (k `lookup` v)+ recurse = unicodize config++removeBlanks :: [Text] -> [Text]+removeBlanks = filter (not . T.null)
test/test.hs view
@@ -7,21 +7,18 @@ main :: IO () main = do- m_config <- loadConfig "test/test.config"- testFile <- case m_config of- Nothing -> die "Could not parse config file."- Just config -> return $ testFile config- let files = ["passthrough", "mono", "di"]- results <- mapM testFile files- unless (all id results) $ do- mapM_ putStrLn $ zipWith (\a b -> a ++ ": " ++ if b then "OK" else "FAILURE") files results- exitFailure+ m_config <- loadConfig "test/test.config"+ testFile <- case m_config of+ Nothing -> die "Could not parse config file."+ Just config -> return $ testFile config+ let files = ["passthrough", "mono", "di"]+ results <- mapM testFile files+ unless (and results) $ do+ mapM_ putStrLn $ zipWith (\a b -> a ++ ": " ++ if b then "OK" else "FAILURE") files results+ exitFailure testFile :: Config -> FilePath -> IO Bool testFile config path = do- input <- T.readFile $ "test/" ++ path ++ ".in"- output <- T.readFile $ "test/" ++ path ++ ".out"- return $ unicodize config input == output--die :: String -> IO a-die msg = putStrLn msg >> exitFailure+ input <- T.readFile $ "test/" ++ path ++ ".in"+ output <- T.readFile $ "test/" ++ path ++ ".out"+ return $ unicodize config input == output
unicoder.cabal view
@@ -1,5 +1,5 @@ name: unicoder-version: 0.4.1+version: 0.5.0 stability: beta synopsis: Make writing in unicode easy. description: Unicoder is a command-line tool transforms text documents, replacing simple@@ -19,13 +19,13 @@ In the interests of giving readers /some/ idea whats going on, with the default settings, .- > \E x. \A y. x \-> y- > \l x,y. x \of x \of y+ > \E.x. \A.y. x \-> y+ > \l.x,y. x \of x \of y . becomes . > ∃x ∀y x → y- > λ x,y. x ∘ x ∘ y+ > λx,y. x ∘ x ∘ y . except that the newline isn't removed (thanks, cabal!). Also, there are a couple important features that I can't seem to get cabal to even parse (thanks again!).@@ -38,33 +38,46 @@ bug-reports: https://github.com/Zankoku-Okuno/unicoder/issues category: Text build-type: Simple-cabal-version: >=1.9.2+cabal-version: >=1.10 data-dir: data data-files: *.conf extra-source-files: README.md, changes.md, test/*.in, test/*.out, test/test.config -executable unicoder- main-is: unicoder.hs- -- other-modules: - build-depends: base ==4.6.*,- text ==0.11.*,- attoparsec >=0.10.0.0,- directory ==1.2.*- library+ hs-source-dirs: src exposed-modules: Text.Unicoder+ build-depends: attoparsec >=0.10.0.0,+ base >=4.8 && <5,+ directory ==1.*,+ filepath ==1.*,+ text >=0.11 && <2+ default-language: Haskell2010+ default-extensions: OverloadedStrings++executable unicoder+ hs-source-dirs: app+ main-is: unicoder.hs other-modules: Paths_unicoder- build-depends: base ==4.6.*,- text ==0.11.*,- attoparsec >=0.10.0.0+ build-depends: base,+ data-default >=0.5,+ directory,+ filepath,+ text,+ twitch ==0.1.7.*,+ unicoder+ default-language: Haskell2010+ default-extensions: OverloadedStrings -Test-Suite test-unicoder- type: exitcode-stdio-1.0- main-is: test/test.hs- build-depends: base ==4.6.*,- text ==0.11.*,- attoparsec >=0.10.0.0+test-suite test-unicoder+ hs-source-dirs: test+ type: exitcode-stdio-1.0+ main-is: test.hs+ build-depends: base,+ text,+ unicoder+ default-language: Haskell2010+ default-extensions: OverloadedStrings source-repository head type: git
− unicoder.hs
@@ -1,98 +0,0 @@-import System.Environment-import System.IO-import System.Directory-import System.Exit-import System.Console.GetOpt-import qualified Data.Text.IO as T-import Control.Monad--import Text.Unicoder--import Paths_unicoder-import Data.Version---main :: IO ()-main = do- (opts, args) <- getOptions- m_config <- loadConfig =<< locateConfig "." (optConfig opts)- config <- case m_config of- Nothing -> die "bad configuration file"- Just config -> return config- -- TODO -o/--output flag, but how for many files?- mapM_ (mainLoop config) args- exitSuccess--mainLoop :: Config -> FilePath -> IO () -mainLoop config filename = do- source <- T.readFile filename- let result = unicodize config source- T.writeFile filename result---die err = do- putErrLn err- exitFailure--putErrLn = hPutStrLn stderr---data Options = Options { optConfig :: FilePath- , optOutput :: Maybe FilePath- } deriving (Show)-startOptions = Options { optConfig = "default"- , optOutput = Nothing- }--getOptions :: IO (Options, [FilePath])-getOptions = do- (actions, args, errors) <- return . getOpt Permute options =<< getArgs- if null errors- then do- opts <- foldl (>>=) (return startOptions) actions- when (null args) $ die "no input files"- return (opts, args)- else do- mapM_ putErrLn errors- exitFailure--options :: [OptDescr (Options -> IO Options)]-options =- [ Option "c" ["config"]- (ReqArg - (\arg opt -> return opt { optConfig = arg })- "file")- "Specify a configuration (usually a programming language)."-- , Option "V" ["version"]- (NoArg- (\_ -> do- putStrLn (showVersion version)- exitSuccess))- "Print version."- , Option "" ["show-config-dir"]- (NoArg- (\_ -> do- putStrLn =<< getDataFileName ""- exitSuccess))- "Print directory where configuration files are stored."- , Option "h" ["help"]- (NoArg- (\_ -> do- prg <- getProgName- putStrLn $ concat ["Unicoder v", showVersion version, "-beta"]- putStrLn "Ease input of unicode glyphs."- putStrLn "This program is lisenced under the 3-clause BSD lisence."- putStrLn (usageInfo (prg ++ " [options] files...") options)- putStrLn "Run on one or more files to replace special sequences of characters (as defined in a config file) with replacements."- putStrLn "The idea is to make it easy to insert unicode symbols: type an escape sequence and run this tool over the source."- putStrLn "--Config Files"- putStrLn " Configuration files end in `.conf`. Use `--show-config-dir` to see where they are stored."- putStrLn " Passing a file to `-c` that contains a slash will use exactly the file you specify, instead of looking one up."- putStrLn " Check out an installed config file to see the syntax."- putStrLn "--More Info"- putStrLn " For more documentation, go to: http://zankoku-okuno.viewdocs.io/unicoder/"- exitSuccess))- "Show help"- ]-