unicoder 0.4.0 → 0.4.1
raw patch · 10 files changed
+157/−107 lines, 10 files
Files
- README.md +15/−4
- Text/Unicoder.hs +75/−48
- changes.md +7/−0
- data/default.conf +1/−0
- test/di.in +1/−0
- test/di.out +1/−0
- test/test.config +1/−0
- test/test.hs +1/−1
- unicoder.cabal +46/−40
- unicoder.hs +9/−14
README.md view
@@ -1,6 +1,8 @@ Unicoder ======== +[](https://travis-ci.org/Zankoku-Okuno/unicoder)+ The unicoder reads in a source file and makes replacements in-place. The goal is to allow ascii interfaces to be able to insert unicode without taking your hands off the keyboard. This can allow for unicode to be entered into source@@ -11,14 +13,15 @@ pair of unicode strings. For example, with the default configuration, unicoder turns `\floor{x} \def \lambda x. (floor x)` into `⌊x⌋ ≡ λ x. (floor x)`.-Admitedly, this is not a great syntax for some kinds of documents (esp. XeLaTeX),+Admittedly, this is not a great syntax for some kinds of documents (esp. XeLaTeX), but that's why we've allowed for configuration of each of the special marks as well as the identifier character set, so Unicoder can be relevant to any type of text data. -There's a detailed explanation of the unicoder algorithm and configuration on our-[Viewdocs](http://zankoku-okuno.viewdocs.io/unicoder/specs.md). It's fairly likely-that the system can be deduced just from examples, though.+There's more documentation on our+[Viewdocs](http://zankoku-okuno.viewdocs.io/unicoder/).+If you're learning to use Unicoder, I would especially recommend our+[examples](http://zankoku-okuno.viewdocs.io/unicoder/examples.md). Examples --------@@ -97,3 +100,11 @@ Thankfully, the pitfalls are realistically enumerable. ++Contribute+==========++Unicoder is in the beta stage. I'm sure there's a bug or two, some cleanup to be+done, and definitely some missing features. Please add any issues or pull requests+to our [github](https://github.com/Zankoku-Okuno/unicoder). You can email me, but+that's usually higher latency than github.
Text/Unicoder.hs view
@@ -3,10 +3,13 @@ unicodize , unicodizeStr , Config- , parseConfigFile+ , locateConfig+ , loadConfig+ , parseConfig ) where import System.IO+import Paths_unicoder import Data.Text (Text) import qualified Data.Text as T@@ -14,23 +17,28 @@ import Data.Attoparsec.Text import Data.Attoparsec.Combinator -import Data.Maybe import Data.Either+import Data.Maybe+import Data.List import Data.Monoid import Control.Applicative import Control.Monad +{-| Perform the unicoder transformation on a 'Text' value. -} unicodize :: Config -> Text -> Text unicodize config input = case parseOnly (xform config) input of Left err -> error "unicoder: internal error" Right val -> val +{-| Perform the unicoder transformation on a 'String' value. -} unicodizeStr :: Config -> String -> String unicodizeStr config = T.unpack . unicodize config . T.pack -data Config = Config { _idChars :: Char -> Bool+{-| Aggregate all settings needed to unicodize. -}+data Config = Config { _fromFile :: FilePath+ , _idChars :: Char -> Bool , _beginMark :: Text , _endMark :: Maybe Text , _betweenMarks :: Maybe (Text, Text)@@ -38,53 +46,68 @@ , _macros1 :: [(Text, (Text, Text))] } -parseConfigFile :: FilePath -> IO (Maybe Config)-parseConfigFile path = withFile path ReadMode $ \fp -> do- xs <- filter (not . T.null) . T.lines <$> T.hGetContents fp- return $ case xs of- [] -> Nothing- (lexer:raw_macros) -> do- --TODO configure begin mark- emptyConfig <- case filter (not . T.null) $ T.splitOn " " lexer of- [idChars] -> return- Config { _idChars = inClass (T.unpack idChars)- , _beginMark = "\\"- , _endMark = Nothing- , _betweenMarks = Nothing- , _macros0 = [], _macros1 = []- }- [begin, idChars] -> return- Config { _idChars = inClass (T.unpack idChars)- , _beginMark = begin- , _endMark = Nothing- , _betweenMarks = Nothing- , _macros0 = [], _macros1 = []- }- [begin, end, idChars] -> return- Config { _idChars = inClass (T.unpack idChars)- , _beginMark = begin- , _endMark = Just end- , _betweenMarks = Nothing- , _macros0 = [], _macros1 = []- }- [begin, end, open, close, idChars] -> return- Config { _idChars = inClass (T.unpack idChars)- , _beginMark = begin- , _endMark = Just end- , _betweenMarks = Just (open, close)- , _macros0 = [], _macros1 = []- }- _ -> Nothing- let (macros0, macros1) = partitionEithers . catMaybes $ parseMacro <$> raw_macros- return $ emptyConfig { _macros0 = macros0, _macros1 = macros1 }+{-| Determine the filesystem location of a config file path.+ If the path does not include a slash, then it is resolved using+ the unicoder built-in locations. If it does include a slash, then+ it is resolved relative to the passed working directory.+-}+locateConfig :: FilePath -- ^ the working directory+ -> FilePath -- ^ the config path to resolve+ -> IO FilePath -- ^ resolved, absolute path to the file+locateConfig cwd path | "/" `isPrefixOf` path = return path+ | '/' `elem` path = return $+ if "/" `isSuffixOf` cwd+ then cwd ++ path+ else cwd ++ "/" ++ path+ | otherwise = getDataFileName (path ++ ".conf") -parseMacro :: T.Text -> Maybe (Either (Text, Text) (Text, (Text, Text)))-parseMacro input = case T.words input of- [k, v] -> Just $ Left (k, v)- [k, v1, v2] -> Just $ Right (k, (v1, v2))- _ -> Nothing+{-| Parse a config file, possibly failing. -}+parseConfig :: FilePath -> Text -> Maybe Config+parseConfig path contents = case removeBlanks $ T.lines contents of+ [] -> Nothing+ (lexer:raw_macros) -> do+ let emptyConfig = Config { _fromFile = path+ , _idChars = undefined+ , _beginMark = "\\"+ , _endMark = Nothing+ , _betweenMarks = Nothing+ , _macros0 = [], _macros1 = []+ }+ lexerConfig <- case removeBlanks $ T.splitOn " " lexer of+ [idChars] -> return $ emptyConfig { _idChars = inClass (T.unpack idChars) }+ [begin, idChars] -> return $+ emptyConfig { _idChars = inClass (T.unpack idChars)+ , _beginMark = begin+ }+ [begin, end, idChars] -> return $+ emptyConfig { _idChars = inClass (T.unpack idChars)+ , _beginMark = begin+ , _endMark = Just end+ }+ [begin, end, open, close, idChars] -> return $+ emptyConfig { _idChars = inClass (T.unpack idChars)+ , _beginMark = begin+ , _endMark = Just end+ , _betweenMarks = Just (open, close)+ }+ _ -> Nothing+ let (macros0, macros1) = partitionEithers . catMaybes $ parseMacro <$> raw_macros+ Just $ lexerConfig { _macros0 = macros0, _macros1 = macros1 }+ where+ parseMacro :: T.Text -> Maybe (Either (Text, Text) (Text, (Text, Text)))+ parseMacro input = case T.words input of+ [k, v] -> Just $ Left (k, v)+ [k, v1, v2] -> Just $ Right (k, (v1, v2))+ _ -> Nothing +{-| Parse the contents of the passed file as unicoder `Config` -}+loadConfig :: FilePath -> IO (Maybe Config)+loadConfig path = do+ contents <- T.readFile path+ return $ parseConfig path contents ++{-| The unicoder algorithm, implemented as an Attoparsec combinator. -} xform :: Config -> Parser Text xform config = mconcat <$> many (passthrough <|> macro <|> strayBegin) <* endOfInput where@@ -107,7 +130,7 @@ (rOpen, rClose) <- name `lookupM` _macros1 config string open inner <- T.pack <$> anyChar `manyTill` string close- return $ rOpen <> inner <> rClose+ return $ rOpen <> recurse inner <> rClose half = do (open, close) <- betweenMarks which <- (const fst <$> string open) <|> (const snd <$> string close)@@ -122,6 +145,10 @@ Nothing -> fail "" Just x -> return x lookupM k v = maybe (fail "") return (k `lookup` v)+ recurse = unicodize config++removeBlanks :: [Text] -> [Text]+removeBlanks = filter (not . T.null) {-TODO a config lint
changes.md view
@@ -1,5 +1,12 @@ Changes =======+v0.4.1+------+ * Fixed issue #1: Failure to xform inside macro argument+ * Fixed issue #2: Missing API docs+ * Library functions for locating and parsing file contents.+ * Removed `parseConfigFile` for being monolithic.+ * Added a `_fromFile` field in `Config`: prepping for imports. v0.4.0 ------
data/default.conf view
@@ -13,6 +13,7 @@ p ′ hole □ +sec § prime ′ ellipsis …
test/di.in view
@@ -1,2 +1,3 @@ \bag{foo} \{bag\bag{foo}\}bag.+\bag{a \to b}
test/di.out view
@@ -1,2 +1,3 @@ ⟅foo⟆ ⟅⟅foo⟆⟆+⟅a → b⟆
test/test.config view
@@ -2,3 +2,4 @@ lambda λ bag ⟅ ⟆+to →
test/test.hs view
@@ -7,7 +7,7 @@ main :: IO () main = do- m_config <- parseConfigFile "test/test.config"+ m_config <- loadConfig "test/test.config" testFile <- case m_config of Nothing -> die "Could not parse config file." Just config -> return $ testFile config
unicoder.cabal view
@@ -1,43 +1,48 @@-name: unicoder-version: 0.4.0-stability: beta-synopsis: Make writing in unicode easy.-description: Unicoder transforms text documents, replacing simple patterns with- unicode equivalents. The patterns can be easily configured by the user.- This package is especially meant to open the vast and expressive array- of unicode identifiers to programmers and language designers, but there's- nothing wrong with a technically savvy user putting unicoder to work- on documents for human consumption.- .- With the default settings,- .- @- \\E x. \\A y. \\\<x \\-> y\\\> \\ldots- \\l x,y. x \\of x \\of y- @- .- becomes- .- @- ∃x ∀y ⟨x → y⟩ …- λ x,y. x ∘ x ∘ y- @- .- Many more possibilities abound just in the default set of characters.- Any system of special characters can be made easy to type with a normal- keyboard as long as unicode supports it.-license: BSD3-license-file: LICENSE-author: Zankoku Okuno-maintainer: zankoku.okuno@gmail.com-copyright: Copyright © 2013, 2014, Okuno Zankoku-category: Text-build-type: Simple-cabal-version: >=1.9.2-data-dir: data-data-files: *.conf-extra-source-files: README.md, changes.md,- test/*.in, test/*.out, test/test.config+name: unicoder+version: 0.4.1+stability: beta+synopsis: Make writing in unicode easy.+description: Unicoder is a command-line tool transforms text documents, replacing simple+ patterns with unicode equivalents. The patterns can be easily configured by+ the user.+ This package is especially meant to open the vast and expressive array+ of unicode identifiers to programmers and language designers, but there's+ nothing wrong with a technically savvy user putting unicoder to work+ on documents for human consumption.+ Any system of special characters can be made easy to type on any keyboard + and in any context as long as unicode supports it.+ .+ Cabal wants to fight me over typesetting some examples, so check out+ <http://zankoku-okuno.viewdocs.io/unicoder/ the real docs>+ for a decent look at the features.+ .+ In the interests of giving readers /some/ idea whats going on,+ with the default settings,+ .+ > \E x. \A y. x \-> y+ > \l x,y. x \of x \of y+ .+ becomes+ .+ > ∃x ∀y x → y+ > λ x,y. x ∘ x ∘ y+ .+ except that the newline isn't removed (thanks, cabal!). Also, there are a couple+ important features that I can't seem to get cabal to even parse (thanks again!).+license: BSD3+license-file: LICENSE+copyright: Copyright © 2013, 2014, Okuno Zankoku+author: Zankoku Okuno+maintainer: zankoku.okuno@gmail.com+homepage: https://github.com/Zankoku-Okuno/unicoder+bug-reports: https://github.com/Zankoku-Okuno/unicoder/issues+category: Text+build-type: Simple+cabal-version: >=1.9.2+data-dir: data+data-files: *.conf+extra-source-files: README.md, changes.md,+ test/*.in, test/*.out, test/test.config executable unicoder main-is: unicoder.hs@@ -49,6 +54,7 @@ library exposed-modules: Text.Unicoder+ other-modules: Paths_unicoder build-depends: base ==4.6.*, text ==0.11.*, attoparsec >=0.10.0.0
unicoder.hs view
@@ -11,16 +11,11 @@ import Paths_unicoder import Data.Version -symbolFile :: Options -> IO FilePath-symbolFile Options { optConfig = which } = - if '/' `elem` which- then return which- else getDataFileName (which ++ ".conf") main :: IO () main = do (opts, args) <- getOptions- m_config <- parseConfigFile =<< symbolFile opts+ m_config <- loadConfig =<< locateConfig "." (optConfig opts) config <- case m_config of Nothing -> die "bad configuration file" Just config -> return config@@ -39,11 +34,10 @@ putErrLn err exitFailure -makePair [a, b] = (a, b) putErrLn = hPutStrLn stderr -data Options = Options { optConfig :: String+data Options = Options { optConfig :: FilePath , optOutput :: Maybe FilePath } deriving (Show) startOptions = Options { optConfig = "default"@@ -86,17 +80,18 @@ (NoArg (\_ -> do prg <- getProgName+ putStrLn $ concat ["Unicoder v", showVersion version, "-beta"] putStrLn "Ease input of unicode glyphs." putStrLn "This program is lisenced under the 3-clause BSD lisence." putStrLn (usageInfo (prg ++ " [options] files...") options) putStrLn "Run on one or more files to replace special sequences of characters (as defined in a config file) with replacements."- putStrLn "The idea is to put in useful unicode symbols and then use those symbols in source code."- putStrLn " -- Config Files"- putStrLn " Configuration files are stored in `/etc/zankoku-okuno/unicoder`, and end in `.conf`."+ putStrLn "The idea is to make it easy to insert unicode symbols: type an escape sequence and run this tool over the source."+ putStrLn "--Config Files"+ putStrLn " Configuration files end in `.conf`. Use `--show-config-dir` to see where they are stored." putStrLn " Passing a file to `-c` that contains a slash will use exactly the file you specify, instead of looking one up."- putStrLn " Configuration files consist of a top line and a body. The body is simply a database with two whitespace-separated fields. \- \The first is the a name, and the second is the replacement. The top line contains one or two whitespace-separated fields. \- \The first (optional) is a separator, which is optionally consumed after matching a name in the database. The second holds all the characters than can be used to define and use a name."+ putStrLn " Check out an installed config file to see the syntax."+ putStrLn "--More Info"+ putStrLn " For more documentation, go to: http://zankoku-okuno.viewdocs.io/unicoder/" exitSuccess)) "Show help" ]