razom-text-util 0.1.0.0 → 0.1.1.0

raw patch · 6 files changed

+73/−3 lines, 6 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

+ Text.Razom.Lexer: tokenizeFile :: Advance Char -> Regex t -> FilePath -> IO (LexResult t)

+ Text.Razom.Lexer: tokenizeString :: Advance Char -> Regex t -> String -> LexResult t

+ Text.Razom.Types: type LexResult t = Either (LexError t) [t]

Files

NEWS +23/−0
razom-text-util.cabal +2/−1
src/Text/Razom.hs +2/−0
src/Text/Razom/Char.hs +1/−2
src/Text/Razom/Lexer.hs +41/−0
src/Text/Razom/Types.hs +4/−0

NEWS view

@@ -1,3 +1,26 @@+razom-text-util 0.1.1.0 -- 2015-05-30+=====================================++General, build and documentation changes:++* (None)++New APIs, features and enhancements:++* Add tokenizer functions++Bug fixes:++* (None)++Dependency changes:++* (None)+++++ razom-text-util 0.1.0.0 -- 2015-05-05 =====================================

razom-text-util.cabal view

@@ -1,5 +1,5 @@ name:                razom-text-util-version:             0.1.0.0+version:             0.1.1.0 synopsis:            Common text/parsing tools for Razom language packages. description:         This is a library of utilities for writing code that                      handles semantic information documents (text files). It is@@ -25,6 +25,7 @@ library   exposed-modules:     Text.Razom                      , Text.Razom.Char+                     , Text.Razom.Lexer                      , Text.Razom.Number                      , Text.Razom.Types                      , Text.Razom.Uid

src/Text/Razom.hs view

@@ -15,6 +15,7 @@  module Text.Razom     ( module Text.Razom.Char+    , module Text.Razom.Lexer     , module Text.Razom.Number     , module Text.Razom.Types     , module Text.Razom.Uid@@ -23,6 +24,7 @@ where  import Text.Razom.Char+import Text.Razom.Lexer import Text.Razom.Number import Text.Razom.Types import Text.Razom.Uid

src/Text/Razom/Char.hs view

@@ -22,5 +22,4 @@  -- | Selects Unicode characters with basic type Graphical. isGraphical :: Char -> Bool-isGraphical c = isLetter c || isMark c || isNumber c || isPunctuation c-             || isSymbol c || generalCategory c == Space+isGraphical = isPrint

+ src/Text/Razom/Lexer.hs view

@@ -0,0 +1,41 @@+{- This file is part of razom-text-util.+ -+ - Written in 2015 by fr33domlover <fr33domlover@rel4tion.org>.+ -+ - ♡ Copying is an act of love. Please copy, reuse and share.+ -+ - The author(s) have dedicated all copyright and related and neighboring+ - rights to this software to the public domain worldwide. This software is+ - distributed without any warranty.+ -+ - You should have received a copy of the CC0 Public Domain Dedication along+ - with this software. If not, see+ - <http://creativecommons.org/publicdomain/zero/1.0/>.+ -}++module Text.Razom.Lexer+    ( tokenizeString+    , tokenizeFile+    )+where++import Data.Position+import Text.Razom.Types+import Text.Regex.Applicative+import System.IO++tokenizeString :: Advance Char -> Regex t -> String -> LexResult t+tokenizeString adv re syms =+    let depos = map $ \ (Positioned x pos) -> x+        re'   = many $ bless re+        syms' = fst $ enrich adv syms+    in case findLongestPrefix re' syms' of+        Just (ts, [])                      ->+            Right $ depos ts+        Just (ts, ((Positioned c pos):ps)) ->+            Left $ LexError pos (depos ts) (c : (depos ps))+        Nothing                            ->+            Left $ LexError firstPosition [] syms++tokenizeFile :: Advance Char -> Regex t -> FilePath -> IO (LexResult t)+tokenizeFile adv re fp = readFile fp >>= return . tokenizeString adv re

src/Text/Razom/Types.hs view

@@ -17,6 +17,7 @@     ( Regex     , PosRegex     , LexError (..)+    , LexResult (..)     ) where @@ -35,3 +36,6 @@ -- (2) Previous tokens parsed successfully -- (3) The rest of the text data LexError t = LexError Position [t] String deriving Show++-- | What is returned by lexical analyzer.+type LexResult t = Either (LexError t) [t]