summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndreyChudnov <>2017-07-11 23:15:00 (GMT)
committerhdiff <hdiff@hdiff.luite.com>2017-07-11 23:15:00 (GMT)
commit6e7d9722a82b17e24a4bd89d97c70027bf69218f (patch)
tree75b7cd93526698134a9cec49ce6c190bff87b22e
parentd402b78bb9240babfd9435904d1c82a88dfca4c8 (diff)
version 0.17.2.00.17.2.0
-rw-r--r--CHANGELOG6
-rw-r--r--language-ecmascript.cabal25
-rw-r--r--src/Language/ECMAScript3/Lexer.hs33
3 files changed, 50 insertions, 14 deletions
diff --git a/CHANGELOG b/CHANGELOG
index 318b41b..0902a89 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,4 +1,10 @@
Version change log.
+
+=0.17.2.0=
+Issue #82: Tighter lexing of identifiers using Unicode character classes (thanks @berdario).
+
+Dependency version bumps.
+
=0.17.1.0=
Improvements to pretty-printing (Github PR #78) + fixed a bug in quasi-quotation (Github PR #77).
diff --git a/language-ecmascript.cabal b/language-ecmascript.cabal
index 3f40434..bface40 100644
--- a/language-ecmascript.cabal
+++ b/language-ecmascript.cabal
@@ -1,8 +1,8 @@
Name: language-ecmascript
-Version: 0.17.1.0
+Version: 0.17.2.0
Cabal-Version: >= 1.10
Copyright: (c) 2007-2012 Brown University, (c) 2008-2010 Claudiu Saftoiu,
- (c) 2012-2015 Stevens Institute of Technology, (c) 2016 Andrey Chudnov, (c) 2016 Eyal Lotem
+ (c) 2012-2015 Stevens Institute of Technology, (c) 2016 Eyal Lotem, (c) 2016-2017 Andrey Chudnov
License: BSD3
License-file: LICENSE
Author: Andrey Chudnov, Arjun Guha, Spiridon Aristides Eliopoulos,
@@ -11,13 +11,13 @@ Maintainer: Andrey Chudnov <oss@chudnov.com>
Homepage: http://github.com/jswebtools/language-ecmascript
Bug-reports: http://github.com/jswebtools/language-ecmascript/issues
Stability: experimental
-Tested-with: GHC==7.4.2, GHC==7.6.3, GHC==7.8.4, GHC==7.10.3
+Tested-with: GHC==7.4.2, GHC==7.6.3, GHC==7.8.4, GHC==7.10.3, GHC==8.0.2
Extra-Source-Files: test/parse-pretty/*.js, test/diff/left/*.js, test/diff/right/*.js, test/diff/expects/*.diff, CHANGELOG
Category: Language
Build-Type: Simple
Synopsis: JavaScript parser and pretty-printer library
Description:
- Tools for working with ECMAScript 3 (popularly known as JavaScript).
+ Tools for working with ECMAScript 3 (popularly known as JavaScript).
Includes a parser, pretty-printer, tools for working with source tree
annotations and an arbitrary instance. See CHANGELOG for a summary of
changes. The package follows the Haskell Package Versioning Policy since version 0.17.0.1.
@@ -29,7 +29,7 @@ Source-repository head
Source-repository this
type: git
location: git://github.com/jswebtools/language-ecmascript.git
- tag: 0.17.1.0
+ tag: 0.17.2.0
Library
Hs-Source-Dirs:
@@ -41,16 +41,17 @@ Library
wl-pprint >= 1.2 && < 2,
containers == 0.*,
uniplate >= 1.6 && <1.7,
- data-default-class >= 0.0.1 && < 0.1,
+ data-default-class >= 0.0.1 && < 0.2,
QuickCheck >= 2.5 && < 3,
template-haskell >= 2.7 && < 3,
Diff == 0.3.*,
- testing-feat >= 0.4.0.2 && < 0.5
+ testing-feat >= 0.4.0.2 && < 0.5,
+ charset >= 0.3
ghc-options:
-fwarn-incomplete-patterns
Exposed-Modules:
- Language.ECMAScript3
- Language.ECMAScript3.Lexer
+ Language.ECMAScript3
+ Language.ECMAScript3.Lexer
Language.ECMAScript3.Parser
Language.ECMAScript3.PrettyPrint
Language.ECMAScript3.Syntax
@@ -83,11 +84,11 @@ Test-Suite test
parsec >= 3 && < 3.2.0,
wl-pprint >= 1.2 && < 2,
containers == 0.*,
- directory >= 1.2 && < 1.3,
+ directory >= 1.2 && < 1.4,
filepath >= 1.3 && < 1.5,
- HUnit >= 1.2 && < 1.4,
+ HUnit >= 1.2 && < 1.7,
QuickCheck >= 2.5 && < 3,
- data-default-class >= 0.0.1 && < 0.1,
+ data-default-class >= 0.0.1 && < 0.2,
test-framework >= 0.8 && < 0.9,
test-framework-hunit >= 0.3.0 && < 0.4,
test-framework-quickcheck2 >= 0.3.0.1 && < 0.4,
diff --git a/src/Language/ECMAScript3/Lexer.hs b/src/Language/ECMAScript3/Lexer.hs
index 1933335..cc22604 100644
--- a/src/Language/ECMAScript3/Lexer.hs
+++ b/src/Language/ECMAScript3/Lexer.hs
@@ -13,6 +13,10 @@ module Language.ECMAScript3.Lexer(lexeme,identifier,reserved,operator,reservedOp
,hexIntLit,decIntLit, decDigits, decDigitsOpt, exponentPart, decLit) where
import Prelude hiding (lex)
+import Data.Char
+import Data.Monoid ((<>), mconcat)
+import qualified Data.CharSet as Set
+import qualified Data.CharSet.Unicode.Category as Set
import Text.Parsec
import qualified Text.Parsec.Token as T
import Language.ECMAScript3.Parser.State
@@ -21,8 +25,33 @@ import Control.Monad.Identity
import Control.Applicative ((<$>), (<*>))
import Data.Maybe (isNothing)
+identifierStartCharSet :: Set.CharSet
+identifierStartCharSet =
+ mconcat
+ [ Set.fromDistinctAscList "$_"
+ , Set.lowercaseLetter
+ , Set.uppercaseLetter
+ , Set.titlecaseLetter
+ , Set.modifierLetter
+ , Set.otherLetter
+ , Set.letterNumber
+ ]
+
+identifierRestCharSet :: Set.CharSet
+identifierRestCharSet =
+ identifierStartCharSet
+ <> mconcat
+ [ Set.nonSpacingMark
+ , Set.spacingCombiningMark
+ , Set.decimalNumber
+ , Set.connectorPunctuation
+ ]
+
identifierStart :: Stream s Identity Char => Parser s Char
-identifierStart = letter <|> oneOf "$_"
+identifierStart = satisfy (flip Set.member identifierStartCharSet) <?> "letter, '$', '_'"
+
+identifierRest :: Stream s Identity Char => Parser s Char
+identifierRest = satisfy (flip Set.member identifierRestCharSet) <?> "letter, digits, '$', '_' ..."
javascriptDef :: Stream s Identity Char =>T.GenLanguageDef s ParserState Identity
javascriptDef =
@@ -31,7 +60,7 @@ javascriptDef =
"//"
False -- no nested comments
identifierStart
- (alphaNum <|> oneOf "$_") -- identifier rest
+ identifierRest
(oneOf "{}<>()~.,?:|&^=!+-*/%!") -- operator start
(oneOf "=<>|&+") -- operator rest
["break", "case", "catch", "const", "continue", "debugger",