version 0.17.2.00.17.2.0

3 files changed, 50 insertions, 14 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 318b41b..0902a89 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,4 +1,10 @@
 Version change log.
+
+=0.17.2.0=
+Issue #82: Tighter lexing of identifiers using Unicode character classes (thanks @berdario).
+
+Dependency version bumps.
+
 =0.17.1.0=
 Improvements to pretty-printing (Github PR #78) + fixed a bug in quasi-quotation (Github PR #77).
 
diff --git a/language-ecmascript.cabal b/language-ecmascript.cabal
index 3f40434..bface40 100644
--- a/language-ecmascript.cabal
+++ b/language-ecmascript.cabal
@@ -1,8 +1,8 @@
 Name:           language-ecmascript
-Version:        0.17.1.0
+Version:        0.17.2.0
 Cabal-Version:	>= 1.10
 Copyright:      (c) 2007-2012 Brown University, (c) 2008-2010 Claudiu Saftoiu,
-                (c) 2012-2015 Stevens Institute of Technology, (c) 2016 Andrey Chudnov, (c) 2016 Eyal Lotem
+                (c) 2012-2015 Stevens Institute of Technology, (c) 2016 Eyal Lotem, (c) 2016-2017 Andrey Chudnov
 License:        BSD3
 License-file:   LICENSE
 Author:         Andrey Chudnov, Arjun Guha, Spiridon Aristides Eliopoulos,
@@ -11,13 +11,13 @@ Maintainer:     Andrey Chudnov <oss@chudnov.com>
 Homepage:       http://github.com/jswebtools/language-ecmascript
 Bug-reports:    http://github.com/jswebtools/language-ecmascript/issues
 Stability:      experimental
-Tested-with:    GHC==7.4.2, GHC==7.6.3, GHC==7.8.4, GHC==7.10.3
+Tested-with:    GHC==7.4.2, GHC==7.6.3, GHC==7.8.4, GHC==7.10.3, GHC==8.0.2
 Extra-Source-Files: test/parse-pretty/*.js, test/diff/left/*.js, test/diff/right/*.js, test/diff/expects/*.diff, CHANGELOG
 Category:       Language
 Build-Type:     Simple
 Synopsis:       JavaScript parser and pretty-printer library
 Description:
-  Tools for working with ECMAScript 3 (popularly known as JavaScript). 
+  Tools for working with ECMAScript 3 (popularly known as JavaScript).
   Includes a parser, pretty-printer, tools for working with source tree
   annotations and an arbitrary instance. See CHANGELOG for a summary of
   changes. The package follows the Haskell Package Versioning Policy since version 0.17.0.1.
@@ -29,7 +29,7 @@ Source-repository head
 Source-repository this
    type: git
    location: git://github.com/jswebtools/language-ecmascript.git
-   tag: 0.17.1.0
+   tag: 0.17.2.0
 
 Library
   Hs-Source-Dirs:
@@ -41,16 +41,17 @@ Library
     wl-pprint >= 1.2 && < 2,
     containers == 0.*,
     uniplate >= 1.6 && <1.7,
-    data-default-class >= 0.0.1 && < 0.1,
+    data-default-class >= 0.0.1 && < 0.2,
     QuickCheck >= 2.5 && < 3,
     template-haskell >= 2.7 && < 3,
     Diff == 0.3.*,
-    testing-feat >= 0.4.0.2 && < 0.5
+    testing-feat >= 0.4.0.2 && < 0.5,
+    charset >= 0.3
   ghc-options:
     -fwarn-incomplete-patterns
   Exposed-Modules:
-    Language.ECMAScript3 
-    Language.ECMAScript3.Lexer 
+    Language.ECMAScript3
+    Language.ECMAScript3.Lexer
     Language.ECMAScript3.Parser
     Language.ECMAScript3.PrettyPrint
     Language.ECMAScript3.Syntax
@@ -83,11 +84,11 @@ Test-Suite test
     parsec >= 3 && < 3.2.0,
     wl-pprint >= 1.2 && < 2,
     containers == 0.*,
-    directory >= 1.2 && < 1.3,
+    directory >= 1.2 && < 1.4,
     filepath >= 1.3 && < 1.5,
-    HUnit >= 1.2 && < 1.4,
+    HUnit >= 1.2 && < 1.7,
     QuickCheck >= 2.5 && < 3,
-    data-default-class >= 0.0.1 && < 0.1,
+    data-default-class >= 0.0.1 && < 0.2,
     test-framework >= 0.8 && < 0.9,
     test-framework-hunit >= 0.3.0 && < 0.4,
     test-framework-quickcheck2 >= 0.3.0.1 && < 0.4,
diff --git a/src/Language/ECMAScript3/Lexer.hs b/src/Language/ECMAScript3/Lexer.hs
index 1933335..cc22604 100644
--- a/src/Language/ECMAScript3/Lexer.hs
+++ b/src/Language/ECMAScript3/Lexer.hs
@@ -13,6 +13,10 @@ module Language.ECMAScript3.Lexer(lexeme,identifier,reserved,operator,reservedOp
                                  ,hexIntLit,decIntLit, decDigits, decDigitsOpt, exponentPart, decLit) where
 
 import Prelude hiding (lex)
+import Data.Char
+import Data.Monoid ((<>), mconcat)
+import qualified Data.CharSet                  as Set
+import qualified Data.CharSet.Unicode.Category as Set
 import Text.Parsec
 import qualified Text.Parsec.Token as T
 import Language.ECMAScript3.Parser.State
@@ -21,8 +25,33 @@ import Control.Monad.Identity
 import Control.Applicative ((<$>), (<*>))
 import Data.Maybe (isNothing)
 
+identifierStartCharSet :: Set.CharSet
+identifierStartCharSet =
+  mconcat
+    [ Set.fromDistinctAscList "$_"
+    , Set.lowercaseLetter
+    , Set.uppercaseLetter
+    , Set.titlecaseLetter
+    , Set.modifierLetter
+    , Set.otherLetter
+    , Set.letterNumber
+    ]
+
+identifierRestCharSet :: Set.CharSet
+identifierRestCharSet =
+  identifierStartCharSet
+    <> mconcat
+         [ Set.nonSpacingMark
+         , Set.spacingCombiningMark
+         , Set.decimalNumber
+         , Set.connectorPunctuation
+         ]
+
 identifierStart :: Stream s Identity Char => Parser s Char
-identifierStart = letter <|> oneOf "$_"
+identifierStart = satisfy (flip Set.member identifierStartCharSet) <?> "letter, '$', '_'"
+
+identifierRest :: Stream s Identity Char => Parser s Char
+identifierRest = satisfy (flip Set.member identifierRestCharSet) <?> "letter, digits, '$', '_' ..."
 
 javascriptDef :: Stream s Identity Char =>T.GenLanguageDef s ParserState Identity
 javascriptDef =
@@ -31,7 +60,7 @@ javascriptDef =
                 "//"
                 False -- no nested comments
                 identifierStart
-                (alphaNum <|> oneOf "$_") -- identifier rest
+                identifierRest
                 (oneOf "{}<>()~.,?:|&^=!+-*/%!") -- operator start
                 (oneOf "=<>|&+") -- operator rest
                 ["break", "case", "catch", "const", "continue", "debugger",