alex-3.2.0: tests/tokens_bytestring_unicode.x

{
{-# LANGUAGE OverloadedStrings #-}
module Main (main) where
import System.Exit
import Data.ByteString.Lazy.Char8 (unpack)
}

%wrapper "basic-bytestring"
%encoding "utf-8"

$digit = 0-9      -- digits
$alpha = [a-zA-Zαβ]    -- alphabetic characters

tokens :-

  $white+        ;
  "--".*         ;
  let            { \_ -> Let }
  in             { \_ -> In }
  $digit+                               { \s -> Int (read (unpack s)) }
  [\=\+\-\*\/\(\)]                      { \s -> Sym (head (unpack s)) }
  $alpha [$alpha $digit \_ \']*         { \s -> Var (unpack s) }

{
-- Each right-hand side has type :: ByteString -> Token

-- The token type:
data Token =
  Let     |
  In      |
  Sym Char  |
  Var String  |
  Int Int    |
  Err
  deriving (Eq,Show)

main = if test1 /= result1 then exitFailure
                           else exitWith ExitSuccess

-- \206\177\206\178\206\178 is "αββ" utf-8 encoded
test1 = alexScanTokens "  let in 012334\n=+*foo \206\177\206\178\206\178 bar__'"
result1 = [Let,In,Int 12334,Sym '=',Sym '+',Sym '*',Var "foo",Var "\206\177\206\178\206\178",Var "bar__'"]
}