diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,16 @@ # Changelog and Acknowledgements +## 2.2.0 +* Switched to UTF-8 to support `text` 2.0, implementing + [#22](https://github.com/sjshuck/hs-pcre2/issues/22). `text` < 2 is no longer + supported. + * Changed type synonym `PCRE2_UCHAR` from `CUShort` to `CUChar` in the + low-level bindings. + * No API changes in the high-level bindings. + * There is a minor regression in the ability to match `\R` against line + separators (U+2028) and paragraph separators (U+2029). See + [#26](https://github.com/sjshuck/hs-pcre2/issues/26). + ## 2.1.1.1 * Updated library, tests, and docs for `mtl` 2.3 and `microlens-platform` 0.4.3.0. The `mtl` part of this is pursuant to diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -63,16 +63,13 @@ speedup over `pcre-light`, and 20× over `regex-pcre`, for longer captures. * Few dependencies. -* Bundled, statically-linked UTF-16 build of up-to-date PCRE2 (version 10.40), +* Bundled, statically-linked UTF-8 build of up-to-date PCRE2 (version 10.40), with a complete, exposed Haskell binding. ## Wishlist * Many performance optimizations. Currently we are as much as 2–3× slower than other libraries for some operations, although things are - improving. (We are already faster than - [regex-base](https://hackage.haskell.org/package/regex-base)/[regex-pcre](https://hackage.haskell.org/package/regex-pcre) - when working with `Text`, even without zero-copying.) If it's really regex - processing that's causing a bottleneck, + improving. If it's really regex processing that's causing a bottleneck, [pcre-light](https://hackage.haskell.org/package/pcre-light)/[-heavy](https://hackage.haskell.org/package/pcre-heavy)/[lens-regex-pcre](https://hackage.haskell.org/package/lens-regex-pcre) are recommended instead of this library for the very best performance. * Make use of DFA matching and JIT compilation. diff --git a/pcre2.cabal b/pcre2.cabal --- a/pcre2.cabal +++ b/pcre2.cabal @@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: pcre2 -version: 2.1.1.1 +version: 2.2.0 synopsis: Regular expressions via the PCRE2 C library (included) description: Please see the README on GitHub at category: Text @@ -109,8 +109,8 @@ Paths_pcre2 hs-source-dirs: src/hs - ghc-options: -W -optc=-DPCRE2_CODE_UNIT_WIDTH=16 -optc=-DPCRE2_STATIC=1 -optc=-Wno-discarded-qualifiers -optc=-Wno-incompatible-pointer-types - cc-options: -DHAVE_CONFIG_H -DPCRE2_CODE_UNIT_WIDTH=16 -DHAVE_INTTYPES_H=1 -DHAVE_LIMITS_H=1 -DHAVE_MEMMOVE_H=1 -DHAVE_STDINT_H=1 -DHAVE_STDLIB_H=1 -DHAVE_STRERROR_H=1 -DHAVE_STRING_H=1 -DSUPPORT_JIT=1 -DSUPPORT_PCRE2_16=1 -DSUPPORT_UNICODE=1 + ghc-options: -W -optc=-DPCRE2_CODE_UNIT_WIDTH=8 -optc=-DPCRE2_STATIC=1 -optc=-Wno-discarded-qualifiers -optc=-Wno-incompatible-pointer-types + cc-options: -DHAVE_CONFIG_H -DPCRE2_CODE_UNIT_WIDTH=8 -DHAVE_INTTYPES_H=1 -DHAVE_LIMITS_H=1 -DHAVE_MEMMOVE_H=1 -DHAVE_STDINT_H=1 -DHAVE_STDLIB_H=1 -DHAVE_STRERROR_H=1 -DHAVE_STRING_H=1 -DSUPPORT_JIT=1 -DSUPPORT_UNICODE=1 include-dirs: src/c src/c/pcre2/src @@ -148,7 +148,7 @@ , microlens , mtl , template-haskell - , text <2 + , text >=2 default-language: Haskell2010 test-suite pcre2-test @@ -168,7 +168,7 @@ , mtl , pcre2 , template-haskell - , text <2 + , text >=2 default-language: Haskell2010 benchmark pcre2-benchmarks @@ -190,5 +190,5 @@ , pcre2 , regex-pcre-builtin , template-haskell - , text <2 + , text >=2 default-language: Haskell2010 diff --git a/src/hs/Text/Regex/Pcre2/Foreign.hs b/src/hs/Text/Regex/Pcre2/Foreign.hs --- a/src/hs/Text/Regex/Pcre2/Foreign.hs +++ b/src/hs/Text/Regex/Pcre2/Foreign.hs @@ -17,7 +17,7 @@ -- | "The @UCHAR@ types define unsigned code units of the appropriate widths. -- For example, @PCRE2_UCHAR16@ is usually defined as @uint16_t@." -type PCRE2_UCHAR = CUShort +type PCRE2_UCHAR = CUChar -- | "The @SPTR@ types are constant pointers to the equivalent @UCHAR@ types, -- that is, they are pointers to vectors of unsigned code units." diff --git a/src/hs/Text/Regex/Pcre2/Internal.hs b/src/hs/Text/Regex/Pcre2/Internal.hs --- a/src/hs/Text/Regex/Pcre2/Internal.hs +++ b/src/hs/Text/Regex/Pcre2/Internal.hs @@ -28,7 +28,7 @@ import Data.Typeable (cast) import Data.Void (Void, absurd) import Foreign -import Foreign.C.Types (CInt(..), CUInt(..), CUShort) +import Foreign.C.Types (CInt(..), CUInt(..), CUChar) import qualified Foreign.Concurrent as Conc import Lens.Micro import Lens.Micro.Extras (preview, view) @@ -88,8 +88,8 @@ -- ** Fast @Text@ slicing data Slice = Slice - {-# UNPACK #-} !Text.I16 - {-# UNPACK #-} !Text.I16 + {-# UNPACK #-} !Text.I8 + {-# UNPACK #-} !Text.I8 -- | Zero-copy slice a 'Text'. An unset capture is represented by a -- `pcre2_UNSET` range and is interpreted in this library as `Text.empty`. @@ -101,8 +101,8 @@ maybeThinSlice text (Slice off offEnd) | off == fromIntegral pcre2_UNSET = Nothing | otherwise = Just $ text - & Text.takeWord16 offEnd - & Text.dropWord16 off + & Text.takeWord8 offEnd + & Text.dropWord8 off -- | Slice a 'Text', copying if it's less than half of the original. Note this -- is a lazy, pure operation. @@ -118,11 +118,11 @@ | otherwise = Text.copy substring -- | Safe, type-restricted `castPtr`. -fromCUs :: Ptr CUShort -> Ptr Word16 +fromCUs :: Ptr CUChar -> Ptr Word8 fromCUs = castPtr -- | Safe, type-restricted `castPtr`. -toCUs :: Ptr Word16 -> Ptr CUShort +toCUs :: Ptr Word8 -> Ptr CUChar toCUs = castPtr -- ** Lens utilities @@ -946,22 +946,16 @@ ovecPtr <- pcre2_callout_block_offset_vector blockPtr top <- pcre2_callout_block_capture_top blockPtr forM (0 :| [1 .. fromIntegral top - 1]) $ \n -> do - [start, end] <- forM [0, 1] $ \i -> peekElemOff ovecPtr $ n * 2 + i - evaluate $ maybeSmartSlice calloutSubject $ Slice - (fromIntegral start) - (fromIntegral end) + [start, end] <- forM [0, 1] $ \i -> + fromIntegral <$> peekElemOff ovecPtr (n * 2 + i) + evaluate $ maybeSmartSlice calloutSubject $ Slice start end calloutMark <- do ptr <- pcre2_callout_block_mark blockPtr if ptr == nullPtr then return Nothing else Just <$> do - -- TODO Replace this with a more obviously best way to slurp a - -- zero-terminated region of memory into a `Text`, given - -- whatever the pcre2callout spec means by "zero-terminated". - len <- fix1 0 $ \continue off -> peekElemOff ptr off >>= \case - 0 -> return off - _ -> continue $ off + 1 + len <- lengthArray0 0 ptr Text.fromPtr (fromCUs ptr) (fromIntegral len) flags <- pcre2_callout_block_callout_flags blockPtr @@ -983,10 +977,9 @@ ovecPtr <- pcre2_substitute_callout_block_ovector blockPtr ovecCount <- pcre2_substitute_callout_block_oveccount blockPtr forM (0 :| [1 .. fromIntegral ovecCount - 1]) $ \n -> do - [start, end] <- forM [0, 1] $ \i -> peekElemOff ovecPtr $ n * 2 + i - evaluate $ maybeSmartSlice subCalloutSubject $ Slice - (fromIntegral start) - (fromIntegral end) + [start, end] <- forM [0, 1] $ \i -> + fromIntegral <$> peekElemOff ovecPtr (n * 2 + i) + evaluate $ maybeSmartSlice subCalloutSubject $ Slice start end subCalloutReplacement <- do outPtr <- pcre2_substitute_callout_block_output blockPtr @@ -1062,7 +1055,7 @@ getWhitelistedSlices :: (Traversable t) => t Int -> FromMatch t getWhitelistedSlices whitelist matchDataPtr = do ovecPtr <- pcre2_get_ovector_pointer matchDataPtr - let peekOvec :: Int -> IO Text.I16 + let peekOvec :: Int -> IO Text.I8 peekOvec = fmap fromIntegral . peekElemOff ovecPtr forM whitelist $ \i -> Slice @@ -1259,7 +1252,7 @@ getErrorMessage :: CInt -> Text getErrorMessage errorCode = unsafePerformIO $ do let bufCUs = 120 - allocaBytes (bufCUs * 2) $ \bufPtr -> do + allocaArray bufCUs $ \bufPtr -> do cus <- pcre2_get_error_message errorCode bufPtr (fromIntegral bufCUs) Text.fromPtr (fromCUs bufPtr) (fromIntegral cus) @@ -1283,6 +1276,7 @@ if len == pcre2_ERROR_BADOPTION then return Nothing -- FIXME Do we really need "+ 1" here? + -- FIXME allocaBytes looks wrong else allocaBytes (fromIntegral (len + 1) * 2) $ \ptr -> do pcre2_config what ptr Just <$> Text.fromPtr ptr (fromIntegral len - 1) diff --git a/src/hs/Text/Regex/Pcre2/TH.hs b/src/hs/Text/Regex/Pcre2/TH.hs --- a/src/hs/Text/Regex/Pcre2/TH.hs +++ b/src/hs/Text/Regex/Pcre2/TH.hs @@ -27,7 +27,7 @@ import Data.Type.Bool (If) import Data.Type.Equality (type (==)) import Foreign -import Foreign.C (CUInt) +import Foreign.C (CUInt, CUChar) import GHC.TypeLits hiding (Text) import qualified GHC.TypeLits as TypeLits import Language.Haskell.TH @@ -135,13 +135,16 @@ let indexes = takeWhile (< nameCount) [0 ..] names <- fmap IM.fromList $ forM indexes $ \i -> do let entryPtr = nameTable `advancePtr` fromIntegral (i * nameEntrySize) - groupNamePtr = entryPtr `advancePtr` 1 - groupNumber <- peek entryPtr + groupNamePtr = entryPtr `advancePtr` 2 + groupNumber <- do + [hi, lo] <- forM [0, 1] $ \off -> + fromIntegral @CUChar <$> peekByteOff entryPtr off + return $ hi `shiftL` 8 + lo groupNameLen <- lengthArray0 0 groupNamePtr groupName <- Text.fromPtr (fromCUs groupNamePtr) (fromIntegral groupNameLen) - return (fromIntegral groupNumber, groupName) + return (groupNumber, groupName) hiCaptNum <- getCodeInfo @CUInt codePtr pcre2_INFO_CAPTURECOUNT diff --git a/test/Spec.hs b/test/Spec.hs --- a/test/Spec.hs +++ b/test/Spec.hs @@ -91,12 +91,9 @@ matchOpt BadEscapeIsLiteral "\\j" "\\j" `shouldBe` Just "j" it "includes compile context options" $ do - let otherBsr - | defaultBsr == BsrUnicode = BsrAnyCrlf - | otherwise = BsrUnicode - lineSep = "\x2028" - matches "\\R" lineSep - `shouldNotBe` matchesOpt (Bsr otherBsr) "\\R" lineSep + let bsrMatchesFF bsr = matchesOpt (Bsr bsr) "\\R" "\f" + bsrMatchesFF BsrUnicode `shouldBe` True + bsrMatchesFF BsrAnyCrlf `shouldBe` False -- We already know it includes compile recursion guards @@ -223,7 +220,7 @@ describe "PCRE2 build configuration" $ do it ("includes Unicode support" `issue` 21) $ do - matchesOpt Ucp "\\w" "aleph: \x2135" `shouldBe` True + matchesOpt Ucp "\\w$" "aleph: \x2135" `shouldBe` True -- | Modify label of `describe`, `it`, etc. to include a link to a Github issue. issue :: String -> Int -> String