diff --git a/BSD3 b/BSD3 deleted file mode 100644 --- a/BSD3 +++ /dev/null @@ -1,26 +0,0 @@ -Copyright (c) Ian Lynagh. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -3. The names of the author may not be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. diff --git a/COPYING b/COPYING deleted file mode 100644 --- a/COPYING +++ /dev/null @@ -1,6 +0,0 @@ - -Copyright (c) Ian Lynagh, 2005. - -This package can be used under either the GPL v2, as in ./GPL-2, or the -3-clause BSD, as in ./BSD3, license. - diff --git a/Changelog b/Changelog deleted file mode 100644 --- a/Changelog +++ /dev/null @@ -1,9 +0,0 @@ -0.2: - -* Export constructors of ConversionResult and derive Show, Read, Eq and - Ord for it. - -0.1: - -* Initial release. - diff --git a/Codec/Text/IConv.hs b/Codec/Text/IConv.hs new file mode 100644 --- /dev/null +++ b/Codec/Text/IConv.hs @@ -0,0 +1,475 @@ +----------------------------------------------------------------------------- +-- | +-- Copyright : (c) 2006-2007 Duncan Coutts +-- License : BSD-style +-- +-- Maintainer : duncan@haskell.org +-- Stability : experimental +-- Portability : portable (H98 + FFI) +-- +-- String encoding conversion +-- +----------------------------------------------------------------------------- +module Codec.Text.IConv ( + + -- | This module provides pure functions for converting the string encoding + -- of strings represented by lazy 'ByteString's. This makes it easy to use + -- either in memory or with disk or network IO. + -- + -- For example, a simple Latin1 to UTF-8 conversion program is just: + -- + -- > import Codec.Text.IConv as IConv + -- > import Data.ByteString.Lazy as ByteString + -- > + -- > main = ByteString.interact (convert "LATIN1" "UTF-8") + -- + -- Or you could lazily read in and convert a UTF-8 file to UTF-32 using: + -- + -- > content <- fmap (IConv.convert "UTF-8" "UTF-32") (readFile file) + -- + -- This module uses the POSIX @iconv()@ library function. The primary + -- advantage of using iconv is that it is widely available, most systems + -- have a wide range of supported string encodings and the conversion speed + -- it typically good. The iconv library is available on all unix systems + -- (since it is required by the POSIX.1 standard) and GNU libiconv is + -- available as a standalone library for other systems, including Windows. + + -- * Simple conversion API + convert, + EncodingName, + + -- * Variant that is lax about conversion errors + convertFuzzy, + Fuzzy(..), + + -- * Variants that are pedantic about conversion errors + convertStrictly, + convertLazily, + ConversionError(..), + reportConversionError, + Span(..), + + ) where + +import Prelude hiding (length, span) + +import Control.Exception (assert) +import qualified Control.Exception as Exception +import Foreign.C.Error as C.Error (Errno, errnoToIOError) + +import qualified Data.ByteString.Lazy as L (ByteString, toChunks, fromChunks) +#ifndef BYTESTRING_IN_BASE +import qualified Data.ByteString.Lazy.Internal as L (defaultChunkSize) +#endif +import qualified Data.ByteString as S + +import qualified Codec.Text.IConv.Internal as IConv +import Codec.Text.IConv.Internal (IConv) + + +-- | A string encoding name, eg @\"UTF-8\"@ or @\"LATIN1\"@. +-- +-- The range of string encodings available is determined by the capabilities +-- of the underlying iconv implementation. +-- +-- When using the GNU C or libiconv libraries, the permitted values are listed +-- by the @iconv --list@ command, and all combinations of the listed values +-- are supported. +-- +type EncodingName = String + +-- | Output spans from encoding conversion. When nothing goes wrong we +-- expect just a bunch of 'Span's. If there are conversion errors we get other +-- span types. +-- +data Span = + + -- | An ordinary output span in the target encoding + Span !S.ByteString + + -- | An error in the conversion process. If this occurs it will be the + -- last span. + | ConversionError !ConversionError + +data ConversionError = + -- | The conversion from the input to output string encoding is not + -- supported by the underlying iconv implementation. This is usually + -- because a named encoding is not recognised or support for it + -- was not enabled on this system. + -- + -- The POSIX standard does not guarantee that all possible combinations + -- of recognised string encoding are supported, however most common + -- implementations do support all possible combinations. + -- + UnsuportedConversion EncodingName EncodingName + + -- | This covers two possible conversion errors: + -- + -- * There is a byte sequence in the input that is not valid in the input + -- encoding. + -- + -- * There is a valid character in the input that has no corresponding + -- character in the output encoding. + -- + -- Unfortunately iconv does not let us distinguish these two cases. In + -- either case, the Int parameter gives the byte offset in the input of + -- the unrecognised bytes or unconvertable character. + -- + | InvalidChar Int + + -- | This error covers the case where the end of the input has trailing + -- bytes that are the initial bytes of a valid character in the input + -- encoding. In other words, it looks like the input ended in the middle of + -- a multi-byte character. This would often be an indication that the input + -- was somehow truncated. Again, the Int parameter is the byte offset in + -- the input where the incomplete character starts. + -- + | IncompleteChar Int + + -- | An unexpected iconv error. The iconv spec lists a number of possible + -- expected errors but does not guarantee that there might not be other + -- errors. + -- + -- This error can occur either immediately, which might indicate that the + -- iconv installation is messed up somehow, or it could occur later which + -- might indicate resource exhaustion or some other internal iconv error. + -- + -- Use 'Foreign.C.Error.errnoToIOError' to get slightly more information + -- on what the error could possibly be. + -- + | UnexpectedError C.Error.Errno + +reportConversionError :: ConversionError -> Exception.Exception +reportConversionError conversionError = case conversionError of + UnsuportedConversion fromEncoding toEncoding + -> err $ "cannot convert from string encoding " + ++ show fromEncoding ++ " to string encoding " + ++ show toEncoding + InvalidChar inputPos -> err $ "invalid input sequence at byte offset " + ++ show inputPos + IncompleteChar inputPos -> err $ "incomplete input sequence at byte offset " + ++ show inputPos + UnexpectedError errno -> Exception.IOException $ C.Error.errnoToIOError + "Codec.Text.IConv: unexpected error" errno + Nothing Nothing + where err msg = Exception.ErrorCall $ "Codec.Text.IConv: " ++ msg + + +{-# NOINLINE convert #-} +-- | Convert text from one named string encoding to another. +-- +-- * The conversion is done lazily. +-- +-- * An exception is thrown if conversion between the two encodings is not +-- supported. +-- +-- * An exception is thrown if there are any encoding conversion errors. +-- +convert :: EncodingName -- ^ Name of input string encoding + -> EncodingName -- ^ Name of output string encoding + -> L.ByteString -- ^ Input text + -> L.ByteString -- ^ Output text +convert fromEncoding toEncoding = + + -- lazily convert the list of spans into an ordinary lazy ByteString: + L.fromChunks + . foldr span [] + . convertLazily fromEncoding toEncoding + + where + span (Span c) cs = c : cs + span (ConversionError e) _ = Exception.throw (reportConversionError e) + + +data Fuzzy = Transliterate | Discard + +-- | Convert text ignoring encoding conversion problems. +-- +-- If invalid byte sequences are found in the input they are ignored and +-- conversion continues if possible. This is not always possible especially +-- with stateful encodings. No placeholder character is inserted into the +-- output so there will be no indication that invalid byte sequences were +-- encountered. +-- +-- If there are characters in the input that have no direct corresponding +-- character in the output encoding then they are dealt in one of two ways, +-- depending on the 'Fuzzy' argument. We can try and 'Transliterate' them into +-- the nearest corresponding character(s) or use a replacement character +-- (typically @\'?\'@ or the Unicode replacement character). Alternatively they +-- can simply be 'Discard'ed. +-- +-- In either case, no exceptions will occur. In the case of unrecoverable +-- errors, the output will simply be truncated. This includes the case of +-- unrecognised or unsupported encoding names; the output will be empty. +-- +-- * This function only works with the GNU iconv implementation which provides +-- this feature beyond what is required by the iconv specification. +-- +convertFuzzy :: Fuzzy -- ^ Whether to try and transliterate or + -- discard characters with no direct conversion + -> EncodingName -- ^ Name of input string encoding + -> EncodingName -- ^ Name of output string encoding + -> L.ByteString -- ^ Input text + -> L.ByteString -- ^ Output text +convertFuzzy fuzzy fromEncoding toEncoding = + + -- lazily convert the list of spans into an ordinary lazy ByteString: + L.fromChunks + . foldr span [] + . convertInternal IgnoreInvalidChar fromEncoding (toEncoding ++ mode) + where + mode = case fuzzy of + Transliterate -> "//IGNORE,TRANSLIT" + Discard -> "//IGNORE" + span (Span c) cs = c : cs + span (ConversionError _) cs = cs + +{-# NOINLINE convertStrictly #-} +-- | This variant does the conversion all in one go, so it is able to report +-- any conversion errors up front. It exposes all the possible error conditions +-- and never throws exceptions +-- +-- The disadvantage is that no output can be produced before the whole input +-- is consumed. This might be problematic for very large inputs. +-- +convertStrictly :: EncodingName -- ^ Name of input string encoding + -> EncodingName -- ^ Name of output string encoding + -> L.ByteString -- ^ Input text + -> Either L.ByteString + ConversionError -- ^ Output text or conversion error +convertStrictly fromEncoding toEncoding = + -- strictly convert the list of spans into an ordinary lazy ByteString + -- or an error + strictify [] + . convertLazily fromEncoding toEncoding + + where + strictify :: [S.ByteString] -> [Span] -> Either L.ByteString ConversionError + strictify cs [] = Left (L.fromChunks (reverse cs)) + strictify cs (Span c : ss) = strictify (c:cs) ss + strictify _ (ConversionError e:_) = Right e + + +{-# NOINLINE convertLazily #-} +-- | This version provides a more complete but less convenient conversion +-- interface. It exposes all the possible error conditions and never throws +-- exceptions. +-- +-- The conversion is still lazy. It returns a list of spans, where a span may +-- be an ordinary span of output text or a conversion error. This somewhat +-- complex interface allows both for lazy conversion and for precise reporting +-- of conversion problems. The other functions 'convert' and 'convertStrictly' +-- are actually simple wrappers on this function. +-- +convertLazily :: EncodingName -- ^ Name of input string encoding + -> EncodingName -- ^ Name of output string encoding + -> L.ByteString -- ^ Input text + -> [Span] -- ^ Output text spans +convertLazily = convertInternal StopOnInvalidChar + + +data InvalidCharBehaviour = StopOnInvalidChar | IgnoreInvalidChar + +convertInternal :: InvalidCharBehaviour + -> EncodingName -> EncodingName + -> L.ByteString -> [Span] +convertInternal ignore fromEncoding toEncoding input = + IConv.run fromEncoding toEncoding $ \status -> case status of + IConv.InitOk -> do IConv.newOutputBuffer outChunkSize + fillInputBuffer ignore (L.toChunks input) + + IConv.UnsupportedConversion -> failConversion (UnsuportedConversion + fromEncoding + toEncoding) + IConv.UnexpectedInitError errno -> failConversion (UnexpectedError errno) + + +fillInputBuffer :: InvalidCharBehaviour -> [S.ByteString] -> IConv [Span] +fillInputBuffer ignore (inChunk : inChunks) = do + IConv.pushInputBuffer inChunk + drainBuffers ignore inChunks + +fillInputBuffer _ignore [] = do + outputBufferBytesAvailable <- IConv.outputBufferBytesAvailable + IConv.finalise + if outputBufferBytesAvailable > 0 + then do outChunk <- IConv.popOutputBuffer + return [Span outChunk] + else return [] + + +drainBuffers :: InvalidCharBehaviour -> [S.ByteString] -> IConv [Span] +drainBuffers ignore inChunks = do + inputBufferEmpty_ <- IConv.inputBufferEmpty + outputBufferFull <- IConv.outputBufferFull + assert (not outputBufferFull && not inputBufferEmpty_) $ return () + -- this invariant guarantees we can always make forward progress + + status <- IConv.iconv + + case status of + IConv.InputEmpty -> do + inputBufferEmpty <- IConv.inputBufferEmpty + assert inputBufferEmpty $ fillInputBuffer ignore inChunks + + IConv.OutputFull -> do + outChunk <- IConv.popOutputBuffer + outChunks <- IConv.unsafeInterleave $ do + IConv.newOutputBuffer outChunkSize + drainBuffers ignore inChunks + return (Span outChunk : outChunks) + + IConv.InvalidChar -> invalidChar ignore inChunks + + IConv.IncompleteChar -> fixupBoundary ignore inChunks + + IConv.UnexpectedError errno -> failConversion (UnexpectedError errno) + +-- | The posix iconv api looks like it's designed specifically for streaming +-- and it is, except for one really really annoying corner case... +-- +-- Suppose you're converting a stream, say by reading a file in 4k chunks. This +-- would seem to be the canonical use case for iconv, reading and converting an +-- input file. However suppose the 4k read chunk happens to split a multi-byte +-- character. Then iconv will stop just before that char and tell us that its +-- an incomplete char. So far so good. Now what we'd like to do is have iconv +-- remember those last few bytes in its conversion state so we can carry on +-- with the next 4k block. Sadly it does not. It requires us to fix things up +-- so that it can carry on with the next block starting with a complete multi- +-- byte character. Do do that we have to somehow copy those few trailing bytes +-- to the beginning of the next block. That's perhaps not too bad in an +-- imperitive context using a mutable input buffer - we'd just copy the few +-- trailing bytes to the beginning of the buffer and do a short read (ie 4k-n +-- the number of trailing bytes). That's not terribly nice since it means the +-- OS has to do IO on non-page aligned buffers which tends to be slower. It's +-- worse for us though since we're not using a mutable input buffer, we're +-- using a lazy bytestring which is a sequence of immutable buffers. +-- +-- So we have to do more cunning things. We could just prepend the trailing +-- bytes to the next block, but that would mean alocating and copying the whole +-- next block just to prepend a couple bytes. This probably happens quite +-- frequently so would be pretty slow. So we have to be even more cunning. +-- +-- The solution is to create a very small buffer to cover the few bytes making +-- up the character spanning the block boundary. So we copy the trailing bytes +-- plus a few from the beginning of the next block. Then we run iconv again on +-- that small buffer. How many bytes from the next block to copy is a slightly +-- tricky issue. If we copy too few there's no guarantee that we have enough to +-- give a complete character. We opt for a maximum size of 16, 'tmpChunkSize' +-- on the theory that no encoding in existance uses that many bytes to encode a +-- single character, so it ought to be enough. Yeah, it's a tad dodgey. +-- +-- Having papered over the block boundary, we still have to cross the boundary +-- of this small buffer. It looks like we've still got the same problem, +-- however this time we should have crossed over into bytes that are wholly +-- part of the large following block so we can abandon our small temp buffer +-- an continue with the following block, with a slight offset for the few bytes +-- taken up by the chars that fit into the small buffer. +-- +-- So yeah, pretty complex. Check out the proof below of the tricky case. +-- +fixupBoundary :: InvalidCharBehaviour -> [S.ByteString] -> IConv [Span] +fixupBoundary _ignore [] = do + inputPos <- IConv.inputPosition + failConversion (IncompleteChar inputPos) +fixupBoundary ignore inChunks@(inChunk : inChunks') = do + inSize <- IConv.inputBufferSize + assert (inSize < tmpChunkSize) $ return () + let extraBytes = tmpChunkSize - inSize + + if S.length inChunk <= extraBytes + then do + IConv.replaceInputBuffer (`S.append` inChunk) + drainBuffers ignore inChunks' + else do + IConv.replaceInputBuffer (`S.append` S.take extraBytes inChunk) + + before <- IConv.inputBufferSize + assert (before == tmpChunkSize) $ return () + + status <- IConv.iconv + after <- IConv.inputBufferSize + let consumed = before - after + + case status of + IConv.InputEmpty -> + assert (consumed == tmpChunkSize) $ + fillInputBuffer ignore (S.drop extraBytes inChunk : inChunks') + + IConv.OutputFull -> do + outChunk <- IConv.popOutputBuffer + outChunks <- IConv.unsafeInterleave $ do + IConv.newOutputBuffer outChunkSize + drainBuffers ignore inChunks + return (Span outChunk : outChunks) + + IConv.InvalidChar -> invalidChar ignore inChunks + + IConv.IncompleteChar -> + assert (inSize < consumed && consumed < tmpChunkSize) $ + -- inSize < consumed < tmpChunkSize + -- => { subtract inSize from each side } + -- 0 < consumed - inSize < tmpChunkSize - inSize + -- => { by definition that extraBytes = tmpChunkSize - inSize } + -- 0 < consumed - inSize < extraBytes + -- => { since we're in the False case of the if, we know: + -- not (S.length inChunk <= extraBytes) + -- = S.length inChunk > extraBytes + -- = extraBytes < S.length inChunk } + -- 0 < consumed - inSize < extraBytes < S.length inChunk + -- + -- And we're done! We know it's safe to drop (consumed - inSize) from + -- inChunk since it's more than 0 and less than the inChunk size, so + -- we're not being left with an empty chunk (which is not allowed). + + drainBuffers ignore (S.drop (consumed - inSize) inChunk : inChunks') + + IConv.UnexpectedError errno -> failConversion (UnexpectedError errno) + + +invalidChar :: InvalidCharBehaviour -> [S.ByteString] -> IConv [Span] +invalidChar StopOnInvalidChar _ = do + inputPos <- IConv.inputPosition + failConversion (InvalidChar inputPos) + +invalidChar IgnoreInvalidChar inChunks = do + inputPos <- IConv.inputPosition + let invalidCharError = ConversionError (InvalidChar inputPos) + outputBufferBytesAvailable <- IConv.outputBufferBytesAvailable + if outputBufferBytesAvailable > 0 + then do outChunk <- IConv.popOutputBuffer + outChunks <- IConv.unsafeInterleave $ do + IConv.newOutputBuffer outChunkSize + inputBufferEmpty <- IConv.inputBufferEmpty + if inputBufferEmpty + then fillInputBuffer IgnoreInvalidChar inChunks + else drainBuffers IgnoreInvalidChar inChunks + return (Span outChunk : invalidCharError : outChunks) + else do outChunks <- IConv.unsafeInterleave $ do + IConv.newOutputBuffer outChunkSize + inputBufferEmpty <- IConv.inputBufferEmpty + if inputBufferEmpty + then fillInputBuffer IgnoreInvalidChar inChunks + else drainBuffers IgnoreInvalidChar inChunks + return (invalidCharError : outChunks) + +failConversion :: ConversionError -> IConv [Span] +failConversion err = do + outputBufferBytesAvailable <- IConv.outputBufferBytesAvailable + IConv.finalise + if outputBufferBytesAvailable > 0 + then do outChunk <- IConv.popOutputBuffer + return [Span outChunk, ConversionError err] + else return [ ConversionError err] + +outChunkSize :: Int +#ifdef BYTESTRING_IN_BASE +outChunkSize = 32 * k - overhead + where k = 1024 + overhead = 16 +#else +outChunkSize = L.defaultChunkSize +#endif + +tmpChunkSize :: Int +tmpChunkSize = 16 diff --git a/Codec/Text/IConv/Internal.hs b/Codec/Text/IConv/Internal.hs new file mode 100644 --- /dev/null +++ b/Codec/Text/IConv/Internal.hs @@ -0,0 +1,385 @@ +----------------------------------------------------------------------------- +-- | +-- Copyright : (c) 2006-2007 Duncan Coutts +-- License : BSD-style +-- +-- Maintainer : duncan@haskell.org +-- Stability : experimental +-- Portability : portable (H98 + FFI) +-- +-- IConv wrapper layer +-- +----------------------------------------------------------------------------- +module Codec.Text.IConv.Internal ( + + -- * The iconv state monad + IConv, + run, + InitStatus(..), + unsafeInterleave, + unsafeLiftIO, + finalise, + + -- * The buisness + iconv, + Status(..), + + -- * Buffer management + -- ** Input buffer + pushInputBuffer, + inputBufferSize, + inputBufferEmpty, + inputPosition, + replaceInputBuffer, + + -- ** Output buffer + newOutputBuffer, + popOutputBuffer, + outputBufferBytesAvailable, + outputBufferFull, + + -- * Debugging +-- consistencyCheck, + dump, + trace + ) where + +import Foreign +import Foreign.C +#ifdef BYTESTRING_IN_BASE +import qualified Data.ByteString.Base as S +#else +import qualified Data.ByteString.Internal as S +#endif +import System.IO.Unsafe (unsafeInterleaveIO) +import System.IO (hPutStrLn, stderr) +import Control.Exception (assert) + +import Prelude hiding (length) + + +pushInputBuffer :: S.ByteString -> IConv () +pushInputBuffer (S.PS inBuffer' inOffset' inLength') = do + + -- must not push a new input buffer if the last one is not used up + inAvail <- gets inLength + assert (inAvail == 0) $ return () + + -- now set the available input buffer ptr and length + modify $ \bufs -> bufs { + inBuffer = inBuffer', + inOffset = inOffset', + inLength = inLength' + } + + +inputBufferEmpty :: IConv Bool +inputBufferEmpty = gets ((==0) . inLength) + + +inputBufferSize :: IConv Int +inputBufferSize = gets inLength + + +inputPosition :: IConv Int +inputPosition = gets inTotal + + +replaceInputBuffer :: (S.ByteString -> S.ByteString) -> IConv () +replaceInputBuffer replace = do + modify $ \bufs@Buffers { + inBuffer = inBuffer, + inOffset = inOffset, + inLength = inLength + } -> case replace (S.PS inBuffer inOffset inLength) of + S.PS inBuffer' inOffset' inLength' -> + bufs { + inBuffer = inBuffer', + inOffset = inOffset', + inLength = inLength' + } + + +newOutputBuffer :: Int -> IConv () +newOutputBuffer size = do + + --must not push a new buffer if there is still data in the old one + outAvail <- gets outLength + assert (outAvail == 0) $ return () + -- Note that there may still be free space in the output buffer, that's ok, + -- you might not want to bother completely filling the output buffer say if + -- there's only a few free bytes left. + + -- now set the available output buffer ptr and length + outBuffer <- unsafeLiftIO $ S.mallocByteString size + modify $ \bufs -> bufs { + outBuffer = outBuffer, + outOffset = 0, + outLength = 0, + outFree = size + } + + +-- get that part of the output buffer that is currently full +-- (might be 0, use outputBufferBytesAvailable to check) +-- this may leave some space remaining in the buffer +popOutputBuffer :: IConv S.ByteString +popOutputBuffer = do + + Buffers { + outBuffer = outBuffer, + outOffset = outOffset, + outLength = outLength + } <- get + + -- there really should be something to pop, otherwise it's silly + assert (outLength > 0) $ return () + + modify $ \buf -> buf { + outOffset = outOffset + outLength, + outLength = 0 + } + + return (S.PS outBuffer outOffset outLength) + + +-- this is the number of bytes available in the output buffer +outputBufferBytesAvailable :: IConv Int +outputBufferBytesAvailable = gets outLength + + +-- you only need to supply a new buffer when there is no more output buffer +-- space remaining +outputBufferFull :: IConv Bool +outputBufferFull = gets ((==0) . outFree) + + +---------------------------- +-- IConv buffer layout +-- + +data Buffers = Buffers { + inBuffer :: {-# UNPACK #-} !(ForeignPtr Word8), -- ^ Current input buffer + inOffset :: {-# UNPACK #-} !Int, -- ^ Current read offset + inLength :: {-# UNPACK #-} !Int, -- ^ Input bytes left + inTotal :: {-# UNPACK #-} !Int, -- ^ Total read offset + outBuffer :: {-# UNPACK #-} !(ForeignPtr Word8), -- ^ Current output buffer + outOffset :: {-# UNPACK #-} !Int, -- ^ Base out offset + outLength :: {-# UNPACK #-} !Int, -- ^ Available output bytes + outFree :: {-# UNPACK #-} !Int -- ^ Free output space + } deriving Show + +nullBuffers :: Buffers +nullBuffers = Buffers S.nullForeignPtr 0 0 0 S.nullForeignPtr 0 0 0 + +{- + - For the output buffer we have this setup: + - + - +-------------+-------------+----------+ + - |### poped ###|** current **| free | + - +-------------+-------------+----------+ + - \ / \ / \ / + - outOffset outLength outFree + - + - The output buffer is allocated by us and pointer to by the outBuf ForeignPtr. + - An initial prefix of the buffer that we have already poped/yielded. This bit + - is immutable, it's already been handed out to the caller, we cannot touch it. + - When we yield we increment the outOffset. The next part of the buffer between + - outBuf + outOffset and outBuf + outOffset + outLength is the current bit that + - has had output data written into it but we have not yet yielded it to the + - caller. Finally, we have the free part of the buffer. This is the bit we + - provide to iconv to be filled. When it is written to, we increase the + - outLength and decrease the outLeft by the number of bytes written. + + - The input buffer layout is much simpler, it's basically just a bytestring: + - + - +------------+------------+ + - |### done ###| remaining | + - +------------+------------+ + - \ / \ / + - inOffset inLength + - + - So when we iconv we increase the inOffset and decrease the inLength by the + - number of bytes read. + -} + + +---------------------------- +-- IConv monad +-- + +newtype IConv a = I { + unI :: ConversionDescriptor + -> Buffers + -> IO (Buffers, a) + } + +instance Monad IConv where + (>>=) = bindI +-- m >>= f = (m `bindI` \a -> consistencyCheck `thenI` returnI a) `bindI` f + (>>) = thenI + return = returnI + +returnI :: a -> IConv a +returnI a = I $ \_ bufs -> return (bufs, a) +{-# INLINE returnI #-} + +bindI :: IConv a -> (a -> IConv b) -> IConv b +bindI m f = I $ \cd bufs -> do + (bufs', a) <- unI m cd bufs + unI (f a) cd bufs' +{-# INLINE bindI #-} + +thenI :: IConv a -> IConv b -> IConv b +thenI m f = I $ \cd bufs -> do + (bufs', _) <- unI m cd bufs + unI f cd bufs' +{-# INLINE thenI #-} + +data InitStatus = InitOk | UnsupportedConversion | UnexpectedInitError Errno + +{-# NOINLINE run #-} +run :: String -> String -> (InitStatus -> IConv a) -> a +run from to m = unsafePerformIO $ do + ptr <- withCString from $ \fromPtr -> + withCString to $ \toPtr -> + c_iconv_open toPtr fromPtr -- note arg reversal + + (cd, status) <- if ptrToIntPtr ptr /= (-1) + then do cd <- newForeignPtr c_iconv_close ptr + return (cd, InitOk) + else do errno <- getErrno + cd <- newForeignPtr_ nullPtr + if errno == eINVAL + then return (cd, UnsupportedConversion) + else return (cd, UnexpectedInitError errno) + (_,a) <- unI (m status) (ConversionDescriptor cd) nullBuffers + return a +#if __GLASGOW_HASKELL__<=604 + where ptrToIntPtr :: Ptr a -> Int + ptrToIntPtr p = p `minusPtr` nullPtr +#endif + +unsafeLiftIO :: IO a -> IConv a +unsafeLiftIO m = I $ \_ bufs -> do + a <- m + return (bufs, a) + +-- It's unsafe because we discard the values here, so if you mutate anything +-- between running this and forcing the result then you'll get an inconsistent +-- iconv state. +unsafeInterleave :: IConv a -> IConv a +unsafeInterleave m = I $ \iconv st -> do + res <- unsafeInterleaveIO (unI m iconv st) + return (st, snd res) + +get :: IConv Buffers +get = I $ \_ buf -> return (buf, buf) + +gets :: (Buffers -> a) -> IConv a +gets getter = I $ \_ buf -> return (buf, getter buf) + +modify :: (Buffers -> Buffers) -> IConv () +modify change = I $ \_ buf -> return (change buf, ()) + +---------------------------- +-- Debug stuff +-- + +trace :: String -> IConv () +trace = unsafeLiftIO . hPutStrLn stderr + + +dump :: IConv () +dump = do + bufs <- get + unsafeLiftIO $ hPutStrLn stderr $ show bufs + +---------------------------- +-- iconv wrapper layer +-- + +data Status = + InputEmpty + | OutputFull + | IncompleteChar + | InvalidChar + | UnexpectedError Errno + +iconv :: IConv Status +iconv = I $ \(ConversionDescriptor cdfptr) bufs@Buffers { + inBuffer = inBuffer, + inOffset = inOffset, + inLength = inLength, + inTotal = inTotal, + outBuffer = outBuffer, + outOffset = outOffset, + outLength = outLength, + outFree = outFree + } -> + assert (outFree > 0) $ + --TODO: optimise all this allocation + withForeignPtr cdfptr $ \cdPtr -> + withForeignPtr inBuffer $ \inBufPtr -> + with (inBufPtr `plusPtr` inOffset) $ \inBufPtrPtr -> + with (fromIntegral inLength) $ \inLengthPtr -> + withForeignPtr outBuffer $ \outBufPtr -> + with (outBufPtr `plusPtr` (outOffset + outLength)) $ \outBufPtrPtr -> + with (fromIntegral outFree) $ \outFreePtr -> do + + result <- c_iconv cdPtr inBufPtrPtr inLengthPtr outBufPtrPtr outFreePtr + inLength' <- fromIntegral `fmap` peek inLengthPtr + outFree' <- fromIntegral `fmap` peek outFreePtr + let inByteCount = inLength - inLength' + outByteCount = outFree - outFree' + bufs' = bufs { + inOffset = inOffset + inByteCount, + inLength = inLength', + inTotal = inTotal + inByteCount, + outLength = outLength + outByteCount, + outFree = outFree' + } + if result /= errVal + then return (bufs', InputEmpty) + else do errno <- getErrno + case () of + _ | errno == e2BIG -> return (bufs', OutputFull) + | errno == eINVAL -> return (bufs', IncompleteChar) + | errno == eILSEQ -> return (bufs', InvalidChar) + | otherwise -> return (bufs', UnexpectedError errno) + + where errVal :: CSize + errVal = (-1) -- (size_t)(-1) + +-- | This never needs to be used as the iconv descriptor will be released +-- automatically when no longer needed, however this can be used to release +-- it early. Only use this when you can guarantee that the iconv will no +-- longer be needed, for example if an error occurs or if the input stream +-- ends. +-- +finalise :: IConv () +finalise = I $ \(ConversionDescriptor cd) bufs -> do + finalizeForeignPtr cd + return (bufs, ()) + + +---------------------- +-- The foreign imports + +newtype ConversionDescriptor = ConversionDescriptor (ForeignPtr ConversionDescriptor) -- iconv_t + +foreign import ccall unsafe "iconv.h iconv_open" + c_iconv_open :: CString -- to code + -> CString -- from code + -> IO (Ptr ConversionDescriptor) + +foreign import ccall unsafe "iconv.h iconv" + c_iconv :: Ptr ConversionDescriptor + -> Ptr (Ptr CChar) -- in buf + -> Ptr CSize -- in buf bytes left + -> Ptr (Ptr CChar) -- out buf + -> Ptr CSize -- out buf bytes left + -> IO CSize + +foreign import ccall unsafe "iconv.h &iconv_close" + c_iconv_close :: FinalizerPtr ConversionDescriptor diff --git a/GPL-2 b/GPL-2 deleted file mode 100644 --- a/GPL-2 +++ /dev/null @@ -1,340 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2006-2007, Duncan Coutts +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/README b/README new file mode 100644 --- /dev/null +++ b/README @@ -0,0 +1,40 @@ + +Codec.Text.IConv +================ + +This is a Haskell binding to the iconv() C library function. + +The only module exported is Codec.Text.IConv, which provides a single +function: + +-- | Convert fromCharset toCharset input output +convert :: String -> String -> Lazy.ByteString -> Lazy.ByteString + +where fromCharset and toCharset are the names of the input and output +character set encodings, and input and output are the input and output text +as lazy ByteStrings. + +An example program to convert the encoding of an input file, similar to the +iconv program, is given in examples/hiconv.hs + +Character set encodings +----------------------- + +To see a list of encoding names which are known by your operating system, +run "iconv --list" in a shell. Likely encodings are listed on the libiconv +web page: + + http://www.gnu.org/software/libiconv/ + +Availability of iconv() +----------------------- + +The iconv(3) function conforms to POSIX.1-2001. It is provided by the GNU C +library: + + http://www.gnu.org/software/libc/manual/html_node/Character-Set-Handling.html + +On systems which do not have a native iconv() implementation you may need to +install libiconv: + + http://www.gnu.org/software/libiconv/ diff --git a/Setup.hs b/Setup.hs --- a/Setup.hs +++ b/Setup.hs @@ -1,8 +1,3 @@ - -module Main (main) where - +#!/usr/bin/runhaskell import Distribution.Simple - -main :: IO () main = defaultMain - diff --git a/Text/Iconv.hs b/Text/Iconv.hs deleted file mode 100644 --- a/Text/Iconv.hs +++ /dev/null @@ -1,135 +0,0 @@ - -module Text.Iconv - ( - Charset, - Bytes, - Fuzziness(Fuzzy, Strict), - convert, - canConvert, - ConversionResult(ConversionSuccess, - ConversionFailed, - ConversionUnsupported), - ) - where - -import Prelude hiding (ioError) -import Control.Exception (bracket, ioError) -import Control.Monad (liftM) -import Foreign.C.Error (getErrno, errnoToIOError, eINVAL, eILSEQ, e2BIG) -import Foreign.C.String (CString, withCString, withCStringLen, peekCStringLen) -import Foreign.C.Types (CInt, CSize) -import Foreign.Ptr (Ptr, nullPtr, plusPtr) -import Foreign.Marshal.Alloc (alloca, allocaBytes) -import Foreign.Storable (peek, poke) -import System.IO.Unsafe (unsafePerformIO) - -type Charset = String -type Bytes = String - -data ConversionResult = ConversionSuccess Bytes - | ConversionFailed - | ConversionUnsupported - deriving (Read, Show, Eq, Ord) - --- We should get this from C, but there isn't a nice way to do so - -type Iconv = Ptr IconvInternals -data IconvInternals - -foreign import ccall unsafe "static iconv.h iconv_open" - c_iconv_open :: CString -> CString -> IO Iconv -foreign import ccall unsafe "static iconv.h iconv" - c_iconv :: Iconv -> Ptr CString -> Ptr CSize - -> Ptr CString -> Ptr CSize -> IO CSize -foreign import ccall unsafe "static iconv.h iconv_close" - c_iconv_close :: Iconv -> IO CInt - -iconv_open :: Charset -> Charset -> IO (Maybe Iconv) -iconv_open from to - = withCString from $ \c_from -> - withCString to $ \c_to -> - do i <- c_iconv_open c_to c_from - if i == plusPtr nullPtr (-1) -- Ugly hack again! - then do e <- getErrno - if e == eINVAL - then return Nothing - else ioError (errnoToIOError "iconv_open" e Nothing Nothing) - else return (Just i) - -iconv :: Bool -> Iconv -> Bytes -> IO (Maybe Bytes) -iconv ignore_ilseq i xs - = withCStringLen xs $ \(c_xs, len) -> - alloca $ \p_xs -> - alloca $ \p_len -> - allocaBytes bs $ \res -> - alloca $ \p_res -> - alloca $ \p_res_len -> - do poke p_xs c_xs - poke p_len (fromIntegral len) - let iconv_core :: Ptr CString -> Ptr CSize -> IO (Maybe Bytes) - iconv_core p_xs' p_len' - = do poke p_res res - poke p_res_len bs - r <- c_iconv i p_xs' p_len' p_res p_res_len - m_e <- if r == -1 then liftM Just getErrno else return Nothing - case m_e of - Just e - | e == eINVAL || (e == eILSEQ && not ignore_ilseq) -> - return Nothing - | e /= e2BIG && e /= eILSEQ -> - ioError (errnoToIOError "iconv" e Nothing Nothing) - _ -> do unused_len <- peek p_res_len - let res_len = bs - fromIntegral unused_len - liftM Just $ peekCStringLen (res, res_len) - let f = do m_res_init <- iconv_core p_xs p_len - case m_res_init of - Nothing -> return Nothing - Just res_init -> - do rem_inp_len <- peek p_len - m_rest <- if rem_inp_len == 0 - then iconv_core nullPtr nullPtr - else f - return $ case m_rest of - Nothing -> Nothing - Just rest -> Just (res_init ++ rest) - f - where bs :: Integral a => a - bs = 1024 - -convert :: Fuzziness -> Charset -> Charset -> Bytes -> ConversionResult -convert fuzziness from to xs - = unsafePerformIO $ - bracket (iconv_open from' to') - (\m_i -> case m_i of - Nothing -> return () - Just i -> - do -- Don't bother checking c_iconv_close suceeeds - c_iconv_close i - return ()) - (\m_i -> case m_i of - Nothing -> return ConversionUnsupported - Just i -> - do m <- iconv ignore_ilseq i xs - case m of - Nothing -> return ConversionFailed - Just x -> return $ ConversionSuccess x) - where (from', to', ignore_ilseq) = account_for_fuzziness fuzziness from to - -canConvert :: Fuzziness -> Charset -> Charset -> Bool -canConvert fuzziness from to = unsafePerformIO $ - do m_i <- iconv_open from to - case m_i of - Nothing -> return False - Just i -> do -- Don't bother checking c_iconv_close suceeeds - c_iconv_close i - return True - where (from', to', _) = account_for_fuzziness fuzziness from to - -data Fuzziness = Fuzzy | Strict - deriving Eq - -account_for_fuzziness :: Fuzziness -> Charset -> Charset - -> (Charset, Charset, Bool) -account_for_fuzziness Strict from to = (from, to, False) -account_for_fuzziness Fuzzy from to = (from, to ++ "//IGNORE//TRANSLIT", True) - diff --git a/examples/hiconv.hs b/examples/hiconv.hs new file mode 100644 --- /dev/null +++ b/examples/hiconv.hs @@ -0,0 +1,130 @@ +{- + - This example is similar to the commandline iconv program. + - Author: Conrad Parker, July 2007 + + Usage: hiconv [options] filename + + -h, -? --help, --usage Display this help and exit + -f encoding --from-code=encoding Convert characters from encoding + -t encoding --to-code=encoding Convert characters to encoding + -c --discard Discard invalid characters from output + --transliterate Transliterate unconvertable characters + -o file --output=file Specify output file (instead of stdout) + + -} + +module Main where + +import Control.Monad (when) +import System.Environment (getArgs, getProgName) +import System.Console.GetOpt (getOpt, usageInfo, + OptDescr(..), ArgDescr(..), ArgOrder(..)) +import System.Exit (exitFailure) + +import qualified Data.ByteString.Lazy as Lazy +import qualified Codec.Text.IConv as IConv + +------------------------------------------------------------ +-- main +-- + +main :: IO () +main = do + args <- getArgs + (config, filenames) <- processArgs args + + let inputFile = head filenames + input <- case inputFile of + "-" -> Lazy.getContents + _ -> Lazy.readFile inputFile + + let convert = case fuzzyConvert config of + Nothing -> IConv.convert + Just fuzz -> IConv.convertFuzzy fuzz + output = convert (fromEncoding config) (toEncoding config) input + o = outputFile config + + case o of + "-" -> Lazy.putStr output + _ -> Lazy.writeFile o output + +------------------------------------------------------------ +-- Option handling +-- + +data Config = + Config { + fromEncoding :: String, + toEncoding :: String, + fuzzyConvert :: Maybe IConv.Fuzzy, + outputFile :: FilePath + } + +defaultConfig = + Config { + fromEncoding = "", + toEncoding = "", + fuzzyConvert = Nothing, + outputFile = "-" + } + +data Option = Help + | FromEncoding String + | ToEncoding String + | Discard | Translit + | OutputFile String + deriving Eq + +options :: [OptDescr Option] +options = [ Option ['h', '?'] ["help", "usage"] (NoArg Help) + "Display this help and exit" + , Option ['f'] ["from-code"] (ReqArg FromEncoding "encoding") + "Convert characters from encoding" + , Option ['t'] ["to-code"] (ReqArg ToEncoding "encoding") + "Convert characters to encoding" + , Option ['c'] ["discard"] (NoArg Discard) + "Discard invalid characters from output" + , Option [] ["transliterate"] (NoArg Translit) + "Transliterate unconvertable characters" + , Option ['o'] ["output"] (ReqArg OutputFile "file") + "Specify output file (instead of stdout)" + ] + +processArgs :: [String] -> IO (Config, [String]) +processArgs args = do + case getOpt Permute options args of + (opts, args, errs) -> do + processHelp opts + let config = processConfig defaultConfig opts + checkConfig errs config args + return (config, args) + +checkConfig :: [String] -> Config -> [String] -> IO () +checkConfig errs config filenames = do + when (any null [fromEncoding config, toEncoding config] || null filenames) $ + processHelp [Help] + when (not (null errs)) $ do + mapM_ putStr errs + processHelp [Help] + +processHelp :: [Option] -> IO () +processHelp opts = do + name <- getProgName + let header = "\nUsage: " ++ name ++ " [options] filename\n" + when (Help `elem` opts) $ do + putStrLn $ usageInfo header options + exitFailure + +processConfig :: Config -> [Option] -> Config +processConfig = foldl processOneOption + where + processOneOption config (FromEncoding f) = + config {fromEncoding = f} + processOneOption config (ToEncoding t) = + config {toEncoding = t} + processOneOption config (OutputFile o) = + config {outputFile = o} + processOneOption config Discard = + config {fuzzyConvert = Just IConv.Discard} + processOneOption config Translit = + config {fuzzyConvert = Just IConv.Transliterate} diff --git a/iconv.cabal b/iconv.cabal --- a/iconv.cabal +++ b/iconv.cabal @@ -1,24 +1,30 @@ -Name: iconv -Version: 0.2 -License: OtherLicense -License-File: COPYING -Copyright: Ian Lynagh, 2005 -Author: Ian Lynagh -Maintainer: igloo@earth.li -Stability: experimental -Homepage: http://urchin.earth.li/~ian/cabal/iconv/ -Synopsis: Perform character set conversion -Description: - Provides an interface to the unix iconv functions for character - set conversion. - . - It makes use of some knowledge of glibc's iconv that isn't - guaranteed by the standard. -Category: Text -Tested-with: GHC==6.6 -Build-depends: base -Extensions: EmptyDataDecls, ForeignFunctionInterface -Extra-source-files: BSD3, GPL-2, Changelog -Exposed-modules: - Text.Iconv +name: iconv +version: 0.4 +copyright: (c) 2006-2007 Duncan Coutts +license: BSD3 +license-file: LICENSE +author: Duncan Coutts +maintainer: Duncan Coutts +synopsis: String encoding conversion +description: Provides an interface to the POSIX iconv library functions + for string encoding conversion. +stability: experimental +build-type: Simple +cabal-version: >= 1.2.1 +extra-source-files: README examples/hiconv.hs +flag bytestring_in_base + +library + exposed-modules: Codec.Text.IConv + other-modules: Codec.Text.IConv.Internal + if flag(bytestring_in_base) + -- bytestring was in base-2.0 and 2.1.1 + build-depends: base >= 2.0 && < 2.2 + cpp-options: -DBYTESTRING_IN_BASE + else + build-depends: base < 2.0 || >= 2.2, bytestring >= 0.9 + extensions: CPP, ForeignFunctionInterface + includes: iconv.h + -- extra-libraries: --may need iconv lib on some systems + ghc-options: -fvia-C -Wall