diff --git a/Text/CSV/Lazy/ByteString.hs b/Text/CSV/Lazy/ByteString.hs --- a/Text/CSV/Lazy/ByteString.hs +++ b/Text/CSV/Lazy/ByteString.hs @@ -50,7 +50,7 @@ -- , ppCSVTableAsTuples import Data.List (groupBy, partition, elemIndex, intercalate, takeWhile - ,(\\), nub) + ,deleteFirstsBy, nub) import Data.Function (on) import Data.Maybe (fromJust) import qualified Data.ByteString.Lazy.Char8 as BS @@ -91,6 +91,7 @@ , csvField :: CSVField } | FieldError { csvField :: CSVField } | DuplicateHeader{ csvColsExpected :: !Int + , csvHeaderSerial :: !Int , csvDuplicate :: !String } | NoData deriving (Eq,Show) @@ -126,10 +127,11 @@ beCareful (Left []) = [] deduplicate (Left (errs@(DuplicateHeader{}:_)):Right heads:rows) = - Right (reverse $ foldl replace [] heads) +-- Right (reverse $ foldl replace [] heads) + Right (replaceInOrder errs (zip heads [0..])) : rows deduplicate rows = rows - +{- replace output header | headerName `elem` map csvFieldContent output = header{ csvFieldContent = headerName @@ -137,6 +139,14 @@ : output | otherwise = header: output where headerName = csvFieldContent header +-} + replaceInOrder [] headers = map fst headers + replaceInOrder _ [] = [] + replaceInOrder (d:dups) ((h,n):headers) + | csvHeaderSerial d == n = h{ csvFieldContent = BS.pack + (csvDuplicate d++"_"++show n) } + : replaceInOrder dups headers + | otherwise = h: replaceInOrder (d:dups) headers -- | The header row of the CSV table, assuming it is non-empty. csvTableHeader :: CSVResult -> [String] @@ -200,11 +210,17 @@ checkDuplicateHeaders :: CSVRow -> CSVResult -> CSVResult checkDuplicateHeaders row result = - let headers = [ csvFieldContent f | f@(CSVField{}) <- row ] - dups = headers \\ nub headers + let headers = [ f | f@(CSVField{}) <- row ] + dups = deleteFirstsBy ((==)`on`csvFieldContent) + headers (nub headers) n = length headers in if null dups then result - else Left (map (DuplicateHeader n . BS.unpack) dups) : result + else Left (map (\d-> DuplicateHeader + { csvColsExpected = n + , csvHeaderSerial = csvColNum d + , csvDuplicate = BS.unpack (csvFieldContent d)}) + dups) + : result @@ -358,7 +374,8 @@ ppCSVError (err@FieldError{}) = ppCSVField (csvField err) ppCSVError (err@DuplicateHeader{}) = "\nThere are two (or more) identical column headers: "++ - show (csvDuplicate err)++"." + show (csvDuplicate err)++"."++ + "\n Column number "++show (csvHeaderSerial err) ppCSVError (NoData{}) = "\nNo usable data (after accounting for any other errors)." diff --git a/Text/CSV/Lazy/String.hs b/Text/CSV/Lazy/String.hs --- a/Text/CSV/Lazy/String.hs +++ b/Text/CSV/Lazy/String.hs @@ -48,7 +48,7 @@ ) where import Data.List (groupBy, partition, elemIndex, intercalate, takeWhile - ,(\\), nub) + ,deleteFirstsBy, nub) import Data.Function (on) import Data.Maybe (fromJust) @@ -87,6 +87,7 @@ , csvField :: CSVField } | FieldError { csvField :: CSVField } | DuplicateHeader{ csvColsExpected :: !Int + , csvHeaderSerial :: !Int , csvDuplicate :: !String } | NoData deriving (Eq,Show) @@ -103,6 +104,7 @@ csvErrors :: CSVResult -> [CSVError] csvErrors r = concat [ v | Left v <- r ] -- | Extract the full table, including invalid rows, repaired with padding. +-- and de-duplicated headers. csvTableFull:: CSVResult -> CSVTable csvTableFull = map beCareful . deduplicate where beCareful (Right row) = row @@ -121,16 +123,25 @@ beCareful (Left []) = [] deduplicate (Left (errs@(DuplicateHeader{}:_)):Right heads:rows) = - Right (reverse $ foldl replace [] heads) +-- Right (reverse $ foldl replace [] heads) + Right (replaceInOrder errs (zip heads [0..])) : rows deduplicate rows = rows - +{- replace output header | headerName `elem` map csvFieldContent output = header{ csvFieldContent=headerName++"_duplicate" } : output | otherwise = header: output where headerName = csvFieldContent header +-} + replaceInOrder [] headers = map fst headers + replaceInOrder _ [] = [] + replaceInOrder (d:dups) ((h,n):headers) + | csvHeaderSerial d == n = h{ csvFieldContent = + (csvDuplicate d++"_"++show n) } + : replaceInOrder dups headers + | otherwise = h: replaceInOrder (d:dups) headers -- | The header row of the CSV table, assuming it is non-empty. csvTableHeader :: CSVResult -> [String] @@ -194,11 +205,17 @@ checkDuplicateHeaders :: CSVRow -> CSVResult -> CSVResult checkDuplicateHeaders row result = - let headers = [ csvFieldContent f | f@(CSVField{}) <- row ] - dups = headers \\ nub headers + let headers = [ f | f@(CSVField{}) <- row ] + dups = deleteFirstsBy ((==)`on`csvFieldContent) + headers (nub headers) n = length headers in if null dups then result - else Left (map (DuplicateHeader n) dups): result + else Left (map (\d-> DuplicateHeader + { csvColsExpected = n + , csvHeaderSerial = csvColNum d + , csvDuplicate = csvFieldContent d }) + dups) + : result -- Reading CSV data is essentially lexical, and can be implemented with a @@ -248,6 +265,10 @@ simple s' (textRow s',1) [] cs where s' = incTableRow . incTextRow $! s {tableCol=1, textCol=1} + simple s begin acc ('\r' :cs) = mkField s begin acc False : + simple s' (textRow s',1) [] cs + where s' = incTableRow . incTextRow $! + s {tableCol=1, textCol=1} simple s begin [] ('"' :cs) = string (incTextCol $! s) begin [] cs simple s begin acc ('"' :cs) = mkError s begin "Start-quote not next to comma": @@ -341,7 +362,8 @@ ppCSVError (err@FieldError{}) = ppCSVField (csvField err) ppCSVError (err@DuplicateHeader{}) = "\nThere are two (or more) identical column headers: "++ - show (csvDuplicate err)++"." + show (csvDuplicate err)++"."++ + "\n Column number "++show (csvHeaderSerial err) ppCSVError (err@NoData{}) = "\nNo usable data (after accounting for any other errors)." diff --git a/changelog.html b/changelog.html --- a/changelog.html +++ b/changelog.html @@ -13,6 +13,12 @@
+

Release 0.5

+ +

Release 0.4