diff --git a/CHANGES.md b/CHANGES.md --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,13 @@ +1.1.1.1 + +* Fix variables with certain characters being incorrectly parsed as symbols +* Fix description of new-style encryption in usage text + +1.1.1 + +* Fix parsing various non-alphanumeric characters in unquoted keywords +* Add separate binary->text mode for new DTBs seen in Fantasia: Music Evolved + 1.1.0.1 * Fix handling of newlines in printing and lexing `.dta` diff --git a/Main.hs b/Main.hs --- a/Main.hs +++ b/Main.hs @@ -15,8 +15,9 @@ (mode : fin : fout : rest) -> withHandleIn fin $ \hin -> withHandleOut fout $ \hout -> case mode of - "-a" -> hFromDTB hin >>= hToDTA hout - "-b" -> hFromDTA hin >>= hToDTB hout . renumberFrom 1 + "-a" -> hFromDTB hin >>= hToDTA hout + "-A" -> hFromDTBv2 hin >>= hToDTA hout + "-b" -> hFromDTA hin >>= hToDTB hout . renumberFrom 1 "-d" -> decryptHandle newCrypt hin hout "-e" -> encryptHandle newCrypt key hin hout "-D" -> decryptHandle oldCrypt hin hout @@ -39,13 +40,19 @@ printUsage = do let v = showVersion version mapM_ (IO.hPutStrLn IO.stderr) - [ "dtab v"++v++", by onyxite. Built on earlier work by xorloser and deimos." + [ "dtab v"++v++", by onyxite. Built on earlier work by xorloser, deimos, and maxton." , "Usage: dtab mode file-in file-out [encrypt-key]" - , "Modes: -a converts DTB (binary) to DTA (text)" + , "" + , "RB3 and earlier text/binary conversion:" + , " -a converts DTB (binary) to DTA (text)" , " -b converts DTA (text) to DTB (binary)" + , "Post-RB3 text/binary conversion:" + , " -A converts new format DTB (binary) to DTA (text)" + , "GH2 360 and later encryption:" , " -d decrypts new-style DTB" , " -e encrypts new-style DTB, with optional key" + , "Pre-360 encryption:" , " -D decrypts old-style DTB" , " -E encrypts old-style DTB, with optional key" - , "Old-style is used in early PS2 games, new-style otherwise." + , "" , "Use a hyphen (-) for stdin (file-in) or stdout (file-out)." ] diff --git a/dtab.cabal b/dtab.cabal --- a/dtab.cabal +++ b/dtab.cabal @@ -1,5 +1,5 @@ name: dtab -version: 1.1.0.1 +version: 1.1.1.1 synopsis: Harmonix (Guitar Hero, Rock Band) DTA/DTB metadata library description: @@ -18,7 +18,7 @@ library build-depends: - base >= 4.6 && < 4.12 + base >= 4.6 && < 5 , bytestring >= 0.10.0.2 , transformers >= 0.3.0.0 , data-binary-ieee754 >= 0.4.4 @@ -41,7 +41,7 @@ Executable dtab build-depends: - base >= 4.6 && < 4.12 + base >= 4.6 && < 5 , bytestring >= 0.10.0.2 , dtab other-modules: Paths_dtab diff --git a/src/Data/DTA.hs b/src/Data/DTA.hs --- a/src/Data/DTA.hs +++ b/src/Data/DTA.hs @@ -3,7 +3,7 @@ {-# LANGUAGE CPP #-} module Data.DTA ( DTA(..), Tree(..), Chunk(..) -, lFromDTB, hFromDTB, fromDTB +, lFromDTB, hFromDTB, hFromDTBv2, fromDTB , lToDTB, hToDTB, toDTB , sFromDTA, hFromDTA, fromDTA , sToDTA, hToDTA, toDTA @@ -17,6 +17,7 @@ withFile) import Data.Binary (decode, encode) +import Data.Binary.Get (runGet) import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as B8 import qualified Data.ByteString.Lazy as BL @@ -37,6 +38,10 @@ hFromDTB :: Handle -> IO DTA hFromDTB h = decode . strictToLazy <$> B.hGetContents h + where strictToLazy b = BL.fromChunks [b] + +hFromDTBv2 :: Handle -> IO DTA +hFromDTBv2 h = runGet (binaryDTA DTAVersion2) . strictToLazy <$> B.hGetContents h where strictToLazy b = BL.fromChunks [b] toDTB :: FilePath -> DTA -> IO () diff --git a/src/Data/DTA/Base.hs b/src/Data/DTA/Base.hs --- a/src/Data/DTA/Base.hs +++ b/src/Data/DTA/Base.hs @@ -3,6 +3,7 @@ module Data.DTA.Base ( DTA(..), Tree(..), Chunk(..) , renumberFrom +, binaryDTA, DTAVersion(..) ) where #if __GLASGOW_HASKELL__ < 710 @@ -61,21 +62,60 @@ -- Binary (DTB) instances -- +data DTAVersion + = DTAVersion1 -- ^ everything before and including RB3 AFAIK + | DTAVersion2 -- ^ seen in Fantasia: Music Evolved + +binaryDTA :: DTAVersion -> Get DTA +binaryDTA version = liftA2 DTA get (binaryTree version) + -- Single byte, then a tree. instance Binary DTA where put (DTA b t) = put b >> put t - get = liftA2 DTA get get + get = binaryDTA DTAVersion1 +binaryTree :: DTAVersion -> Get Tree +binaryTree version = case version of + DTAVersion1 -> do + len <- getWord16le + nid <- getWord32le + xs <- replicateM (fromIntegral len) (binaryChunk version) + return $ Tree nid xs + DTAVersion2 -> do + _unk <- getWord32le -- always zero? + len <- getWord32le + nid <- fromIntegral <$> getWord16le + xs <- replicateM (fromIntegral len) (binaryChunk version) + return $ Tree nid xs + -- 2-byte length, 4-byte node ID, then each element in sequence. instance Binary Tree where put (Tree nid chks) = do putWord16le $ fromIntegral $ length chks putWord32le nid mapM_ put chks - get = do - len <- getWord16le - liftA2 Tree getWord32le $ replicateM (fromIntegral len) get + get = binaryTree DTAVersion1 +binaryChunk :: DTAVersion -> Get Chunk +binaryChunk version = getWord32le >>= \cid -> case cid of + 0x0 -> Int . fromIntegral <$> getWord32le + 0x1 -> Float <$> getFloat32le + 0x2 -> Var <$> getLenStr + 0x5 -> Key <$> getLenStr + 0x6 -> skip 4 >> return Unhandled + 0x7 -> IfDef <$> getLenStr + 0x8 -> skip 4 >> return Else + 0x9 -> skip 4 >> return EndIf + 0x10 -> Parens <$> binaryTree version + 0x11 -> Braces <$> binaryTree version + 0x12 -> String <$> getLenStr + 0x13 -> Brackets <$> binaryTree version + 0x20 -> Define <$> getLenStr + 0x21 -> Include <$> getLenStr + 0x22 -> Merge <$> getLenStr + 0x23 -> IfNDef <$> getLenStr + _ -> fail $ "Unidentified DTB chunk with ID " ++ show cid + -- 4-byte chunk type ID, then at least 4 bytes of chunk data. instance Binary Chunk where put c = case c of @@ -95,24 +135,7 @@ Include b -> putWord32le 0x21 >> putLenStr b Merge b -> putWord32le 0x22 >> putLenStr b IfNDef b -> putWord32le 0x23 >> putLenStr b - get = getWord32le >>= \cid -> case cid of - 0x0 -> Int . fromIntegral <$> getWord32le - 0x1 -> Float <$> getFloat32le - 0x2 -> Var <$> getLenStr - 0x5 -> Key <$> getLenStr - 0x6 -> skip 4 >> return Unhandled - 0x7 -> IfDef <$> getLenStr - 0x8 -> skip 4 >> return Else - 0x9 -> skip 4 >> return EndIf - 0x10 -> Parens <$> get - 0x11 -> Braces <$> get - 0x12 -> String <$> getLenStr - 0x13 -> Brackets <$> get - 0x20 -> Define <$> getLenStr - 0x21 -> Include <$> getLenStr - 0x22 -> Merge <$> getLenStr - 0x23 -> IfNDef <$> getLenStr - _ -> fail $ "Unidentified DTB chunk with ID " ++ show cid + get = binaryChunk DTAVersion1 -- | DTB string format: 4-byte length, then a string in latin-1. putLenStr :: B.ByteString -> Put diff --git a/src/Data/DTA/Lex.x b/src/Data/DTA/Lex.x --- a/src/Data/DTA/Lex.x +++ b/src/Data/DTA/Lex.x @@ -10,7 +10,6 @@ %wrapper "posn" $digit = 0-9 -$alpha = [a-zA-Z] tokens :- @@ -30,20 +29,24 @@ -- Numbers. Longest match rule means N.N is float, not int. (\+ | \-)? $digit+ { \pn str -> (pn, Int $ read $ dropWhile (== '+') str) } (\+ | \-)? $digit+ (\. $digit+)? (e \-? $digit+)? { \pn str -> (pn, Float $ read $ dropWhile (== '+') str) } +(\+ | \-)? \. $digit+ (e \-? $digit+)? { \pn str -> (pn, Float $ read $ case dropWhile (== '+') str of + '-' : rest -> '-' : '0' : rest + s -> '0' : s + ) +} -- Variable names. -\$ ($alpha | $digit | _)+ { \pn str -> (pn, Var $ B8.pack $ tail str) } +\$ (. # $white # [ \( \) \{ \} \[ \] ])+ { \pn str -> (pn, Var $ B8.pack $ tail str) } -- This reserved word needs to come before the general keyword rule. "kDataUnhandled" { \pn _ -> (pn, Unhandled) } --- Raw keywords. Note: these can start with digits, like "3sand7s", as long as --- they also have letters in them. -($alpha | $digit | _ | \/ | \. | \- | \= | \# | \< | \>)+ { \pn str -> (pn, Key $ B8.pack str) } --- Quoted keywords. -' ([^'] | \\')* ' { \pn str -> (pn, Key $ B8.pack $ readKey str) } - -- Quoted strings. \" ([^\"] | \n)* \" { \pn str -> (pn, String $ B8.pack $ readString str) } +-- Quoted keywords. +' ([^'] | \\')* ' { \pn str -> (pn, Key $ B8.pack $ readKey str) } +-- Raw keywords. Note: these can start with digits, like "3sand7s", as long as +-- they also have letters in them. +(. # $white # [ \( \) \{ \} \[ \] ])+ { \pn str -> (pn, Key $ B8.pack str) } -- Subtrees. \( { \pn _ -> (pn, LParen) } diff --git a/src/Data/DTA/PrettyPrint.hs b/src/Data/DTA/PrettyPrint.hs --- a/src/Data/DTA/PrettyPrint.hs +++ b/src/Data/DTA/PrettyPrint.hs @@ -48,7 +48,7 @@ -- | Produces a raw keyword or single-quoted string literal. ppKey :: String -> PP.Doc ppKey s - | all (\c -> isAlphaNum c || elem c "_/.-=#<>") s = PP.text s + | all (\c -> isAlphaNum c || elem c "_/.-=#<>&!") s && not (null s) = PP.text s | otherwise = let -- simply convert a double-quoted string to single-quoted string f "" = ""