diff --git a/doc/TODO b/doc/TODO --- a/doc/TODO +++ b/doc/TODO @@ -17,3 +17,5 @@ * SportInfoListing 5. Try mass import of Auto_Racing_Schedule_XML. + + diff --git a/htsn-import.cabal b/htsn-import.cabal --- a/htsn-import.cabal +++ b/htsn-import.cabal @@ -1,5 +1,5 @@ name: htsn-import -version: 0.0.4 +version: 0.0.5 cabal-version: >= 1.8 author: Michael Orlitzky maintainer: Michael Orlitzky @@ -73,25 +73,25 @@ executable htsn-import build-depends: - base == 4.*, + base >= 4.6 && < 5, cmdargs >= 0.10.6, - configurator == 0.2.*, - directory == 1.2.*, - filepath == 1.3.*, - hslogger == 1.2.*, - htsn-common == 0.0.1, - hxt == 9.3.*, - groundhog == 0.4.*, - groundhog-postgresql == 0.4.*, - groundhog-sqlite == 0.4.*, - groundhog-th == 0.4.*, - MissingH == 1.2.*, - old-locale == 1.0.*, - tasty == 0.8.*, - tasty-hunit == 0.8.*, - time == 1.4.*, - transformers == 0.3.*, - tuple == 0.2.* + configurator >= 0.2, + directory >= 1.2, + filepath >= 1.3, + hslogger >= 1.2, + htsn-common >= 0.0.1, + hxt >= 9.3, + groundhog >= 0.5, + groundhog-postgresql >= 0.5, + groundhog-sqlite >= 0.5, + groundhog-th >= 0.5, + MissingH >= 1.2, + old-locale >= 1, + tasty >= 0.8, + tasty-hunit >= 0.8, + time >= 1.4, + transformers >= 0.3, + tuple >= 0.2 main-is: Main.hs @@ -112,12 +112,14 @@ TSN.Picklers TSN.XmlImport TSN.XML.AutoRacingSchedule + TSN.XML.GameInfo TSN.XML.Heartbeat TSN.XML.Injuries TSN.XML.InjuriesDetail TSN.XML.News TSN.XML.Odds TSN.XML.Scores + TSN.XML.SportInfo TSN.XML.Weather Xml @@ -150,25 +152,25 @@ hs-source-dirs: src test main-is: TestSuite.hs build-depends: - base == 4.*, + base >= 4.6 && < 5, cmdargs >= 0.10.6, - configurator == 0.2.*, - directory == 1.2.*, - filepath == 1.3.*, - hslogger == 1.2.*, - htsn-common == 0.0.1, - hxt == 9.3.*, - groundhog == 0.4.*, - groundhog-postgresql == 0.4.*, - groundhog-sqlite == 0.4.*, - groundhog-th == 0.4.*, - MissingH == 1.2.*, - old-locale == 1.0.*, - tasty == 0.8.*, - tasty-hunit == 0.8.*, - time == 1.4.*, - transformers == 0.3.*, - tuple == 0.2.* + configurator >= 0.2, + directory >= 1.2, + filepath >= 1.3, + hslogger >= 1.2, + htsn-common >= 0.0.1, + hxt >= 9.3, + groundhog >= 0.5, + groundhog-postgresql >= 0.5, + groundhog-sqlite >= 0.5, + groundhog-th >= 0.5, + MissingH >= 1.2, + old-locale >= 1, + tasty >= 0.8, + tasty-hunit >= 0.8, + time >= 1.4, + transformers >= 0.3, + tuple >= 0.2 -- It's not entirely clear to me why I have to reproduce all of this. ghc-options: @@ -190,9 +192,9 @@ hs-source-dirs: test main-is: Doctests.hs build-depends: - base == 4.*, + base >= 4.6 && < 5, -- Additional test dependencies. - doctest == 0.9.* + doctest >= 0.9 -- It's not entirely clear to me why I have to reproduce all of this. ghc-options: @@ -221,26 +223,26 @@ main-is: ShellTests.hs build-depends: - base == 4.*, + base >= 4.6 && < 5, cmdargs >= 0.10.6, - configurator == 0.2.*, - directory == 1.2.*, - filepath == 1.3.*, - hslogger == 1.2.*, - htsn-common == 0.0.1, - hxt == 9.3.*, - groundhog == 0.4.*, - groundhog-postgresql == 0.4.*, - groundhog-sqlite == 0.4.*, - groundhog-th == 0.4.*, - MissingH == 1.2.*, - old-locale == 1.0.*, - process == 1.1.*, - tasty == 0.8.*, - tasty-hunit == 0.8.*, - time == 1.4.*, - transformers == 0.3.*, - tuple == 0.2.* + configurator >= 0.2, + directory >= 1.2, + filepath >= 1.3, + hslogger >= 1.2, + htsn-common >= 0.0.1, + hxt >= 9.3, + groundhog >= 0.5, + groundhog-postgresql >= 0.5, + groundhog-sqlite >= 0.5, + groundhog-th >= 0.5, + MissingH >= 1.2, + old-locale >= 1, + process >= 1.1, + tasty >= 0.8, + tasty-hunit >= 0.8, + time >= 1.4, + transformers >= 0.3, + tuple >= 0.2 diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -48,6 +48,7 @@ import qualified TSN.XML.AutoRacingSchedule as AutoRacingSchedule ( dtd, pickle_message ) +import qualified TSN.XML.GameInfo as GameInfo ( dtds ) import qualified TSN.XML.Heartbeat as Heartbeat ( dtd, verify ) import qualified TSN.XML.Injuries as Injuries ( dtd, pickle_message ) import qualified TSN.XML.InjuriesDetail as InjuriesDetail ( @@ -56,6 +57,7 @@ import qualified TSN.XML.News as News ( dtd, pickle_message ) import qualified TSN.XML.Odds as Odds ( dtd, pickle_message ) import qualified TSN.XML.Scores as Scores ( dtd, pickle_message ) +import qualified TSN.XML.SportInfo as SportInfo ( dtds ) import qualified TSN.XML.Weather as Weather ( dtd, pickle_message ) import Xml ( DtdName(..), parse_opts ) @@ -187,6 +189,10 @@ | dtd == Weather.dtd = do let m = unpickleDoc Weather.pickle_message xml maybe (return $ ImportFailed errmsg) migrate_and_import m + + | dtd `elem` GameInfo.dtds = undefined + + | dtd `elem` SportInfo.dtds = undefined | otherwise = do let infomsg = diff --git a/src/TSN/Picklers.hs b/src/TSN/Picklers.hs --- a/src/TSN/Picklers.hs +++ b/src/TSN/Picklers.hs @@ -132,7 +132,7 @@ (to_time_stamp, from_time_stamp) `xpWrapMaybe` xpText where -- This omits the timezone and trailing space. - format = " %B %-d, %Y, at " ++ xp_time_format ++ " ET " + format = "%B %-d, %Y, at " ++ xp_time_format ++ " ET" five_hours :: NominalDiffTime five_hours = 5 * 60 * 60 diff --git a/src/TSN/XML/AutoRacingSchedule.hs b/src/TSN/XML/AutoRacingSchedule.hs --- a/src/TSN/XML/AutoRacingSchedule.hs +++ b/src/TSN/XML/AutoRacingSchedule.hs @@ -29,7 +29,7 @@ import Data.Tuple.Curry ( uncurryN ) import Database.Groundhog ( countAll, - executeRaw, + deleteAll, migrate, runMigration, silentMigrationLogger ) @@ -555,8 +555,7 @@ migrate b migrate c _ <- dbimport sched - -- No idea how 'delete' works, so do this instead. - executeRaw False "DELETE FROM auto_racing_schedules;" [] + deleteAll a count_a <- countAll a count_b <- countAll b count_c <- countAll c diff --git a/src/TSN/XML/GameInfo.hs b/src/TSN/XML/GameInfo.hs new file mode 100644 --- /dev/null +++ b/src/TSN/XML/GameInfo.hs @@ -0,0 +1,39 @@ +-- | GameInfo represents a collection of DTDs that we don't really +-- handle but want to make available. The raw XML gets stored in the +-- database along with the XML_File_ID, but we don't parse any of it. +-- +-- See also: TSN.XML.SportInfo +-- +module TSN.XML.GameInfo ( + dtds ) +where + +-- | The DTDs for everything that we consider "Game Info." +-- +-- TODO: This is the list from the old implementation. We need to +-- make sure that we are really receiving XML for these DTDs +-- (i.e. the names are correct). +-- +dtds :: [String] +dtds = + [ "CBASK_Lineup_XML.dtd", + "cbaskpreviewxml.dtd", + "cflpreviewxml.dtd", + "Matchup_NBA_NHL_XML.dtd", + "mlbpreviewxml.dtd", + "MLB_Gaming_Matchup_XML.dtd", + "MLB.dtd", + "MLB_Lineup_XML.dtd", + "MLB_Matchup_XML.dtd", + "MLS_Preview_XML.dtd", + "NBA_Gaming_Matchup_XML.dtd", + "NBA.dtd", + "NBA_Playoff_Matchup_XML.dtd", + "NBALineupXML.dtd", + "nbapreviewxml.dtd", + "NCAA_FB_Preview_XML.dtd", + "nflpreviewxml.dtd", + "NFL_NCAA_FB_Matchup_XML.dtd", + "nhlpreviewxml.dtd", + "recapxml.dtd", + "WorldBaseballPreviewXML.dtd" ] diff --git a/src/TSN/XML/Injuries.hs b/src/TSN/XML/Injuries.hs --- a/src/TSN/XML/Injuries.hs +++ b/src/TSN/XML/Injuries.hs @@ -32,7 +32,7 @@ import Data.Typeable ( Typeable ) import Database.Groundhog ( countAll, - executeRaw, + deleteAll, migrate, runMigration, silentMigrationLogger ) @@ -346,8 +346,7 @@ migrate a migrate b _ <- dbimport inj - -- No idea how 'delete' works, so do this instead. - executeRaw False "DELETE FROM injuries;" [] + deleteAll a count_a <- countAll a count_b <- countAll b return $ count_a + count_b diff --git a/src/TSN/XML/InjuriesDetail.hs b/src/TSN/XML/InjuriesDetail.hs --- a/src/TSN/XML/InjuriesDetail.hs +++ b/src/TSN/XML/InjuriesDetail.hs @@ -30,7 +30,7 @@ import Database.Groundhog ( DefaultKey, countAll, - executeRaw, + deleteAll, migrate, runMigration, silentMigrationLogger ) @@ -463,8 +463,7 @@ migrate b migrate c _ <- dbimport inj - -- No idea how 'delete' works, so do this instead. - executeRaw False "DELETE FROM injuries_detail;" [] + deleteAll a count_a <- countAll a count_b <- countAll b count_c <- countAll c diff --git a/src/TSN/XML/News.hs b/src/TSN/XML/News.hs --- a/src/TSN/XML/News.hs +++ b/src/TSN/XML/News.hs @@ -31,7 +31,7 @@ import Data.Typeable ( Typeable ) import Database.Groundhog ( countAll, - executeRaw, + deleteAll, insert_, migrate, runMigration, @@ -530,8 +530,7 @@ migrate d migrate e _ <- dbimport news - -- No idea how 'delete' works, so do this instead. - executeRaw False "DELETE FROM news;" [] + deleteAll a count_a <- countAll a count_b <- countAll b count_c <- countAll c diff --git a/src/TSN/XML/Odds.hs b/src/TSN/XML/Odds.hs --- a/src/TSN/XML/Odds.hs +++ b/src/TSN/XML/Odds.hs @@ -33,7 +33,7 @@ (=.), (==.), countAll, - executeRaw, + deleteAll, insert_, migrate, runMigration, @@ -837,8 +837,7 @@ migrate e migrate f _ <- dbimport odds - -- No idea how 'delete' works, so do this instead. - executeRaw False "DELETE FROM odds;" [] + deleteAll a count_a <- countAll a count_b <- countAll b count_c <- countAll c diff --git a/src/TSN/XML/Scores.hs b/src/TSN/XML/Scores.hs --- a/src/TSN/XML/Scores.hs +++ b/src/TSN/XML/Scores.hs @@ -30,7 +30,7 @@ import Data.Typeable ( Typeable ) import Database.Groundhog ( countAll, - executeRaw, + deleteAll, insert, insert_, migrate, @@ -618,7 +618,7 @@ migrate f _ <- dbimport score -- No idea how 'delete' works, so do this instead. - executeRaw False "DELETE FROM scores;" [] + deleteAll a count_a <- countAll a count_b <- countAll b count_c <- countAll c diff --git a/src/TSN/XML/SportInfo.hs b/src/TSN/XML/SportInfo.hs new file mode 100644 --- /dev/null +++ b/src/TSN/XML/SportInfo.hs @@ -0,0 +1,167 @@ +-- | SportInfo represents a collection of DTDs that we don't really +-- handle but want to make available. The raw XML gets stored in the +-- database along with the XML_File_ID, but we don't parse any of it. +-- +-- See also: TSN.XML.GameInfo +-- +module TSN.XML.SportInfo ( + dtds ) +where + +-- | The DTDs for everything that we consider "Sport Info." +-- +-- TODO: This is the list from the old implementation. We need to +-- make sure that we are really receiving XML for these DTDs +-- (i.e. the names are correct). +-- +dtds :: [String] +dtds = + [ "CBASK_3PPctXML.dtd", + "Cbask_All_Tourn_Teams_XML.dtd", + "CBASK_AssistsXML.dtd", + "Cbask_Awards_XML.dtd", + "CBASK_BlocksXML.dtd", + "CBask_BlocksXML.dtd", + "Cbask_Conf_Standings_XML.dtd", + "Cbask_DivII_III_Indv_Stats_XML.dtd", + "Cbask_DivII_Team_Stats_XML.dtd", + "Cbask_DivIII_Team_Stats_XML.dtd", + "CBASK_FGPctXML.dtd", + "CBask_FGPctXML.dtd", + "CBASK_FoulsXML.dtd", + "CBASK_FTPctXML.dtd", + "Cbask_Indv_No_Avg_XML.dtd", + "Cbask_Indv_Scoring_XML.dtd", + "Cbask_Indv_Shooting_XML.dtd", + "CBASK_MinutesXML.dtd", + "Cbask_Polls_XML.dtd", + "CBASK_ReboundsXML.dtd", + "CBASK_ScoringLeadersXML.dtd", + "CBASK_StealsXML.dtd", + "Cbask_Team_Scoring_Rebound_Margin_XML.dtd", + "Cbask_Team_Scoring_XML.dtd", + "Cbask_Team_Shooting_Pct_XML.dtd", + "Cbask_Team_ThreePT_Made_XML.dtd", + "Cbask_Team_ThreePT_PCT_XML.dtd", + "Cbask_Team_Win_Pct_XML.dtd", + "Cbask_Top_Twenty_Five_XML.dtd", + "CBASK_TopTwentyFiveResult_XML.dtd", + "Cbask_Tourn_Awards_XML.dtd", + "Cbask_Tourn_Champs_XML.dtd", + "Cbask_Tourn_Indiv_XML.dtd", + "Cbask_Tourn_Leaders_XML.dtd", + "Cbask_Tourn_MVP_XML.dtd", + "Cbask_Tourn_Records_XML.dtd", + "LeagueScheduleXML.dtd", + "minorscoresxml.dtd", + "Minor_Baseball_League_Leaders_XML.dtd", + "Minor_Baseball_Standings_XML.dtd", + "Minor_Baseball_Transactions_XML.dtd", + "mlbbattingavgxml.dtd", + "mlbdoublesleadersxml.dtd", + "MLBGamesPlayedXML.dtd", + "MLBGIDPXML.dtd", + "MLBHitByPitchXML.dtd", + "mlbhitsleadersxml.dtd", + "mlbhomerunsxml.dtd", + "MLBHRFreqXML.dtd", + "MLBIntWalksXML.dtd", + "MLBKORateXML.dtd", + "mlbonbasepctxml.dtd", + "MLBOPSXML.dtd", + "MLBPlateAppsXML.dtd", + "mlbrbisxml.dtd", + "mlbrunsleadersxml.dtd", + "MLBSacFliesXML.dtd", + "MLBSacrificesXML.dtd", + "MLBSBSuccessXML.dtd", + "mlbsluggingpctxml.dtd", + "mlbstandxml.dtd", + "mlbstandxml_preseason.dtd", + "mlbstolenbasexml.dtd", + "mlbtotalbasesleadersxml.dtd", + "mlbtriplesleadersxml.dtd", + "MLBWalkRateXML.dtd", + "mlbwalksleadersxml.dtd", + "MLBXtraBaseHitsXML.dtd", + "MLB_ERA_Leaders.dtd", + "MLB_Fielding_XML.dtd", + "MLB_Pitching_Appearances_Leaders.dtd", + "MLB_Pitching_Balks_Leaders.dtd", + "MLB_Pitching_CG_Leaders.dtd", + "MLB_Pitching_ER_Allowed_Leaders.dtd", + "MLB_Pitching_Hits_Allowed_Leaders.dtd", + "MLB_Pitching_Hit_Batters_Leaders.dtd", + "MLB_Pitching_HR_Allowed_Leaders.dtd", + "MLB_Pitching_IP_Leaders.dtd", + "MLB_Pitching_Runs_Allowed_Leaders.dtd", + "MLB_Pitching_Saves_Leaders.dtd", + "MLB_Pitching_Shut_Outs_Leaders.dtd", + "MLB_Pitching_Starts_Leaders.dtd", + "MLB_Pitching_Strike_Outs_Leaders.dtd", + "MLB_Pitching_Walks_Leaders.dtd", + "MLB_Pitching_WHIP_Leaders.dtd", + "MLB_Pitching_Wild_Pitches_Leaders.dtd", + "MLB_Pitching_Win_Percentage_Leaders.dtd", + "MLB_Pitching_WL_Leaders.dtd", + "NBA_Team_Stats_XML.dtd", + "NBA3PPctXML.dtd", + "NBAAssistsXML.dtd", + "NBABlocksXML.dtd", + "nbaconfrecxml.dtd", + "nbadaysxml.dtd", + "nbadivisionsxml.dtd", + "NBAFGPctXML.dtd", + "NBAFoulsXML.dtd", + "NBAFTPctXML.dtd", + "NBAMinutesXML.dtd", + "NBAReboundsXML.dtd", + "NBAScorersXML.dtd", + "nbastandxml.dtd", + "NBAStealsXML.dtd", + "nbateamleadersxml.dtd", + "nbatripledoublexml.dtd", + "NBATurnoversXML.dtd", + "NCAA_Conference_Schedule_XML.dtd", + "nflfirstdownxml.dtd", + "NFLFumbleLeaderXML.dtd", + "NFLGiveTakeXML.dtd", + "NFLGrassTurfDomeOutsideXML.dtd", + "NFLInside20XML.dtd", + "NFLInterceptionLeadersXML.dtd", + "NFLKickoffsXML.dtd", + "NFLMondayNightXML.dtd", + "NFLPassingLeadersXML.dtd", + "NFLPassLeadXML.dtd", + "NFLQBStartsXML.dtd", + "NFLReceivingLeadersXML.dtd", + "NFLRushingLeadersXML.dtd", + "NFLSackLeadersXML.dtd", + "nflstandxml.dtd", + "NFLTackleFFLeadersXML.dtd", + "NFLTeamRankingsXML.dtd", + "NFLTopKickoffReturnXML.dtd", + "NFLTopPerformanceXML.dtd", + "NFLTopPuntReturnXML.dtd", + "NFLTotalYardageXML.dtd", + "NFLYardsXML.dtd", + "NFL_KickingLeaders_XML.dtd", + "NFL_NBA_Draft_XML.dtd", + "NFL_PuntingLeaders_XML.dtd", + "NFL_Roster_XML.dtd", + "NFL_Team_Stats_XML.dtd", + "Transactions_XML.dtd", + "Weekly_Sched_XML.dtd", + "WNBA_Team_Leaders_XML.dtd", + "WNBA3PPctXML.dtd", + "WNBAAssistsXML.dtd", + "WNBABlocksXML.dtd", + "WNBAFGPctXML.dtd", + "WNBAFoulsXML.dtd", + "WNBAFTPctXML.dtd", + "WNBAMinutesXML.dtd", + "WNBAReboundsXML.dtd", + "WNBAScorersXML.dtd", + "wnbastandxml.dtd", + "WNBAStealsXML.dtd", + "WNBATurnoversXML.dtd" ] diff --git a/src/TSN/XML/Weather.hs b/src/TSN/XML/Weather.hs --- a/src/TSN/XML/Weather.hs +++ b/src/TSN/XML/Weather.hs @@ -26,7 +26,7 @@ import Data.Tuple.Curry ( uncurryN ) import Database.Groundhog ( countAll, - executeRaw, + deleteAll, migrate, runMigration, silentMigrationLogger ) @@ -412,8 +412,7 @@ migrate b migrate c _ <- dbimport weather - -- No idea how 'delete' works, so do this instead. - executeRaw False "DELETE FROM weather;" [] + deleteAll a count_a <- countAll a count_b <- countAll b count_c <- countAll c diff --git a/test/ShellTests.hs b/test/ShellTests.hs --- a/test/ShellTests.hs +++ b/test/ShellTests.hs @@ -1,7 +1,7 @@ module Main where -import System.Cmd ( system ) +import System.Process ( system ) import System.Exit ( exitWith ) main :: IO ()