diff --git a/doc/dbschema/scoresxml.png b/doc/dbschema/scoresxml.png Binary files a/doc/dbschema/scoresxml.png and b/doc/dbschema/scoresxml.png differ diff --git a/htsn-import.cabal b/htsn-import.cabal --- a/htsn-import.cabal +++ b/htsn-import.cabal @@ -1,5 +1,5 @@ name: htsn-import -version: 0.1.2 +version: 0.1.3 cabal-version: >= 1.8 author: Michael Orlitzky maintainer: Michael Orlitzky diff --git a/src/TSN/XML/AutoRacingResults.hs b/src/TSN/XML/AutoRacingResults.hs --- a/src/TSN/XML/AutoRacingResults.hs +++ b/src/TSN/XML/AutoRacingResults.hs @@ -24,6 +24,7 @@ -- System imports. import Control.Monad ( forM_ ) import Data.Data ( Data ) +import Data.Maybe ( fromMaybe ) import Data.Time ( UTCTime(..) ) import Data.Tuple.Curry ( uncurryN ) import Data.Typeable ( Typeable ) @@ -362,9 +363,9 @@ -- the database with an (embedded) 'MostLapsLeading' with three -- missing fields. most_laps_leading = - case xml_most_laps_leading of - Just mll -> mll - Nothing -> MostLapsLeading Nothing Nothing Nothing + fromMaybe (MostLapsLeading Nothing Nothing Nothing) + xml_most_laps_leading + -- | This allows us to insert the XML representation -- 'AutoRacingResultsRaceInformationXml' directly. diff --git a/src/TSN/XML/Odds.hs b/src/TSN/XML/Odds.hs --- a/src/TSN/XML/Odds.hs +++ b/src/TSN/XML/Odds.hs @@ -61,8 +61,8 @@ xpWrap ) -- Local imports. -import TSN.Codegen ( - tsn_codegen_config ) +import TSN.Codegen ( tsn_codegen_config ) +import TSN.Database ( insert_or_select ) import TSN.DbImport ( DbImport(..), ImportResult(..), run_dbmigrate ) import TSN.Picklers ( xp_date_padded, xp_tba_time, xp_time_stamp ) import TSN.Team ( FromXmlFkTeams(..), Team(..) ) @@ -108,10 +108,18 @@ -- with a 'String' and then attempt to 'read' a 'Double' later when we -- go to insert the thing. -- +-- The client_id and name shouldn't really be optional, but TSN has +-- started to send us empty casinos: +-- +-- \\ +-- +-- We need to parse these, but we'll silently drop them during the +-- database import. +-- data OddsGameCasinoXml = OddsGameCasinoXml { - xml_casino_client_id :: Int, - xml_casino_name :: String, + xml_casino_client_id :: Maybe Int, + xml_casino_name :: Maybe String, xml_casino_line :: Maybe String } deriving (Eq, Show) @@ -126,24 +134,25 @@ instance ToDb OddsGameCasinoXml where -- | The database representation of an 'OddsGameCasinoXml' is an - -- 'OddsCasino'. + -- 'OddsCasino'. When our XML representation is missing a + -- client_id or a name, we want to ignore it. So in that case, + -- when we convert to the database type, we want 'Nothing'. -- - type Db OddsGameCasinoXml = OddsCasino + type Db OddsGameCasinoXml = Maybe OddsCasino instance FromXml OddsGameCasinoXml where - -- | We convert from XML to the database by dropping the line field. + -- | We convert from XML to the database by dropping the + -- 'xml_casino_line' field. If either the 'xml_casino_client_id' + -- or 'xml_casino_name' is missing ('Nothing'), we'll return + -- 'Nothing'. -- - from_xml OddsGameCasinoXml{..} = - OddsCasino { - casino_client_id = xml_casino_client_id, - casino_name = xml_casino_name } + from_xml (OddsGameCasinoXml Nothing _ _) = Nothing + from_xml (OddsGameCasinoXml _ Nothing _) = Nothing + from_xml (OddsGameCasinoXml (Just c) (Just n) _) = + Just OddsCasino { casino_client_id = c, casino_name = n } --- | This allows us to insert the XML representation 'OddsGameCasinoXml' --- directly. --- -instance XmlImport OddsGameCasinoXml -- * OddsGameTeamXml / OddsGameTeamStarterXml @@ -514,46 +523,75 @@ -- Finally, we insert the lines. The over/under entries for this -- game and the lines for the casinos all wind up in the same -- table, odds_games_lines. We can insert the over/under entries - -- freely with empty away/home lines: - forM_ (xml_over_under_casinos game) $ \c -> do - -- Start by inderting the casino. - ou_casino_id <- insert_xml_or_select c + -- freely with empty away/home lines. + -- + -- Before we continue, we drop all casinos that are missing + -- either a client_id or name field. + -- + let ou_casinos = filter nonempty_casino $ xml_over_under_casinos game - -- Now add the over/under entry with the casino's id. - let ogl = OddsGameLine { - ogl_odds_games_id = game_id, - ogl_odds_casinos_id = ou_casino_id, - ogl_over_under = (xml_casino_line c), - ogl_away_line = Nothing, - ogl_home_line = Nothing } + forM_ ou_casinos $ \c -> + -- Since we already filtered out the casinos without a + -- client_id or a name, the database conversion should always + -- return (Just something). + case (from_xml c) of + Nothing -> return () -- Should never happen, we filtered them out. + Just casino -> do + -- Start by inserting the casino. + ou_casino_id <- insert_or_select casino - insert_ ogl + -- Now add the over/under entry with the casino's id. + let ogl = OddsGameLine { + ogl_odds_games_id = game_id, + ogl_odds_casinos_id = ou_casino_id, + ogl_over_under = (xml_casino_line c), + ogl_away_line = Nothing, + ogl_home_line = Nothing } + insert_ ogl + -- ...but then when we insert the home/away team lines, we -- prefer to update the existing entry rather than overwrite it -- or add a new record. - forM_ (xml_team_casinos $ xml_away_team game) $ \c -> do - -- insert, or more likely retrieve the existing, casino - a_casino_id <- insert_xml_or_select c + let away_casinos = filter nonempty_casino $ + xml_team_casinos (xml_away_team game) - -- Get a Maybe Double instead of the Maybe String that's in there. - let away_line = home_away_line c + forM_ away_casinos $ \c -> + case (from_xml c) of + Nothing -> return () -- Should never happen, we filtered them out. + Just casino -> do + -- insert, or more likely retrieve the existing, casino + a_casino_id <- insert_or_select casino - -- Unconditionally update that casino's away team line with ours. - update [Ogl_Away_Line =. away_line] $ -- WHERE - Ogl_Odds_Casinos_Id ==. a_casino_id + -- Get a Maybe Double instead of the Maybe String that's in there. + let away_line = home_away_line c + -- Unconditionally update that casino's away team line with ours. + update [Ogl_Away_Line =. away_line] $ -- WHERE + Ogl_Odds_Casinos_Id ==. a_casino_id + -- Repeat all that for the home team. - forM_ (xml_team_casinos $ xml_home_team game) $ \c ->do - h_casino_id <- insert_xml_or_select c - let home_line = home_away_line c - update [Ogl_Home_Line =. home_line] $ -- WHERE - Ogl_Odds_Casinos_Id ==. h_casino_id + let home_casinos = filter nonempty_casino $ + xml_team_casinos (xml_home_team game) + forM_ home_casinos $ \c -> + case (from_xml c) of + Nothing -> return () -- Should never happen, we filtered them out. + Just casino -> do + h_casino_id <- insert_or_select casino + let home_line = home_away_line c + update [Ogl_Home_Line =. home_line] $ -- WHERE + Ogl_Odds_Casinos_Id ==. h_casino_id + return game_id return ImportSucceeded + where + nonempty_casino :: OddsGameCasinoXml -> Bool + nonempty_casino (OddsGameCasinoXml Nothing _ _) = False + nonempty_casino (OddsGameCasinoXml _ Nothing _) = False + nonempty_casino _ = True -- -- Pickling @@ -580,8 +618,8 @@ xpElem "Casino" $ xpWrap (from_tuple, to_tuple) $ xpTriple - (xpAttr "ClientID" xpInt) - (xpAttr "Name" xpText) + (xpAttr "ClientID" $ xpOption xpInt) + (xpAttr "Name" $ xpOption xpText) (xpOption xpText) where from_tuple = uncurryN OddsGameCasinoXml @@ -769,7 +807,10 @@ "test/xml/Odds_XML-missing-starters.xml", check "pickle composed with unpickle is the identity (TBA game time)" - "test/xml/Odds_XML-tba-game-time.xml"] + "test/xml/Odds_XML-tba-game-time.xml", + + check "pickle composed with unpickle is the identity (empty casino)" + "test/xml/Odds_XML-empty-casino.xml" ] where check desc path = testCase desc $ do (expected, actual) <- pickle_unpickle pickle_message path @@ -799,7 +840,10 @@ "test/xml/Odds_XML-missing-starters.xml", check "unpickling succeeds (TBA game time)" - "test/xml/Odds_XML-tba-game-time.xml" ] + "test/xml/Odds_XML-tba-game-time.xml", + + check "unpickling succeeds (empty casino)" + "test/xml/Odds_XML-empty-casino.xml" ] where check desc path = testCase desc $ do actual <- unpickleable path pickle_message @@ -842,6 +886,10 @@ check "deleting odds deleted its children (TBA game time)" "test/xml/Odds_XML-tba-game-time.xml" 119 -- 5 casinos, 114 teams + , + check "deleting odds deleted its children (empty casino)" + "test/xml/Odds_XML-empty-casino.xml" + 11 -- 5 casinos, 6 teams ] where check desc path expected = testCase desc $ do diff --git a/src/TSN/XML/Scores.hs b/src/TSN/XML/Scores.hs --- a/src/TSN/XML/Scores.hs +++ b/src/TSN/XML/Scores.hs @@ -159,7 +159,7 @@ -- data ScoreGameStatus = ScoreGameStatus { - db_status_numeral :: Int, + db_status_numeral :: Maybe Int, db_status_type :: Maybe String, -- ^ These are probably only one-character, -- long, but they all take the same -- amount of space in Postgres. @@ -431,20 +431,26 @@ --- | Convert a 'ScoreGameStatus' to/from \. +-- | Convert a 'ScoreGameStatus' to/from \. The \"type\" +-- attribute can be either missing or empty, so we're really parsing +-- a double-Maybe here. We use the monad join to collapse it into +-- one. See also: the hteam/vteam picklers. -- pickle_status :: PU ScoreGameStatus pickle_status = xpElem "status" $ xpWrap (from_tuple, to_tuple) $ - xpTriple (xpAttr "numeral" xpInt) - (xpOption $ xpAttr "type" xpText) + xpTriple (xpAttr "numeral" $ xpOption xpInt) + (xpOption $ xpAttr "type" $ xpOption xpText) xpText where - from_tuple = uncurryN ScoreGameStatus - to_tuple ScoreGameStatus{..} = (db_status_numeral, - db_status_type, - db_status_text) + from_tuple (x,y,z) = ScoreGameStatus x (join y) z + to_tuple ScoreGameStatus{..} = + (db_status_numeral, s, db_status_text) + where + s = case db_status_type of + Nothing -> Nothing + Just _ -> Just db_status_type -- | Convert a 'ScoreGameXml' to/from \. @@ -552,7 +558,13 @@ "test/xml/scoresxml-no-locations.xml", check "pickle composed with unpickle is the identity (pitcher, no type)" - "test/xml/scoresxml-pitcher-no-type.xml"] + "test/xml/scoresxml-pitcher-no-type.xml", + + check "pickle composed with unpickle is the identity (empty numeral)" + "test/xml/scoresxml-empty-numeral.xml", + + check "pickle composed with unpickle is the identity (empty type)" + "test/xml/scoresxml-empty-type.xml" ] where check desc path = testCase desc $ do (expected, actual) <- pickle_unpickle pickle_message path @@ -570,7 +582,13 @@ "test/xml/scoresxml-no-locations.xml", check "unpickling succeeds (pitcher, no type)" - "test/xml/scoresxml-pitcher-no-type.xml" ] + "test/xml/scoresxml-pitcher-no-type.xml", + + check "unpickling succeeds (empty numeral)" + "test/xml/scoresxml-empty-numeral.xml", + + check "unpickling succeeds (empty type)" + "test/xml/scoresxml-empty-type.xml" ] where check desc path = testCase desc $ do actual <- unpickleable path pickle_message @@ -593,7 +611,15 @@ check "unpickling succeeds (pitcher, no type)" "test/xml/scoresxml-pitcher-no-type.xml" - 3 -- 2 teams, 1 location + 3, -- 2 teams, 1 location + + check "unpickling succeeds (empty numeral)" + "test/xml/scoresxml-empty-numeral.xml" + 3, -- 2 teams, 1 location + + check "unpickling succeeds (empty type)" + "test/xml/scoresxml-empty-type.xml" + 4 -- 2 teams, 2 locations ] where check desc path expected = testCase desc $ do diff --git a/test/shell/import-duplicates.test b/test/shell/import-duplicates.test --- a/test/shell/import-duplicates.test +++ b/test/shell/import-duplicates.test @@ -16,7 +16,7 @@ # and a newsxml that aren't really supposed to import. find ./test/xml -maxdepth 1 -name '*.xml' | wc -l >>> -35 +38 >>>= 0 # Run the imports again; we should get complaints about the duplicate @@ -24,7 +24,7 @@ # occurrences of the string 'ERROR'. ./dist/build/htsn-import/htsn-import -c 'shelltest.sqlite3' test/xml/*.xml 2>&1 | grep ERROR | wc -l >>> -62 +68 >>>= 0 # Finally, clean up after ourselves. diff --git a/test/xml/Odds_XML-empty-casino.xml b/test/xml/Odds_XML-empty-casino.xml new file mode 100644 --- /dev/null +++ b/test/xml/Odds_XML-empty-casino.xml @@ -0,0 +1,1 @@ + 21985005 ADO;NHL-ODDS Odds NHL Las Vegas National Hockey League Line Current Line as of 12:30 P.M. ET 19287 10/15/2014 08:00 PM 123 051 CAL Calgary +225 +230 +240 +220 +225 124 052 CHI Chicago -265 -270 -280 -260 -265 5.5p 5.5u 5.5p 5.5p 5.5p 19285 10/15/2014 08:00 PM 121 053 BOS Boston -120 -120 -125 -115 -120 125 054 DET Detroit even even +105 -105 even 5p 5u 5p 5p 5p 19289 10/15/2014 10:30 PM 126 055 EDM Edmonton +160 +160 +165 +165 +165 141 056 ARI Arizona -185 -190 -190 -185 -185 5.5u 5.5u 5.5u 5.5u 5.5u October 15, 2014, at 12:27 PM ET diff --git a/test/xml/scoresxml-empty-numeral.xml b/test/xml/scoresxml-empty-numeral.xml new file mode 100644 --- /dev/null +++ b/test/xml/scoresxml-empty-numeral.xml @@ -0,0 +1,1 @@ + 21984547 BC-ACF+109:104*11712999 Scores NBA Sacramento CA USA Sacramento Brooklyn 117 129 FINAL October 15, 2014, at 10:17 AM ET diff --git a/test/xml/scoresxml-empty-type.xml b/test/xml/scoresxml-empty-type.xml new file mode 100644 --- /dev/null +++ b/test/xml/scoresxml-empty-type.xml @@ -0,0 +1,1 @@ + 21986921 BC-ADP+126:141* 4 6 3R 1:05 19289 19289 Scores NHL Edmonton ALBERTA CANADA Arizona AZ USA Regular Edmonton Arizona 4 6 1:05 3 Period Lauri Korpikoski slap shot is over the net. October 16, 2014, at 01:27 AM ET