htsn-import 0.1.2 → 0.1.3

raw patch · 9 files changed

+139/−61 lines, 9 files

Files

doc/dbschema/scoresxml.png binary
htsn-import.cabal +1/−1
src/TSN/XML/AutoRacingResults.hs +4/−3
src/TSN/XML/Odds.hs +92/−44
src/TSN/XML/Scores.hs +37/−11
test/shell/import-duplicates.test +2/−2
test/xml/Odds_XML-empty-casino.xml +1/−0
test/xml/scoresxml-empty-numeral.xml +1/−0
test/xml/scoresxml-empty-type.xml +1/−0

doc/dbschema/scoresxml.png view

binary file changed (123239 → 123044 bytes)

htsn-import.cabal view

@@ -1,5 +1,5 @@ name:           htsn-import-version:        0.1.2+version:        0.1.3 cabal-version:  >= 1.8 author:         Michael Orlitzky maintainer:	Michael Orlitzky <michael@orlitzky.com>

src/TSN/XML/AutoRacingResults.hs view

@@ -24,6 +24,7 @@ -- System imports. import Control.Monad ( forM_ ) import Data.Data ( Data )+import Data.Maybe ( fromMaybe ) import Data.Time ( UTCTime(..) ) import Data.Tuple.Curry ( uncurryN ) import Data.Typeable ( Typeable )@@ -362,9 +363,9 @@       -- the database with an (embedded) 'MostLapsLeading' with three       -- missing fields.       most_laps_leading =-        case xml_most_laps_leading of-          Just mll -> mll-          Nothing  -> MostLapsLeading Nothing Nothing Nothing+        fromMaybe (MostLapsLeading Nothing Nothing Nothing)+                  xml_most_laps_leading+  -- | This allows us to insert the XML representation --   'AutoRacingResultsRaceInformationXml' directly.

src/TSN/XML/Odds.hs view

@@ -61,8 +61,8 @@   xpWrap )  -- Local imports.-import TSN.Codegen (-  tsn_codegen_config )+import TSN.Codegen ( tsn_codegen_config )+import TSN.Database ( insert_or_select ) import TSN.DbImport ( DbImport(..), ImportResult(..), run_dbmigrate ) import TSN.Picklers ( xp_date_padded, xp_tba_time, xp_time_stamp ) import TSN.Team ( FromXmlFkTeams(..), Team(..) )@@ -108,10 +108,18 @@ --   with a 'String' and then attempt to 'read' a 'Double' later when we --   go to insert the thing. --+--   The client_id and name shouldn't really be optional, but TSN has+--   started to send us empty casinos:+--+--     \<Casino ClientID=\"\" Name=\"\"\>\</Casino\>+--+--   We need to parse these, but we'll silently drop them during the+--   database import.+-- data OddsGameCasinoXml =   OddsGameCasinoXml {-    xml_casino_client_id :: Int,-    xml_casino_name      :: String,+    xml_casino_client_id :: Maybe Int,+    xml_casino_name      :: Maybe String,     xml_casino_line      :: Maybe String }   deriving (Eq, Show) @@ -126,24 +134,25 @@  instance ToDb OddsGameCasinoXml where   -- | The database representation of an 'OddsGameCasinoXml' is an-  --   'OddsCasino'.+  --   'OddsCasino'. When our XML representation is missing a+  --   client_id or a name, we want to ignore it. So in that case,+  --   when we convert to the database type, we want 'Nothing'.   ---  type Db OddsGameCasinoXml = OddsCasino+  type Db OddsGameCasinoXml = Maybe OddsCasino   instance FromXml OddsGameCasinoXml where-  -- | We convert from XML to the database by dropping the line field.+  -- | We convert from XML to the database by dropping the+  --   'xml_casino_line' field. If either the 'xml_casino_client_id'+  --   or 'xml_casino_name' is missing ('Nothing'), we'll return+  --   'Nothing'.   ---  from_xml OddsGameCasinoXml{..} =-    OddsCasino {-      casino_client_id = xml_casino_client_id,-      casino_name      = xml_casino_name }+  from_xml (OddsGameCasinoXml Nothing _ _) = Nothing+  from_xml (OddsGameCasinoXml _ Nothing _) = Nothing +  from_xml (OddsGameCasinoXml (Just c) (Just n) _) =+    Just OddsCasino { casino_client_id = c, casino_name = n } --- | This allows us to insert the XML representation 'OddsGameCasinoXml'---   directly.----instance XmlImport OddsGameCasinoXml   -- * OddsGameTeamXml / OddsGameTeamStarterXml@@ -514,46 +523,75 @@       -- Finally, we insert the lines. The over/under entries for this       -- game and the lines for the casinos all wind up in the same       -- table, odds_games_lines. We can insert the over/under entries-      -- freely with empty away/home lines:-      forM_ (xml_over_under_casinos game) $ \c -> do-        -- Start by inderting the casino.-        ou_casino_id <- insert_xml_or_select c+      -- freely with empty away/home lines.+      --+      -- Before we continue, we drop all casinos that are missing+      -- either a client_id or name field.+      --+      let ou_casinos = filter nonempty_casino $ xml_over_under_casinos game -        -- Now add the over/under entry with the casino's id.-        let ogl = OddsGameLine {-                    ogl_odds_games_id = game_id,-                    ogl_odds_casinos_id = ou_casino_id,-                    ogl_over_under = (xml_casino_line c),-                    ogl_away_line = Nothing,-                    ogl_home_line = Nothing }+      forM_ ou_casinos $ \c ->+        -- Since we already filtered out the casinos without a+        -- client_id or a name, the database conversion should always+        -- return (Just something).+        case (from_xml c) of+          Nothing -> return () -- Should never happen, we filtered them out.+          Just casino -> do+            -- Start by inserting the casino.+            ou_casino_id <- insert_or_select casino -        insert_ ogl+            -- Now add the over/under entry with the casino's id.+            let ogl = OddsGameLine {+                        ogl_odds_games_id = game_id,+                        ogl_odds_casinos_id = ou_casino_id,+                        ogl_over_under = (xml_casino_line c),+                        ogl_away_line = Nothing,+                        ogl_home_line = Nothing } +            insert_ ogl+       -- ...but then when we insert the home/away team lines, we       -- prefer to update the existing entry rather than overwrite it       -- or add a new record.-      forM_ (xml_team_casinos $ xml_away_team game) $ \c -> do-        -- insert, or more likely retrieve the existing, casino-        a_casino_id <- insert_xml_or_select c+      let away_casinos = filter nonempty_casino $+                           xml_team_casinos (xml_away_team game) -        -- Get a Maybe Double instead of the Maybe String that's in there.-        let away_line = home_away_line c+      forM_ away_casinos $ \c ->+        case (from_xml c) of+          Nothing -> return () -- Should never happen, we filtered them out.+          Just casino -> do+            -- insert, or more likely retrieve the existing, casino+            a_casino_id <- insert_or_select casino -        -- Unconditionally update that casino's away team line with ours.-        update [Ogl_Away_Line =. away_line] $ -- WHERE-          Ogl_Odds_Casinos_Id ==. a_casino_id+            -- Get a Maybe Double instead of the Maybe String that's in there.+            let away_line = home_away_line c +            -- Unconditionally update that casino's away team line with ours.+            update [Ogl_Away_Line =. away_line] $ -- WHERE+              Ogl_Odds_Casinos_Id ==. a_casino_id+       -- Repeat all that for the home team.-      forM_ (xml_team_casinos $ xml_home_team game) $ \c ->do-        h_casino_id <- insert_xml_or_select c-        let home_line = home_away_line c-        update [Ogl_Home_Line =. home_line] $ -- WHERE-          Ogl_Odds_Casinos_Id ==. h_casino_id+      let home_casinos = filter nonempty_casino $+                           xml_team_casinos (xml_home_team game) +      forM_ home_casinos $ \c ->+        case (from_xml c) of+          Nothing -> return () -- Should never happen, we filtered them out.+          Just casino -> do+            h_casino_id <- insert_or_select casino+            let home_line = home_away_line c+            update [Ogl_Home_Line =. home_line] $ -- WHERE+              Ogl_Odds_Casinos_Id ==. h_casino_id+       return game_id      return ImportSucceeded +    where+      nonempty_casino :: OddsGameCasinoXml -> Bool+      nonempty_casino (OddsGameCasinoXml Nothing _ _) = False+      nonempty_casino (OddsGameCasinoXml _ Nothing _) = False+      nonempty_casino _ = True  -- -- Pickling@@ -580,8 +618,8 @@   xpElem "Casino" $   xpWrap (from_tuple, to_tuple) $   xpTriple-    (xpAttr "ClientID" xpInt)-    (xpAttr "Name" xpText)+    (xpAttr "ClientID" $ xpOption xpInt)+    (xpAttr "Name" $ xpOption xpText)     (xpOption xpText)   where     from_tuple = uncurryN OddsGameCasinoXml@@ -769,7 +807,10 @@           "test/xml/Odds_XML-missing-starters.xml",      check "pickle composed with unpickle is the identity (TBA game time)"-          "test/xml/Odds_XML-tba-game-time.xml"]+          "test/xml/Odds_XML-tba-game-time.xml",++    check "pickle composed with unpickle is the identity (empty casino)"+          "test/xml/Odds_XML-empty-casino.xml" ]   where     check desc path = testCase desc $ do       (expected, actual) <- pickle_unpickle pickle_message path@@ -799,7 +840,10 @@           "test/xml/Odds_XML-missing-starters.xml",      check "unpickling succeeds (TBA game time)"-          "test/xml/Odds_XML-tba-game-time.xml" ]+          "test/xml/Odds_XML-tba-game-time.xml",++    check "unpickling succeeds (empty casino)"+          "test/xml/Odds_XML-empty-casino.xml" ]   where     check desc path = testCase desc $ do       actual <- unpickleable path pickle_message@@ -842,6 +886,10 @@     check "deleting odds deleted its children (TBA game time)"           "test/xml/Odds_XML-tba-game-time.xml"           119 -- 5 casinos, 114 teams+    ,+    check "deleting odds deleted its children (empty casino)"+          "test/xml/Odds_XML-empty-casino.xml"+          11 -- 5 casinos, 6 teams     ]   where     check desc path expected = testCase desc $ do

src/TSN/XML/Scores.hs view

@@ -159,7 +159,7 @@ -- data ScoreGameStatus =   ScoreGameStatus {-    db_status_numeral :: Int,+    db_status_numeral :: Maybe Int,     db_status_type :: Maybe String, -- ^ These are probably only one-character,                                     --   long, but they all take the same                                     --    amount of space in Postgres.@@ -431,20 +431,26 @@   --- | Convert a 'ScoreGameStatus' to/from \<status\>.+-- | Convert a 'ScoreGameStatus' to/from \<status\>. The \"type\"+--   attribute can be either missing or empty, so we're really parsing+--   a double-Maybe here. We use the monad join to collapse it into+--   one. See also: the hteam/vteam picklers. -- pickle_status :: PU ScoreGameStatus pickle_status =   xpElem "status" $     xpWrap (from_tuple, to_tuple) $-      xpTriple (xpAttr "numeral" xpInt)-               (xpOption $ xpAttr "type" xpText)+      xpTriple (xpAttr "numeral" $ xpOption xpInt)+               (xpOption $ xpAttr "type" $ xpOption xpText)                xpText   where-    from_tuple = uncurryN ScoreGameStatus-    to_tuple ScoreGameStatus{..} = (db_status_numeral,-                                    db_status_type,-                                    db_status_text)+    from_tuple (x,y,z) = ScoreGameStatus x (join y) z+    to_tuple ScoreGameStatus{..} =+      (db_status_numeral, s, db_status_text)+      where+      s = case db_status_type of+            Nothing -> Nothing+            Just _  -> Just db_status_type   -- | Convert a 'ScoreGameXml' to/from \<game\>.@@ -552,7 +558,13 @@           "test/xml/scoresxml-no-locations.xml",      check "pickle composed with unpickle is the identity (pitcher, no type)"-          "test/xml/scoresxml-pitcher-no-type.xml"]+          "test/xml/scoresxml-pitcher-no-type.xml",++    check "pickle composed with unpickle is the identity (empty numeral)"+          "test/xml/scoresxml-empty-numeral.xml",++    check "pickle composed with unpickle is the identity (empty type)"+          "test/xml/scoresxml-empty-type.xml" ]   where     check desc path = testCase desc $ do       (expected, actual) <- pickle_unpickle pickle_message path@@ -570,7 +582,13 @@           "test/xml/scoresxml-no-locations.xml",      check "unpickling succeeds (pitcher, no type)"-          "test/xml/scoresxml-pitcher-no-type.xml" ]+          "test/xml/scoresxml-pitcher-no-type.xml",++    check "unpickling succeeds (empty numeral)"+          "test/xml/scoresxml-empty-numeral.xml",++    check "unpickling succeeds (empty type)"+          "test/xml/scoresxml-empty-type.xml" ]   where     check desc path = testCase desc $ do       actual <- unpickleable path pickle_message@@ -593,7 +611,15 @@      check "unpickling succeeds (pitcher, no type)"           "test/xml/scoresxml-pitcher-no-type.xml"-          3 -- 2 teams, 1 location+          3, -- 2 teams, 1 location++    check "unpickling succeeds (empty numeral)"+          "test/xml/scoresxml-empty-numeral.xml"+          3, -- 2 teams, 1 location++    check "unpickling succeeds (empty type)"+          "test/xml/scoresxml-empty-type.xml"+          4 -- 2 teams, 2 locations   ]   where     check desc path expected = testCase desc $ do

test/shell/import-duplicates.test view

@@ -16,7 +16,7 @@ # and a newsxml that aren't really supposed to import. find ./test/xml -maxdepth 1 -name '*.xml' | wc -l >>>-35+38 >>>= 0  # Run the imports again; we should get complaints about the duplicate@@ -24,7 +24,7 @@ # occurrences of the string 'ERROR'. ./dist/build/htsn-import/htsn-import -c 'shelltest.sqlite3' test/xml/*.xml 2>&1 | grep ERROR | wc -l >>>-62+68 >>>= 0  # Finally, clean up after ourselves.

+ test/xml/Odds_XML-empty-casino.xml view

@@ -0,0 +1,1 @@+<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD Statistics 1.0/EN" "Odds_XML.dtd">
<message>
<XML_File_ID>21985005</XML_File_ID>
<heading>ADO;NHL-ODDS</heading>
<category>Odds</category>
<sport>NHL</sport>
<Title>Las Vegas National Hockey League Line</Title>
<Line_Time>Current Line as of 12:30 P.M. ET</Line_Time>
<Game>
<GameID>19287</GameID>
<Game_Date>10/15/2014</Game_Date>
<Game_Time>08:00 PM</Game_Time>
<AwayTeam>
<AwayTeamID>123</AwayTeamID>
<AwayRotationNumber>051</AwayRotationNumber>
<AwayAbbr>CAL</AwayAbbr>
<AwayTeamName>Calgary</AwayTeamName>
<Casino ClientID="116" Name="5Dimes">+225</Casino>
<Casino ClientID="110" Name="Betus">+230</Casino>
<Casino ClientID="104" Name="BOVADA">+240</Casino>
<Casino ClientID="121" Name="Mirage">+220</Casino>
<Casino ClientID="127" Name="DonBest Consensus">+225</Casino>
<Casino ClientID="" Name=""></Casino>
</AwayTeam>
<HomeTeam>
<HomeTeamID>124</HomeTeamID>
<HomeRotationNumber>052</HomeRotationNumber>
<HomeAbbr>CHI</HomeAbbr>
<HomeTeamName>Chicago</HomeTeamName>
<Casino ClientID="116" Name="5Dimes">-265</Casino>
<Casino ClientID="110" Name="Betus">-270</Casino>
<Casino ClientID="104" Name="BOVADA">-280</Casino>
<Casino ClientID="121" Name="Mirage">-260</Casino>
<Casino ClientID="127" Name="DonBest Consensus">-265</Casino>
<Casino ClientID="" Name=""></Casino>
</HomeTeam>
<Over_Under>
<Casino ClientID="116" Name="5Dimes">5.5p</Casino>
<Casino ClientID="110" Name="Betus">5.5u</Casino>
<Casino ClientID="104" Name="BOVADA">5.5p</Casino>
<Casino ClientID="121" Name="Mirage">5.5p</Casino>
<Casino ClientID="127" Name="DonBest Consensus">5.5p</Casino>
<Casino ClientID="" Name=""></Casino>
</Over_Under>
</Game>
<Game>
<GameID>19285</GameID>
<Game_Date>10/15/2014</Game_Date>
<Game_Time>08:00 PM</Game_Time>
<AwayTeam>
<AwayTeamID>121</AwayTeamID>
<AwayRotationNumber>053</AwayRotationNumber>
<AwayAbbr>BOS</AwayAbbr>
<AwayTeamName>Boston</AwayTeamName>
<Casino ClientID="116" Name="5Dimes">-120</Casino>
<Casino ClientID="110" Name="Betus">-120</Casino>
<Casino ClientID="104" Name="BOVADA">-125</Casino>
<Casino ClientID="121" Name="Mirage">-115</Casino>
<Casino ClientID="127" Name="DonBest Consensus">-120</Casino>
<Casino ClientID="" Name=""></Casino>
</AwayTeam>
<HomeTeam>
<HomeTeamID>125</HomeTeamID>
<HomeRotationNumber>054</HomeRotationNumber>
<HomeAbbr>DET</HomeAbbr>
<HomeTeamName>Detroit</HomeTeamName>
<Casino ClientID="116" Name="5Dimes">even</Casino>
<Casino ClientID="110" Name="Betus">even</Casino>
<Casino ClientID="104" Name="BOVADA">+105</Casino>
<Casino ClientID="121" Name="Mirage">-105</Casino>
<Casino ClientID="127" Name="DonBest Consensus">even</Casino>
<Casino ClientID="" Name=""></Casino>
</HomeTeam>
<Over_Under>
<Casino ClientID="116" Name="5Dimes">5p</Casino>
<Casino ClientID="110" Name="Betus">5u</Casino>
<Casino ClientID="104" Name="BOVADA">5p</Casino>
<Casino ClientID="121" Name="Mirage">5p</Casino>
<Casino ClientID="127" Name="DonBest Consensus">5p</Casino>
<Casino ClientID="" Name=""></Casino>
</Over_Under>
</Game>
<Game>
<GameID>19289</GameID>
<Game_Date>10/15/2014</Game_Date>
<Game_Time>10:30 PM</Game_Time>
<AwayTeam>
<AwayTeamID>126</AwayTeamID>
<AwayRotationNumber>055</AwayRotationNumber>
<AwayAbbr>EDM</AwayAbbr>
<AwayTeamName>Edmonton</AwayTeamName>
<Casino ClientID="116" Name="5Dimes">+160</Casino>
<Casino ClientID="110" Name="Betus">+160</Casino>
<Casino ClientID="104" Name="BOVADA">+165</Casino>
<Casino ClientID="121" Name="Mirage">+165</Casino>
<Casino ClientID="127" Name="DonBest Consensus">+165</Casino>
<Casino ClientID="" Name=""></Casino>
</AwayTeam>
<HomeTeam>
<HomeTeamID>141</HomeTeamID>
<HomeRotationNumber>056</HomeRotationNumber>
<HomeAbbr>ARI</HomeAbbr>
<HomeTeamName>Arizona</HomeTeamName>
<Casino ClientID="116" Name="5Dimes">-185</Casino>
<Casino ClientID="110" Name="Betus">-190</Casino>
<Casino ClientID="104" Name="BOVADA">-190</Casino>
<Casino ClientID="121" Name="Mirage">-185</Casino>
<Casino ClientID="127" Name="DonBest Consensus">-185</Casino>
<Casino ClientID="" Name=""></Casino>
</HomeTeam>
<Over_Under>
<Casino ClientID="116" Name="5Dimes">5.5u</Casino>
<Casino ClientID="110" Name="Betus">5.5u</Casino>
<Casino ClientID="104" Name="BOVADA">5.5u</Casino>
<Casino ClientID="121" Name="Mirage">5.5u</Casino>
<Casino ClientID="127" Name="DonBest Consensus">5.5u</Casino>
<Casino ClientID="" Name=""></Casino>
</Over_Under>
</Game>
<time_stamp> October 15, 2014, at 12:27 PM ET </time_stamp>
</message>

+ test/xml/scoresxml-empty-numeral.xml view

@@ -0,0 +1,1 @@+<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD Scores 1.0/EN" "scoresxml.dtd">
<message>
<XML_File_ID>21984547</XML_File_ID>
<heading>BC-ACF+109:104*11712999</heading>
<game_id></game_id>
<schedule_id></schedule_id>
<category>Scores</category>
<sport>NBA</sport>
<location>
<city>Sacramento</city>
<state>CA</state>
<country>USA</country>
</location>
<seasontype></seasontype>
<game>
<vteam id="109">Sacramento</vteam>
<hteam id="104">Brooklyn</hteam>
<vscore>117</vscore>
<hscore>129</hscore>
<status numeral="" type="F">FINAL</status>
</game>
<time_stamp> October 15, 2014, at 10:17 AM ET </time_stamp>
</message>

+ test/xml/scoresxml-empty-type.xml view

@@ -0,0 +1,1 @@+<?xml version="1.0" standalone="no" ?>
<!DOCTYPE message PUBLIC "-//TSN//DTD Scores 1.0/EN" "scoresxml.dtd">
<message>
<XML_File_ID>21986921</XML_File_ID>
<heading>BC-ADP+126:141*  4  6 3R 1:05</heading>
<game_id>19289</game_id>
<schedule_id>19289</schedule_id>
<category>Scores</category>
<sport>NHL</sport>
<location>
<city>Edmonton</city>
<state>ALBERTA</state>
<country>CANADA</country>
</location>
<location>
<city>Arizona</city>
<state>AZ</state>
<country>USA</country>
</location>
<seasontype>Regular</seasontype>
<game>
<vteam id="126">Edmonton</vteam>
<hteam id="141">Arizona</hteam>
<vscore>4</vscore>
<hscore>6</hscore>
<time_r>1:05</time_r>
<status numeral="3" type="">3 Period</status>
<notes>Lauri Korpikoski slap shot is over the net.</notes>
</game>
<time_stamp> October 16, 2014, at 01:27 AM ET </time_stamp>
</message>