diff --git a/doc/dbschema/Odds_XML.png b/doc/dbschema/Odds_XML.png Binary files a/doc/dbschema/Odds_XML.png and b/doc/dbschema/Odds_XML.png differ diff --git a/doc/dbschema/jfilexml.png b/doc/dbschema/jfilexml.png Binary files a/doc/dbschema/jfilexml.png and b/doc/dbschema/jfilexml.png differ diff --git a/doc/dbschema/newsxml.png b/doc/dbschema/newsxml.png Binary files a/doc/dbschema/newsxml.png and b/doc/dbschema/newsxml.png differ diff --git a/htsn-import.cabal b/htsn-import.cabal --- a/htsn-import.cabal +++ b/htsn-import.cabal @@ -1,5 +1,5 @@ name: htsn-import -version: 0.1.1 +version: 0.1.2 cabal-version: >= 1.8 author: Michael Orlitzky maintainer: Michael Orlitzky diff --git a/src/TSN/XML/JFile.hs b/src/TSN/XML/JFile.hs --- a/src/TSN/XML/JFile.hs +++ b/src/TSN/XML/JFile.hs @@ -67,7 +67,7 @@ xp_date, xp_date_padded, xp_datetime, - xp_time, + xp_tba_time, xp_time_dots, xp_time_stamp ) import TSN.Team ( @@ -214,7 +214,7 @@ db_schedule_id :: Int, db_odds_info :: JFileGameOddsInfo, db_season_type :: Maybe String, - db_game_time :: UTCTime, + db_game_time :: Maybe UTCTime, db_vleague :: Maybe String, db_hleague :: Maybe String, db_vscore :: Int, @@ -236,7 +236,7 @@ xml_odds_info :: JFileGameOddsInfo, xml_season_type :: Maybe String, xml_game_date :: UTCTime, - xml_game_time :: UTCTime, + xml_game_time :: Maybe UTCTime, xml_vteam :: VTeam, xml_vleague :: Maybe String, xml_hteam :: HTeam, @@ -297,11 +297,15 @@ db_time_remaining = xml_time_remaining, db_game_status = xml_game_status } where - -- | Make the database \"game time\" from the XML - -- date/time. Simply take the day part from one and the time - -- from the other. - -- - make_game_time d t = UTCTime (utctDay d) (utctDayTime t) + -- | Construct the database game time from the XML \ + -- and \ elements. The \ elements + -- sometimes have a value of \"TBA\"; in that case, we don't + -- want to pretend that we know the time by setting it to + -- e.g. midnight, so instead we make the entire date/time + -- Nothing. + make_game_time :: UTCTime -> Maybe UTCTime -> Maybe UTCTime + make_game_time _ Nothing = Nothing + make_game_time d (Just t) = Just $ UTCTime (utctDay d) (utctDayTime t) -- | This allows us to insert the XML representation @@ -453,7 +457,7 @@ pickle_odds_info (xpElem "seasontype" (xpOption xpText)) (xpElem "Game_Date" xp_date_padded) - (xpElem "Game_Time" xp_time) + (xpElem "Game_Time" xp_tba_time) pickle_away_team (xpOption $ xpElem "vleague" xpText) pickle_home_team @@ -612,7 +616,10 @@ [ check "pickle composed with unpickle is the identity" "test/xml/jfilexml.xml", check "pickle composed with unpickle is the identity (missing fields)" - "test/xml/jfilexml-missing-fields.xml" ] + "test/xml/jfilexml-missing-fields.xml", + + check "pickle composed with unpickle is the identity (TBA game time)" + "test/xml/jfilexml-tba-game-time.xml"] where check desc path = testCase desc $ do (expected, actual) <- pickle_unpickle pickle_message path @@ -625,8 +632,12 @@ test_unpickle_succeeds :: TestTree test_unpickle_succeeds = testGroup "unpickle tests" [ check "unpickling succeeds" "test/xml/jfilexml.xml", + check "unpickling succeeds (missing fields)" - "test/xml/jfilexml-missing-fields.xml" ] + "test/xml/jfilexml-missing-fields.xml", + + check "unpickling succeeds (TBA game time)" + "test/xml/jfilexml-tba-game-time.xml" ] where check desc path = testCase desc $ do actual <- unpickleable path pickle_message @@ -643,10 +654,15 @@ test_on_delete_cascade = testGroup "cascading delete tests" [ check "deleting auto_racing_results deletes its children" "test/xml/jfilexml.xml" - 20, + 20, -- teams + check "deleting auto_racing_results deletes its children (missing fields)" "test/xml/jfilexml-missing-fields.xml" - 44 ] + 44, + + check "deleting auto_racing_results deletes its children (TBA game time)" + "test/xml/jfilexml-tba-game-time.xml" + 8 ] where check desc path expected = testCase desc $ do results <- unsafe_unpickle path pickle_message diff --git a/src/TSN/XML/News.hs b/src/TSN/XML/News.hs --- a/src/TSN/XML/News.hs +++ b/src/TSN/XML/News.hs @@ -120,7 +120,7 @@ xml_url :: Maybe String, xml_teams :: [NewsTeam], xml_locations :: [Location], - xml_sms :: String, + xml_sms :: Maybe String, xml_editor :: Maybe String, xml_text :: Maybe String, -- Text and continue seem to show up in pairs, xml_continue :: Maybe String, -- either both present or both missing. @@ -138,7 +138,7 @@ db_mid :: MsgId, db_sport :: String, db_url :: Maybe String, - db_sms :: String, + db_sms :: Maybe String, db_editor :: Maybe String, db_text :: Maybe String, db_continue :: Maybe String, @@ -397,7 +397,7 @@ (xpElem "url" $ xpOption xpText) (xpList pickle_news_team) (xpList pickle_location) - (xpElem "SMS" xpText) + (xpElem "SMS" $ xpOption xpText) (xpOption (xpElem "Editor" xpText)) (xpOption (xpElem "text" xpText)) pickle_continue @@ -492,7 +492,10 @@ "test/xml/newsxml.xml", check "pickle composed with unpickle is the identity (with Editor)" - "test/xml/newsxml-with-editor.xml" ] + "test/xml/newsxml-with-editor.xml", + + check "pickle composed with unpickle is the identity (empty SMS)" + "test/xml/newsxml-empty-sms.xml" ] where check desc path = testCase desc $ do (expected, actual) <- pickle_unpickle pickle_message path @@ -507,7 +510,10 @@ "test/xml/newsxml.xml", check "unpickling succeeds (with Editor)" - "test/xml/newsxml-with-editor.xml" ] + "test/xml/newsxml-with-editor.xml", + + check "unpickling succeeds (empty SMS)" + "test/xml/newsxml-empty-sms.xml" ] where check desc path = testCase desc $ do actual <- unpickleable path pickle_message @@ -523,6 +529,10 @@ [ check "deleting news deletes its children" "test/xml/newsxml.xml" 4 -- 2 news_teams and 2 news_locations that should remain. + , + check "deleting news deletes its children (empty SMS)" + "test/xml/newsxml-empty-sms.xml" + 4 -- 2 news_teams and 2 news_locations ] where check desc path expected = testCase desc $ do diff --git a/src/TSN/XML/Odds.hs b/src/TSN/XML/Odds.hs --- a/src/TSN/XML/Odds.hs +++ b/src/TSN/XML/Odds.hs @@ -64,7 +64,7 @@ import TSN.Codegen ( tsn_codegen_config ) import TSN.DbImport ( DbImport(..), ImportResult(..), run_dbmigrate ) -import TSN.Picklers ( xp_date_padded, xp_time, xp_time_stamp ) +import TSN.Picklers ( xp_date_padded, xp_tba_time, xp_time_stamp ) import TSN.Team ( FromXmlFkTeams(..), Team(..) ) import TSN.XmlImport ( XmlImport(..), XmlImportFkTeams(..) ) import Xml ( @@ -251,7 +251,7 @@ db_away_team_id :: DefaultKey Team, db_home_team_id :: DefaultKey Team, db_game_id :: Int, - db_game_time :: UTCTime, -- ^ Contains both the date and time. + db_game_time :: Maybe UTCTime, -- ^ Contains both the date and time. db_away_team_rotation_number :: Maybe Int, db_home_team_rotation_number :: Maybe Int, db_away_team_starter_id :: Maybe Int, @@ -266,7 +266,7 @@ OddsGameXml { xml_game_id :: Int, xml_game_date :: UTCTime, -- ^ Contains only the date - xml_game_time :: UTCTime, -- ^ Contains only the time + xml_game_time :: Maybe UTCTime, -- ^ Contains only the time xml_away_team :: OddsGameTeamXml, xml_home_team :: OddsGameTeamXml, xml_over_under :: OddsGameOverUnderXml } @@ -306,9 +306,7 @@ db_home_team_id = fk_home, db_game_id = xml_game_id, - db_game_time = UTCTime - (utctDay xml_game_date) -- Take the day part from one, - (utctDayTime xml_game_time), -- the time from the other. + db_game_time = make_game_time xml_game_date xml_game_time, db_away_team_rotation_number = (xml_team_rotation_number xml_away_team), @@ -331,6 +329,16 @@ -- so we combine the two maybes with join. db_home_team_starter_name = join (xml_starter_name <$> xml_team_starter xml_home_team) } + where + -- | Construct the database game time from the XML \ + -- and \ elements. The \ elements + -- sometimes have a value of \"TBA\"; in that case, we don't + -- want to pretend that we know the time by setting it to + -- e.g. midnight, so instead we make the entire date/time + -- Nothing. + make_game_time :: UTCTime -> Maybe UTCTime -> Maybe UTCTime + make_game_time _ Nothing = Nothing + make_game_time d (Just t) = Just $ UTCTime (utctDay d) (utctDayTime t) -- | This lets us insert the XML representation 'OddsGameXml' directly. @@ -680,7 +688,7 @@ xp6Tuple (xpElem "GameID" xpInt) (xpElem "Game_Date" xp_date_padded) - (xpElem "Game_Time" xp_time) + (xpElem "Game_Time" xp_tba_time) pickle_away_team pickle_home_team pickle_over_under @@ -758,7 +766,10 @@ "test/xml/Odds_XML-league-name.xml", check "pickle composed with unpickle is the identity (missing starters)" - "test/xml/Odds_XML-missing-starters.xml" ] + "test/xml/Odds_XML-missing-starters.xml", + + check "pickle composed with unpickle is the identity (TBA game time)" + "test/xml/Odds_XML-tba-game-time.xml"] where check desc path = testCase desc $ do (expected, actual) <- pickle_unpickle pickle_message path @@ -785,7 +796,10 @@ "test/xml/Odds_XML-league-name.xml", check "unpickling succeeds (missing starters)" - "test/xml/Odds_XML-missing-starters.xml" ] + "test/xml/Odds_XML-missing-starters.xml", + + check "unpickling succeeds (TBA game time)" + "test/xml/Odds_XML-tba-game-time.xml" ] where check desc path = testCase desc $ do actual <- unpickleable path pickle_message @@ -824,6 +838,10 @@ check "deleting odds deleted its children (missing starters)" "test/xml/Odds_XML-missing-starters.xml" 7 -- 5 casinos, 2 teams + , + check "deleting odds deleted its children (TBA game time)" + "test/xml/Odds_XML-tba-game-time.xml" + 119 -- 5 casinos, 114 teams ] where check desc path expected = testCase desc $ do diff --git a/test/shell/import-duplicates.test b/test/shell/import-duplicates.test --- a/test/shell/import-duplicates.test +++ b/test/shell/import-duplicates.test @@ -16,7 +16,7 @@ # and a newsxml that aren't really supposed to import. find ./test/xml -maxdepth 1 -name '*.xml' | wc -l >>> -32 +35 >>>= 0 # Run the imports again; we should get complaints about the duplicate @@ -24,7 +24,7 @@ # occurrences of the string 'ERROR'. ./dist/build/htsn-import/htsn-import -c 'shelltest.sqlite3' test/xml/*.xml 2>&1 | grep ERROR | wc -l >>> -56 +62 >>>= 0 # Finally, clean up after ourselves. diff --git a/test/xml/Odds_XML-tba-game-time.xml b/test/xml/Odds_XML-tba-game-time.xml new file mode 100644 --- /dev/null +++ b/test/xml/Odds_XML-tba-game-time.xml @@ -0,0 +1,1 @@ + 21898449 AFO;CFB-ODDS Odds CFOOT Las Vegas College Football Line Current Line as of 9:30 A.M. ET THURSDAY, OCTOBER 2ND 43419 10/02/2014 07:00 PM 710 303 FAU Fla Atlantic -9.5 -9.5 -9.5 820 304 FIU FIU Time change 44061 10/02/2014 07:00 PM 648 305 UCF UCF 509 306 HOUSTON Houston -2.5 -2.5 -3 43098 10/02/2014 10:30 PM 457 307 ARIZONA Arizona 576 308 OREGON Oregon -23 -22.5 -23 FRIDAY, OCTOBER 3RD 42862 10/03/2014 07:00 PM 531 309 LOUISVILLE Louisville -3.5 -3.5 -3.5 603 310 SYRACUSE Syracuse 43717 10/03/2014 10:00 PM 589 311 SANDIEGOST San Diego St 499 312 FRESNOST Fresno State 43493 10/03/2014 10:15 PM 621 313 UTAHST Utah State 471 314 BYU BYU SATURDAY, OCTOBER 4TH Time change 42708 10/04/2014 04:00 PM 622 315 VANDERBILT Vanderbilt 502 316 GEORGIA Georgia -32.5 -32.5 -32.5 Time change 42797 10/04/2014 03:30 PM 626 317 WAKEFOREST Wake Forest 498 318 FLORIDAST Florida State -38 -38 -38 Time change 42808 10/04/2014 07:30 PM 538 319 MIAFLA Miami-Florida -1 -1 -1 504 320 GATECH Georgia Tech Time change 43178 10/04/2014 02:30 PM 563 321 NTEXAS North Texas 515 322 INDIANA Indiana -13.5 -13.5 -14 43014 10/04/2014 12:00 PM 584 323 PURDUE Purdue 513 324 ILLINOIS Illinois 42994 10/04/2014 12:00 PM 572 325 OHIOST Ohio State 534 326 MARYLAND Maryland 44017 10/04/2014 12:00 PM 596 327 SMU SMU 491 328 ECAROLINA East Carolina -38 -38 -38 Time change 44025 10/04/2014 07:00 PM 537 329 MEMPHIS Memphis 476 330 CINCINNATI Cincinnati -5.5 -5.5 -5 Time change 42874 10/04/2014 07:30 PM 581 331 PITT Pittsburgh 623 332 VIRGINIA Virginia -4.5 -4.5 -4.5 Time change 43931 10/04/2014 03:30 PM 573 333 OHIOU Ohio U 475 334 CMICHIGAN Central Mich 42951 10/04/2014 04:00 PM 521 335 KANSAS Kansas 635 336 WVIRGINIA West Virginia -23.5 -24 -24 Time change 43951 10/04/2014 07:00 PM 616 337 TOLEDO Toledo -9 -9 -9 633 338 WMICHIGAN Western Mich 43448 10/04/2014 12:00 PM 533 339 MARSHALL Marshall -18 -18 -18 YX1 340 ODU Old Dominion Time change 42833 10/04/2014 12:30 PM 625 341 VATECH Virginia Tech -1 -1 -1 560 342 NCAROLINA North Carolina Time change 43923 10/04/2014 05:00 PM 523 343 KENTST Kent State 567 344 NILLINOIS Northern Ill -23.5 -23.5 -24 Time change 43062 10/04/2014 03:30 PM 638 345 WISCONSIN Wisconsin -9.5 -9.5 -9.5 569 346 NWESTERN Northwestern Time change 43917 10/04/2014 02:30 PM 535 347 UMASS Massachusetts 539 348 MIAOHIO Miami-Ohio -4.5 -4 -4 Time change 43981 10/04/2014 06:00 PM YX2 349 SALABAMA South Alabama -5 -5 -5.5 456 350 APPALACHAN Appalachian St Time change 42716 10/04/2014 07:30 PM 591 351 SCAROLINA So Carolina -7 -7 -7 524 352 KENTUCKY Kentucky 42898 10/04/2014 12:00 PM 464 353 BALLST Ball State 461 354 ARMY Army -2.5 -3 -3 Time change 43905 10/04/2014 03:30 PM 664 355 BUFFALO Buffalo 470 356 BOWLINGGRN Bowling Green -5.5 -5.5 -5 43897 10/04/2014 02:00 PM 495 357 EMICHIGAN Eastern Mich 452 358 AKRON Akron -21 -21 -21 Time change 42957 10/04/2014 07:00 PM 615 359 TEXASTECH Texas Tech 522 360 KANSASST Kansas State Time change 43745 10/04/2014 07:00 PM 507 361 HAWAII Hawaii 586 362 RICE Rice -6 -6 -6 Time change 43738 10/04/2014 03:00 PM 618 363 TULSA Tulsa 481 364 COLORADOST Colorado State -14.5 -14.5 -14.5 Time change 43961 10/04/2014 07:00 PM Y32 365 GASTATE Georgia State 601 366 ULLAFAYETE Louisiana -17 -17 -17 42843 10/04/2014 03:30 PM 602 367 STANFORD Stanford -1.5 -1.5 -1.5 571 368 NOTREDAME Notre Dame 43111 10/04/2014 TBA 458 369 ARIZONAST Arizona State 594 370 USC USC -10 -10 -10 Time change 42697 10/04/2014 07:00 PM 529 371 LSU LSU 462 372 AUBURN Auburn -8.5 -8.5 -8.5 42910 10/04/2014 03:30 PM 552 373 NAVY Navy -5 -5 -5 451 374 AIRFORCE Air Force 42966 10/04/2014 03:30 PM 574 375 OKLAHOMA Oklahoma -4.5 -4.5 -4.5 612 376 TCU TCU Time change 42594 10/04/2014 12:00 PM 496 377 FLORIDA Florida -1 -1 -1 605 378 TENNESSEE Tennessee 42929 10/04/2014 03:30 PM 465 379 BAYLOR Baylor -15 -15 -14.5 609 380 TEXAS Texas Time change 42940 10/04/2014 12:00 PM 518 381 IOWAST Iowa State 575 382 OKLAHOMAST Oklahoma State -17 -17 -17 42685 10/04/2014 03:30 PM 453 383 ALABAMA Alabama -5.5 -5.5 -5.5 544 384 MISS Ole Miss Time change 42743 10/04/2014 12:00 PM 610 385 TEXASAM Texas A&M 545 386 MISSST Miss State -1 -1 -1 43165 10/04/2014 TBA 620 387 UTAH Utah 619 388 UCLA UCLA -13 -13 -12.5 43438 10/04/2014 07:00 PM 613 389 UTEP Texas-El Paso 530 390 LOUISTECH Louisiana Tech -12.5 -12.5 -13 Time change 43456 10/04/2014 12:00 PM 597 391 SOTHRNMISS Southern Miss 542 392 MIDTENNST Middle Tennessee -17 -17 -16.5 Time change 43132 10/04/2014 04:00 PM 577 393 OREGONST Oregon State -7.5 -7.5 -7.5 480 394 COLORADO Colorado Time change 43121 10/04/2014 10:30 PM 474 395 CALIFORNIA California 628 396 WASHSTATE Washington St -3 -3.5 -3 43984 10/04/2014 07:00 PM 511 397 IDAHO Idaho 600 398 TEXASST Texas State -14.5 -14.5 -14.5 43031 10/04/2014 07:00 PM 540 399 MICHIGAN Michigan 588 400 RUTGERS Rutgers -3.5 -3.5 -3.5 Time change 43484 10/04/2014 03:30 PM 557 401 NMEXICO New Mexico YX5 402 TXSANANTO TX-San Antonio -18.5 -18.5 -18.5 43952 10/04/2014 07:00 PM 565 403 ULMONROE ULM 460 404 ARKANSASST Arkansas State -10 -10 -10 Time change 42816 10/04/2014 03:30 PM 562 405 NCSTATE NC State 478 406 CLEMSON Clemson -14 -14 -14 Time change 43409 10/04/2014 07:00 PM 703 407 UAB UAB 632 408 WKENTUCKY Western Ky -13.5 -13.5 -13.5 43040 10/04/2014 08:00 PM 553 409 NEBRASKA Nebraska 541 410 MICHIGANST Michigan State -9.5 -9.5 -9.5 43995 10/04/2014 08:00 PM 503 411 GASOUTHERN Ga Southern -17 -17 -17 558 412 NMEXICOST New Mexico St Time change 43753 10/04/2014 08:00 PM 554 413 UNLV UNLV 590 414 SANJOSEST San Jose St 43726 10/04/2014 10:30 PM 467 415 BOISEST Boise State -2.5 -2.5 -2.5 555 416 NEVADA Nevada September 29, 2014, at 09:37 AM ET diff --git a/test/xml/jfilexml-tba-game-time.xml b/test/xml/jfilexml-tba-game-time.xml new file mode 100644 --- /dev/null +++ b/test/xml/jfilexml-tba-game-time.xml @@ -0,0 +1,1 @@ + 21933332 BC-AAJ JFILE MLB 44542 44542 PostSeason 10/06/2014 TBA Los Angeles NL St. Louis NL 0 0 3:00 AM 44543 44543 PostSeason 10/06/2014 TBA Washington NL San Francisco NL 0 0 3:00 AM 44533 44533 10/5/2014 032 027 DET BAL Detroit Baltimore D.Price B.Norris 10/5/2014 3:45:00 PM 918 917 3:30 P.M. True ALDS - Baltimore leads 2-0 PostSeason 10/05/2014 03:45 PM Baltimore AL Detroit AL 0 0 3:00 AM 44532 44532 10/5/2014 033 029 KAN ANA Kansas City LA Angels J.Shields C.Wilson 10/5/2014 7:37:00 PM 920 919 3:30 P.M. True ALDS - Kansas City leads 2-0 PostSeason 10/05/2014 07:37 PM LA Angels AL Kansas City AL 0 0 7:37 PM October 5, 2014, at 04:10 PM ET diff --git a/test/xml/newsxml-empty-sms.xml b/test/xml/newsxml-empty-sms.xml new file mode 100644 --- /dev/null +++ b/test/xml/newsxml-empty-sms.xml @@ -0,0 +1,1 @@ + 21904559 AFN;UTAH-UCLA 0 News CFOOT UCLA UTAH U-C-L-A CA USA Utah UT USA Pasadena, CA (SportsNetwork.com) - October 1, 2014, at 11:11 AM ET