diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Revision history for dataframe +## 2.2.0.0 + +A large performance and ML release. + +### Highlights +* **Much faster I/O and analytics.** CSV reading, group-by, joins and sorting were rebuilt on compact unboxed / `PackedText` columns, parallel open-addressing hash tables, and vectorized aggregation; the default reader drops `cassava` for a single-pass scanner and `dataframe-fastcsv` adds multicore chunking. The end-to-end join + group-by pipeline runs several times faster with much lower memory and scales with `-threaded` / `+RTS -N`; results are byte-identical (golden-tested). +* **Lazy engine on par with eager** — bounded-source queries route through the same fast paths (streaming preserved for unbounded); lazy `sortBy` now also orders non-`Text` columns correctly. +* **New ML library (`dataframe-learn`).** scikit-learn-style estimators behind a uniform `fit` / `predict`: linear / ridge / lasso / logistic regression, SVM, trees, boosting, k-means, GMM, DBSCAN, PCA and kernel PCA, symbolic regression and feature synthesis, with metrics and cross-validation. Pure and deterministic; every model also compiles to a dataframe `Expr`. +* **Typed joins are checked at compile time** — keys must exist in both schemas with matching types (previously a runtime failure or silent empty result). + +### Breaking changes +* The `Column` GADT gains a `PackedText` constructor — exhaustive matches need a new arm (`materializePacked` decodes it back to boxed `Text`). CSV reads are now strict / fully forced; schema columns parse as their declared type; ragged rows pad with null instead of silently misaligning columns; overflowing integers parse as `Double`. `dataframe-learn`'s old beam-search synthesis and per-model `fit*` helpers are replaced by `fit` / `predict`. +* Coordinated major bumps (`dataframe-core`, `dataframe-learn` → `1.1.0.0`; umbrella `dataframe` → `2.2.0.0`) with inter-package lower bounds tightened so a newer package cannot resolve against an incompatible sibling. Drops `cassava` and `unordered-containers`; requires `text >= 2.1`. + ## 2.1.0.3 ### Packaging * Fix dependency resolution for the `dataframe` meta-package and its satellites diff --git a/data/ml/blobs.csv b/data/ml/blobs.csv new file mode 100644 --- /dev/null +++ b/data/ml/blobs.csv @@ -0,0 +1,151 @@ +x,y,cluster +10.338264760556216,0.847533060653685,2.0 +-8.030169963007346,5.6974805690319235,0.0 +10.72808765826904,0.8199785012860177,2.0 +-9.06787703268451,6.456351950062689,0.0 +-1.9752744232671209,4.798741021511607,1.0 +-7.950069649558368,6.313857688658719,0.0 +9.185263691410944,-0.02689531876281781,2.0 +-1.2924664642268644,5.094345005459218,1.0 +-8.386915515325072,6.748967461389298,0.0 +9.47365658115339,-0.3512223275732281,2.0 +-9.794694684045528,6.026719329863166,0.0 +-8.474546521207202,4.194596399725956,0.0 +-1.8006579280809625,4.866203894410084,1.0 +-8.798016496863006,3.7677237632249145,0.0 +9.608066918226854,1.550543508284739,2.0 +-0.46295420029132506,4.91792529440678,1.0 +9.085876965700251,0.8012523279907401,2.0 +-7.2757229200427895,5.372267655941989,0.0 +9.40388503768542,1.2438256022898402,2.0 +-8.208254840130058,6.1863971123041805,0.0 +-9.8405055750545,4.1558969180185,0.0 +-1.0130643168373106,3.870050967862403,1.0 +-1.750299605496018,5.8673653480977395,1.0 +-7.250170505806332,6.133910399530022,0.0 +-0.8427472133887682,4.013725740204026,1.0 +-8.783546167149966,7.221633621411183,0.0 +-7.801247618806028,5.454386723800413,0.0 +10.043495267038827,1.5187267683935621,2.0 +-9.428707750327403,4.758362395941994,0.0 +-0.7221251049321777,4.573801938428627,1.0 +-8.768888987279134,4.073279941977919,0.0 +-2.3850503444929023,3.3944007047064506,1.0 +9.52005242972435,1.344762816352543,2.0 +-1.6485932394255796,4.637081202200104,1.0 +-7.634316973046253,5.265196389986278,0.0 +-2.873923777802405,6.120339997469102,1.0 +-8.01938470265847,4.996106087665121,0.0 +8.650080809399894,0.5164250456305327,2.0 +-6.944219122655304,5.788217322112048,0.0 +9.411501808610677,1.1207212024133286,2.0 +8.727685388903897,0.7881397821440224,2.0 +9.940157883783744,0.31129800696905785,2.0 +-1.5441245562713808,3.7925654469165777,1.0 +-8.668033475458893,4.43578271480966,0.0 +-8.974177391694264,5.461137235845716,0.0 +-8.579735970574333,5.350794772331189,0.0 +-0.8507452109763282,5.35913594514609,1.0 +9.799881364516688,1.4185320368386882,2.0 +7.715643621051086,1.4926421993810217,2.0 +9.204875175494418,0.6321707789639058,2.0 +10.106053390869812,0.1776808950835841,2.0 +-8.549654572872772,6.552844582677501,0.0 +10.555503658306312,1.421054556659824,2.0 +-0.8409150384679815,4.759055557379698,1.0 +10.580079627664762,0.7854064702080787,2.0 +-10.125366055017796,5.068648373068846,0.0 +-0.36588996340355817,3.8233854113954955,1.0 +9.426230450546782,0.5860742620560562,2.0 +-1.885479568645208,4.623149715673959,1.0 +9.223577182486164,1.5910144728588864,2.0 +-7.2711793961873274,5.428189471188245,0.0 +9.453560211741014,-0.7499716472172048,2.0 +-1.984038662573283,4.364283036544825,1.0 +-8.392687024351563,5.800438033246547,0.0 +11.030025753690552,0.7766578881818996,2.0 +-2.2915284396471147,5.001466504584993,1.0 +-1.6145496750257826,4.777612786934544,1.0 +10.168318216276937,0.6986657131606543,2.0 +-2.2266293822669425,4.0573617784558555,1.0 +-8.254266273512885,4.37715621930642,0.0 +-9.437210089000642,6.767956346573112,0.0 +10.590353086938768,0.2993071633588282,2.0 +-2.79339785001823,4.767968183672174,1.0 +-1.2140195803496816,4.126048681774031,1.0 +-8.141432121082447,4.859540119431719,0.0 +9.089586732772219,-0.9279621694174798,2.0 +-7.1532744595716755,5.721844273167379,0.0 +-8.510143036409373,4.437832885485033,0.0 +-1.138567419393835,2.986258859412659,1.0 +-8.034838116698193,6.139679036447002,0.0 +-1.4384684377722188,5.3796868046940745,1.0 +-1.2202242152011087,3.034384563178521,1.0 +9.10502555609624,0.429281886212569,2.0 +10.585464339946231,0.008965041738488957,2.0 +-7.659707807990062,6.078774657537933,0.0 +9.823590118434234,0.6308585417952228,2.0 +-2.476845488661084,5.076283717062412,1.0 +-8.553522718036747,6.958005682199637,0.0 +-8.069594713163454,5.389290712648975,0.0 +-1.4505216161787537,3.810535489405869,1.0 +0.14236866854605823,4.102802122310017,1.0 +-8.713989201960986,4.654390170035729,0.0 +-1.462202464398755,4.709150238434744,1.0 +9.139446464353718,0.9973645548743786,2.0 +-2.0942032865826947,3.003362783235963,1.0 +9.384194636323251,0.5881152884036329,2.0 +-0.09043844586306131,4.594778743582171,1.0 +-8.630656062367228,5.125759989711783,0.0 +-8.258304908114546,5.178692149281611,0.0 +-1.6564983727218412,3.57173364605436,1.0 +-8.747527831262346,5.220834523095475,0.0 +8.289191474819738,0.9762956762429598,2.0 +-0.2667896158894523,5.2537491444891975,1.0 +-1.4567017014328651,3.240421719648844,1.0 +-2.578532846092601,5.281315504987536,1.0 +-7.060905510177389,5.334844843271841,0.0 +9.544505126636004,1.2726118372517006,2.0 +-1.789088183709013,4.9356797992475885,1.0 +-9.012750362421714,5.553681884762233,0.0 +-2.0885293775780265,3.819536691473433,1.0 +-8.160597657718792,4.443878519201542,0.0 +9.953505807271908,0.7455103282555472,2.0 +-8.628118580800681,4.175965562435486,0.0 +-0.4807339560151918,4.5869562471000975,1.0 +9.458262920723499,0.3546075672900924,2.0 +-1.3869004247455148,5.027138844442485,1.0 +-1.3046003840083444,5.232595722796476,1.0 +10.903405766013115,0.2879089162540264,2.0 +-8.167336731114846,7.396451887048452,0.0 +-1.1612914315034135,3.4000871166542685,1.0 +-1.7253529304545083,4.179477773883045,1.0 +-0.9912710391638034,5.444758663793727,1.0 +-2.3277031463654856,5.054227064960736,1.0 +8.147437705805391,1.071564094656846,2.0 +9.877041844526358,1.7070321683770884,2.0 +-0.3870567318504991,4.922009547520749,1.0 +-8.713603202383117,6.635883998983632,0.0 +9.549394662765959,1.8843175122058762,2.0 +-6.665876623219847,6.293607308205353,0.0 +-9.6218049853493,5.999675143347125,0.0 +-8.780572710866066,4.886490355985694,0.0 +10.03429276137997,0.5300549650285779,2.0 +9.157196652555177,1.2812079409319164,2.0 +8.1447728798683,0.5915572830522724,2.0 +9.146956239818431,1.017246090363041,2.0 +-9.165426205789453,5.897872124592354,0.0 +9.959392438724798,0.19080965893214255,2.0 +8.486697738274069,1.4784940535418505,2.0 +-0.3582643293862715,5.840807754114097,1.0 +-2.2186345261641804,4.615624117910637,1.0 +10.47604297470776,0.4921540622294554,2.0 +-2.7604144732798868,4.9784338438022795,1.0 +9.937212447233478,-0.1112822931713835,2.0 +-8.483627961806011,5.53784507561264,0.0 +9.270455589800079,1.4007210675022714,2.0 +0.25622916659958644,3.2714846427335473,1.0 +10.148892613250826,-0.16841584321546238,2.0 +9.702422433216821,2.162194444522933,2.0 +-9.104972635421447,5.6000283031270515,0.0 +-0.46045498366900267,4.696212695686656,1.0 diff --git a/data/ml/golden.json b/data/ml/golden.json new file mode 100644 --- /dev/null +++ b/data/ml/golden.json @@ -0,0 +1,68 @@ +{ + "gbm": { + "accuracy": 1.0 + }, + "kmeans": { + "inertia": 173.75480764755966 + }, + "linear_svc": { + "accuracy": 1.0 + }, + "logistic_binary": { + "accuracy": 1.0 + }, + "logistic_iris": { + "accuracy": 0.9733333333333334 + }, + "ols": { + "coef": [ + -10.009866299810287, + -239.81564367242294, + 519.8459200544604, + 324.38464550232385, + -792.1756385522323, + 476.7390210052585, + 101.04326793803448, + 177.06323767134674, + 751.2736995571044, + 67.62669218370488 + ], + "intercept": 152.13348416289597 + }, + "pca": { + "components_abs": [ + [ + 0.3613865917853659, + 0.08452251406457255, + 0.8566706059498347, + 0.3582891971515517 + ], + [ + 0.6565887712868534, + 0.7301614347850159, + 0.17337266279585964, + 0.0754810199174582 + ] + ], + "evr": [ + 0.9246187232017291, + 0.05306648311706544 + ] + }, + "ridge": { + "alpha": 1.0, + "coef": [ + 29.466111893477002, + -83.15427636187523, + 306.3526801506861, + 201.62773437326962, + 5.909614367497247, + -29.515495079689597, + -152.04028006186414, + 117.31173160030173, + 262.9442900143125, + 111.87895643952352 + ], + "intercept": 152.133484162896 + } +} diff --git a/data/ml/iris.csv b/data/ml/iris.csv new file mode 100644 --- /dev/null +++ b/data/ml/iris.csv @@ -0,0 +1,151 @@ +sepal_length,sepal_width,petal_length,petal_width,species +5.1,3.5,1.4,0.2,0.0 +4.9,3.0,1.4,0.2,0.0 +4.7,3.2,1.3,0.2,0.0 +4.6,3.1,1.5,0.2,0.0 +5.0,3.6,1.4,0.2,0.0 +5.4,3.9,1.7,0.4,0.0 +4.6,3.4,1.4,0.3,0.0 +5.0,3.4,1.5,0.2,0.0 +4.4,2.9,1.4,0.2,0.0 +4.9,3.1,1.5,0.1,0.0 +5.4,3.7,1.5,0.2,0.0 +4.8,3.4,1.6,0.2,0.0 +4.8,3.0,1.4,0.1,0.0 +4.3,3.0,1.1,0.1,0.0 +5.8,4.0,1.2,0.2,0.0 +5.7,4.4,1.5,0.4,0.0 +5.4,3.9,1.3,0.4,0.0 +5.1,3.5,1.4,0.3,0.0 +5.7,3.8,1.7,0.3,0.0 +5.1,3.8,1.5,0.3,0.0 +5.4,3.4,1.7,0.2,0.0 +5.1,3.7,1.5,0.4,0.0 +4.6,3.6,1.0,0.2,0.0 +5.1,3.3,1.7,0.5,0.0 +4.8,3.4,1.9,0.2,0.0 +5.0,3.0,1.6,0.2,0.0 +5.0,3.4,1.6,0.4,0.0 +5.2,3.5,1.5,0.2,0.0 +5.2,3.4,1.4,0.2,0.0 +4.7,3.2,1.6,0.2,0.0 +4.8,3.1,1.6,0.2,0.0 +5.4,3.4,1.5,0.4,0.0 +5.2,4.1,1.5,0.1,0.0 +5.5,4.2,1.4,0.2,0.0 +4.9,3.1,1.5,0.2,0.0 +5.0,3.2,1.2,0.2,0.0 +5.5,3.5,1.3,0.2,0.0 +4.9,3.6,1.4,0.1,0.0 +4.4,3.0,1.3,0.2,0.0 +5.1,3.4,1.5,0.2,0.0 +5.0,3.5,1.3,0.3,0.0 +4.5,2.3,1.3,0.3,0.0 +4.4,3.2,1.3,0.2,0.0 +5.0,3.5,1.6,0.6,0.0 +5.1,3.8,1.9,0.4,0.0 +4.8,3.0,1.4,0.3,0.0 +5.1,3.8,1.6,0.2,0.0 +4.6,3.2,1.4,0.2,0.0 +5.3,3.7,1.5,0.2,0.0 +5.0,3.3,1.4,0.2,0.0 +7.0,3.2,4.7,1.4,1.0 +6.4,3.2,4.5,1.5,1.0 +6.9,3.1,4.9,1.5,1.0 +5.5,2.3,4.0,1.3,1.0 +6.5,2.8,4.6,1.5,1.0 +5.7,2.8,4.5,1.3,1.0 +6.3,3.3,4.7,1.6,1.0 +4.9,2.4,3.3,1.0,1.0 +6.6,2.9,4.6,1.3,1.0 +5.2,2.7,3.9,1.4,1.0 +5.0,2.0,3.5,1.0,1.0 +5.9,3.0,4.2,1.5,1.0 +6.0,2.2,4.0,1.0,1.0 +6.1,2.9,4.7,1.4,1.0 +5.6,2.9,3.6,1.3,1.0 +6.7,3.1,4.4,1.4,1.0 +5.6,3.0,4.5,1.5,1.0 +5.8,2.7,4.1,1.0,1.0 +6.2,2.2,4.5,1.5,1.0 +5.6,2.5,3.9,1.1,1.0 +5.9,3.2,4.8,1.8,1.0 +6.1,2.8,4.0,1.3,1.0 +6.3,2.5,4.9,1.5,1.0 +6.1,2.8,4.7,1.2,1.0 +6.4,2.9,4.3,1.3,1.0 +6.6,3.0,4.4,1.4,1.0 +6.8,2.8,4.8,1.4,1.0 +6.7,3.0,5.0,1.7,1.0 +6.0,2.9,4.5,1.5,1.0 +5.7,2.6,3.5,1.0,1.0 +5.5,2.4,3.8,1.1,1.0 +5.5,2.4,3.7,1.0,1.0 +5.8,2.7,3.9,1.2,1.0 +6.0,2.7,5.1,1.6,1.0 +5.4,3.0,4.5,1.5,1.0 +6.0,3.4,4.5,1.6,1.0 +6.7,3.1,4.7,1.5,1.0 +6.3,2.3,4.4,1.3,1.0 +5.6,3.0,4.1,1.3,1.0 +5.5,2.5,4.0,1.3,1.0 +5.5,2.6,4.4,1.2,1.0 +6.1,3.0,4.6,1.4,1.0 +5.8,2.6,4.0,1.2,1.0 +5.0,2.3,3.3,1.0,1.0 +5.6,2.7,4.2,1.3,1.0 +5.7,3.0,4.2,1.2,1.0 +5.7,2.9,4.2,1.3,1.0 +6.2,2.9,4.3,1.3,1.0 +5.1,2.5,3.0,1.1,1.0 +5.7,2.8,4.1,1.3,1.0 +6.3,3.3,6.0,2.5,2.0 +5.8,2.7,5.1,1.9,2.0 +7.1,3.0,5.9,2.1,2.0 +6.3,2.9,5.6,1.8,2.0 +6.5,3.0,5.8,2.2,2.0 +7.6,3.0,6.6,2.1,2.0 +4.9,2.5,4.5,1.7,2.0 +7.3,2.9,6.3,1.8,2.0 +6.7,2.5,5.8,1.8,2.0 +7.2,3.6,6.1,2.5,2.0 +6.5,3.2,5.1,2.0,2.0 +6.4,2.7,5.3,1.9,2.0 +6.8,3.0,5.5,2.1,2.0 +5.7,2.5,5.0,2.0,2.0 +5.8,2.8,5.1,2.4,2.0 +6.4,3.2,5.3,2.3,2.0 +6.5,3.0,5.5,1.8,2.0 +7.7,3.8,6.7,2.2,2.0 +7.7,2.6,6.9,2.3,2.0 +6.0,2.2,5.0,1.5,2.0 +6.9,3.2,5.7,2.3,2.0 +5.6,2.8,4.9,2.0,2.0 +7.7,2.8,6.7,2.0,2.0 +6.3,2.7,4.9,1.8,2.0 +6.7,3.3,5.7,2.1,2.0 +7.2,3.2,6.0,1.8,2.0 +6.2,2.8,4.8,1.8,2.0 +6.1,3.0,4.9,1.8,2.0 +6.4,2.8,5.6,2.1,2.0 +7.2,3.0,5.8,1.6,2.0 +7.4,2.8,6.1,1.9,2.0 +7.9,3.8,6.4,2.0,2.0 +6.4,2.8,5.6,2.2,2.0 +6.3,2.8,5.1,1.5,2.0 +6.1,2.6,5.6,1.4,2.0 +7.7,3.0,6.1,2.3,2.0 +6.3,3.4,5.6,2.4,2.0 +6.4,3.1,5.5,1.8,2.0 +6.0,3.0,4.8,1.8,2.0 +6.9,3.1,5.4,2.1,2.0 +6.7,3.1,5.6,2.4,2.0 +6.9,3.1,5.1,2.3,2.0 +5.8,2.7,5.1,1.9,2.0 +6.8,3.2,5.9,2.3,2.0 +6.7,3.3,5.7,2.5,2.0 +6.7,3.0,5.2,2.3,2.0 +6.3,2.5,5.0,1.9,2.0 +6.5,3.0,5.2,2.0,2.0 +6.2,3.4,5.4,2.3,2.0 +5.9,3.0,5.1,1.8,2.0 diff --git a/data/ml/iris_binary.csv b/data/ml/iris_binary.csv new file mode 100644 --- /dev/null +++ b/data/ml/iris_binary.csv @@ -0,0 +1,151 @@ +sepal_length,sepal_width,petal_length,petal_width,label +5.1,3.5,1.4,0.2,0.0 +4.9,3.0,1.4,0.2,0.0 +4.7,3.2,1.3,0.2,0.0 +4.6,3.1,1.5,0.2,0.0 +5.0,3.6,1.4,0.2,0.0 +5.4,3.9,1.7,0.4,0.0 +4.6,3.4,1.4,0.3,0.0 +5.0,3.4,1.5,0.2,0.0 +4.4,2.9,1.4,0.2,0.0 +4.9,3.1,1.5,0.1,0.0 +5.4,3.7,1.5,0.2,0.0 +4.8,3.4,1.6,0.2,0.0 +4.8,3.0,1.4,0.1,0.0 +4.3,3.0,1.1,0.1,0.0 +5.8,4.0,1.2,0.2,0.0 +5.7,4.4,1.5,0.4,0.0 +5.4,3.9,1.3,0.4,0.0 +5.1,3.5,1.4,0.3,0.0 +5.7,3.8,1.7,0.3,0.0 +5.1,3.8,1.5,0.3,0.0 +5.4,3.4,1.7,0.2,0.0 +5.1,3.7,1.5,0.4,0.0 +4.6,3.6,1.0,0.2,0.0 +5.1,3.3,1.7,0.5,0.0 +4.8,3.4,1.9,0.2,0.0 +5.0,3.0,1.6,0.2,0.0 +5.0,3.4,1.6,0.4,0.0 +5.2,3.5,1.5,0.2,0.0 +5.2,3.4,1.4,0.2,0.0 +4.7,3.2,1.6,0.2,0.0 +4.8,3.1,1.6,0.2,0.0 +5.4,3.4,1.5,0.4,0.0 +5.2,4.1,1.5,0.1,0.0 +5.5,4.2,1.4,0.2,0.0 +4.9,3.1,1.5,0.2,0.0 +5.0,3.2,1.2,0.2,0.0 +5.5,3.5,1.3,0.2,0.0 +4.9,3.6,1.4,0.1,0.0 +4.4,3.0,1.3,0.2,0.0 +5.1,3.4,1.5,0.2,0.0 +5.0,3.5,1.3,0.3,0.0 +4.5,2.3,1.3,0.3,0.0 +4.4,3.2,1.3,0.2,0.0 +5.0,3.5,1.6,0.6,0.0 +5.1,3.8,1.9,0.4,0.0 +4.8,3.0,1.4,0.3,0.0 +5.1,3.8,1.6,0.2,0.0 +4.6,3.2,1.4,0.2,0.0 +5.3,3.7,1.5,0.2,0.0 +5.0,3.3,1.4,0.2,0.0 +7.0,3.2,4.7,1.4,1.0 +6.4,3.2,4.5,1.5,1.0 +6.9,3.1,4.9,1.5,1.0 +5.5,2.3,4.0,1.3,1.0 +6.5,2.8,4.6,1.5,1.0 +5.7,2.8,4.5,1.3,1.0 +6.3,3.3,4.7,1.6,1.0 +4.9,2.4,3.3,1.0,1.0 +6.6,2.9,4.6,1.3,1.0 +5.2,2.7,3.9,1.4,1.0 +5.0,2.0,3.5,1.0,1.0 +5.9,3.0,4.2,1.5,1.0 +6.0,2.2,4.0,1.0,1.0 +6.1,2.9,4.7,1.4,1.0 +5.6,2.9,3.6,1.3,1.0 +6.7,3.1,4.4,1.4,1.0 +5.6,3.0,4.5,1.5,1.0 +5.8,2.7,4.1,1.0,1.0 +6.2,2.2,4.5,1.5,1.0 +5.6,2.5,3.9,1.1,1.0 +5.9,3.2,4.8,1.8,1.0 +6.1,2.8,4.0,1.3,1.0 +6.3,2.5,4.9,1.5,1.0 +6.1,2.8,4.7,1.2,1.0 +6.4,2.9,4.3,1.3,1.0 +6.6,3.0,4.4,1.4,1.0 +6.8,2.8,4.8,1.4,1.0 +6.7,3.0,5.0,1.7,1.0 +6.0,2.9,4.5,1.5,1.0 +5.7,2.6,3.5,1.0,1.0 +5.5,2.4,3.8,1.1,1.0 +5.5,2.4,3.7,1.0,1.0 +5.8,2.7,3.9,1.2,1.0 +6.0,2.7,5.1,1.6,1.0 +5.4,3.0,4.5,1.5,1.0 +6.0,3.4,4.5,1.6,1.0 +6.7,3.1,4.7,1.5,1.0 +6.3,2.3,4.4,1.3,1.0 +5.6,3.0,4.1,1.3,1.0 +5.5,2.5,4.0,1.3,1.0 +5.5,2.6,4.4,1.2,1.0 +6.1,3.0,4.6,1.4,1.0 +5.8,2.6,4.0,1.2,1.0 +5.0,2.3,3.3,1.0,1.0 +5.6,2.7,4.2,1.3,1.0 +5.7,3.0,4.2,1.2,1.0 +5.7,2.9,4.2,1.3,1.0 +6.2,2.9,4.3,1.3,1.0 +5.1,2.5,3.0,1.1,1.0 +5.7,2.8,4.1,1.3,1.0 +6.3,3.3,6.0,2.5,1.0 +5.8,2.7,5.1,1.9,1.0 +7.1,3.0,5.9,2.1,1.0 +6.3,2.9,5.6,1.8,1.0 +6.5,3.0,5.8,2.2,1.0 +7.6,3.0,6.6,2.1,1.0 +4.9,2.5,4.5,1.7,1.0 +7.3,2.9,6.3,1.8,1.0 +6.7,2.5,5.8,1.8,1.0 +7.2,3.6,6.1,2.5,1.0 +6.5,3.2,5.1,2.0,1.0 +6.4,2.7,5.3,1.9,1.0 +6.8,3.0,5.5,2.1,1.0 +5.7,2.5,5.0,2.0,1.0 +5.8,2.8,5.1,2.4,1.0 +6.4,3.2,5.3,2.3,1.0 +6.5,3.0,5.5,1.8,1.0 +7.7,3.8,6.7,2.2,1.0 +7.7,2.6,6.9,2.3,1.0 +6.0,2.2,5.0,1.5,1.0 +6.9,3.2,5.7,2.3,1.0 +5.6,2.8,4.9,2.0,1.0 +7.7,2.8,6.7,2.0,1.0 +6.3,2.7,4.9,1.8,1.0 +6.7,3.3,5.7,2.1,1.0 +7.2,3.2,6.0,1.8,1.0 +6.2,2.8,4.8,1.8,1.0 +6.1,3.0,4.9,1.8,1.0 +6.4,2.8,5.6,2.1,1.0 +7.2,3.0,5.8,1.6,1.0 +7.4,2.8,6.1,1.9,1.0 +7.9,3.8,6.4,2.0,1.0 +6.4,2.8,5.6,2.2,1.0 +6.3,2.8,5.1,1.5,1.0 +6.1,2.6,5.6,1.4,1.0 +7.7,3.0,6.1,2.3,1.0 +6.3,3.4,5.6,2.4,1.0 +6.4,3.1,5.5,1.8,1.0 +6.0,3.0,4.8,1.8,1.0 +6.9,3.1,5.4,2.1,1.0 +6.7,3.1,5.6,2.4,1.0 +6.9,3.1,5.1,2.3,1.0 +5.8,2.7,5.1,1.9,1.0 +6.8,3.2,5.9,2.3,1.0 +6.7,3.3,5.7,2.5,1.0 +6.7,3.0,5.2,2.3,1.0 +6.3,2.5,5.0,1.9,1.0 +6.5,3.0,5.2,2.0,1.0 +6.2,3.4,5.4,2.3,1.0 +5.9,3.0,5.1,1.8,1.0 diff --git a/data/ml/regression.csv b/data/ml/regression.csv new file mode 100644 --- /dev/null +++ b/data/ml/regression.csv @@ -0,0 +1,443 @@ +f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,target +0.038075906433423026,0.05068011873981862,0.061696206518683294,0.0218723855140367,-0.04422349842444599,-0.03482076283769895,-0.04340084565202491,-0.002592261998183278,0.019907486170462722,-0.01764612515980379,151.0 +-0.0018820165277906047,-0.044641636506989144,-0.051474061238800654,-0.02632752814785296,-0.008448724111216851,-0.019163339748222204,0.07441156407875721,-0.03949338287409329,-0.0683315470939731,-0.092204049626824,75.0 +0.08529890629667548,0.05068011873981862,0.04445121333659049,-0.00567042229275739,-0.04559945128264711,-0.03419446591411989,-0.03235593223976409,-0.002592261998183278,0.002861309289833047,-0.025930338989472702,141.0 +-0.0890629393522567,-0.044641636506989144,-0.011595014505211082,-0.03665608107540074,0.01219056876179996,0.02499059336410222,-0.036037570043851025,0.03430885887772673,0.022687744966501246,-0.009361911330134878,206.0 +0.005383060374248237,-0.044641636506989144,-0.03638469220446948,0.0218723855140367,0.003934851612593237,0.015596139510416171,0.008142083605192267,-0.002592261998183278,-0.03198763948805312,-0.04664087356364498,135.0 +-0.09269547780327612,-0.044641636506989144,-0.040695940499992665,-0.019441826196154435,-0.06899064987206617,-0.07928784441181291,0.04127682384197474,-0.0763945037500033,-0.041176166918895155,-0.09634615654165846,97.0 +-0.045472477940023646,0.05068011873981862,-0.047162812943277475,-0.015998975220305175,-0.04009563984984263,-0.02480001206043385,0.0007788079970183853,-0.03949338287409329,-0.06291687914365544,-0.03835665973397607,138.0 +0.06350367559055897,0.05068011873981862,-0.0018947058402839008,0.0666294482000771,0.09061988167926385,0.10891438112369757,0.022868634821540033,0.01770335448356722,-0.0358161925842373,0.0030644094143684884,63.0 +0.04170844488444244,0.05068011873981862,0.061696206518683294,-0.04009893205125,-0.013952535544021335,0.0062016856567301245,-0.028674294435677143,-0.002592261998183278,-0.014959693812643405,0.0113486232440374,110.0 +-0.07090024709715959,-0.044641636506989144,0.039062152967186486,-0.03321323009955148,-0.012576582685820214,-0.034507614375909414,-0.024992656631590206,-0.002592261998183278,0.06773705306493534,-0.013504018244969336,310.0 +-0.09632801625429555,-0.044641636506989144,-0.08380842345522464,0.008100981610639655,-0.10338947132709418,-0.09056118903623617,-0.01394774321932938,-0.0763945037500033,-0.06291687914365544,-0.03421455281914162,101.0 +0.027178291080364757,0.05068011873981862,0.0175059114895705,-0.03321323009955148,-0.007072771253015731,0.045971540304001066,-0.06549067247654655,0.07120997975363674,-0.09643494994048712,-0.05906719430814835,69.0 +0.016280675727306498,-0.044641636506989144,-0.028840007687303888,-0.009113273268606652,-0.004320865536613489,-0.009768885894536158,0.04495846164606168,-0.03949338287409329,-0.030747917533098208,-0.042498766648810526,179.0 +0.005383060374248237,0.05068011873981862,-0.0018947058402839008,0.008100981610639655,-0.004320865536613489,-0.015718706668537318,-0.002902829807068556,-0.002592261998183278,0.038393928263466416,-0.013504018244969336,185.0 +0.04534098333546186,-0.044641636506989144,-0.02560657146566148,-0.012556124244455912,0.017694380194604446,-6.128357906057276e-05,0.0817748396869311,-0.03949338287409329,-0.03198763948805312,-0.07563562196748617,118.0 +-0.052737554842062495,0.05068011873981862,-0.018061886948495892,0.08040085210347414,0.08924392882106273,0.10766178727653941,-0.03971920784793797,0.10811110062954676,0.036060333995316066,-0.042498766648810526,171.0 +-0.005514554978810025,-0.044641636506989144,0.0422955891888289,0.049415193320830796,0.024574144485610048,-0.02386056667506523,0.07441156407875721,-0.03949338287409329,0.05227699103843915,0.027917050903375224,166.0 +0.0707687524925978,0.05068011873981862,0.012116851120166501,0.056300895272529315,0.0342058144930179,0.04941617338368593,-0.03971920784793797,0.03430885887772673,0.02736404910541198,-0.0010776975004659671,144.0 +-0.03820740103798481,-0.044641636506989144,-0.010517202431330305,-0.03665608107540074,-0.037343734133440394,-0.019476488210011744,-0.028674294435677143,-0.002592261998183278,-0.018113692315690322,-0.01764612515980379,97.0 +-0.027309785684926546,-0.044641636506989144,-0.018061886948495892,-0.04009893205125,-0.0029449126784123676,-0.011334628203483833,0.0375951860378878,-0.03949338287409329,-0.00894339609006817,-0.05492508739331389,168.0 +-0.04910501639104307,-0.044641636506989144,-0.05686312160820465,-0.04354178302709927,-0.04559945128264711,-0.043275771306016404,0.0007788079970183853,-0.03949338287409329,-0.011896851335695978,0.015490730158871856,68.0 +-0.08543040090123728,0.05068011873981862,-0.022373135244019075,0.0012152796589411327,-0.037343734133440394,-0.02636575436938152,0.01550535921336615,-0.03949338287409329,-0.07213275338232743,-0.01764612515980379,49.0 +-0.08543040090123728,-0.044641636506989144,-0.004050329988045492,-0.009113273268606652,-0.0029449126784123676,0.0077674279656778,0.022868634821540033,-0.03949338287409329,-0.061175799045152635,-0.013504018244969336,68.0 +0.04534098333546186,0.05068011873981862,0.06061839444480248,0.031064797619554236,0.028702003060213414,-0.04734670130928034,-0.05444575906428573,0.07120997975363674,0.13359728192191356,0.13561183068907107,245.0 +-0.06363517019512076,-0.044641636506989144,0.03582871674554409,-0.0228846771720037,-0.030463969842434782,-0.018850191286432668,-0.006584467611155497,-0.002592261998183278,-0.025953110560258,-0.05492508739331389,184.0 +-0.06726770864614018,0.05068011873981862,-0.012672826579091896,-0.04009893205125,-0.015328488402222454,0.00463594334778245,-0.05812739686837268,0.03430885887772673,0.019196469166885697,-0.03421455281914162,202.0 +-0.1072256316073538,-0.044641636506989144,-0.07734155101193986,-0.02632752814785296,-0.08962994274508297,-0.09619786134844781,0.026550272625626974,-0.0763945037500033,-0.042570854118219384,-0.005219804415300423,137.0 +-0.02367724723390713,-0.044641636506989144,0.05954058237092167,-0.04009893205125,-0.04284754556624487,-0.04358891976780594,0.01182372140927921,-0.03949338287409329,-0.015998872510179042,0.040343371647878594,85.0 +0.0526060602375007,-0.044641636506989144,-0.0212953231701383,-0.07452744180974262,-0.04009563984984263,-0.03763909899380476,-0.006584467611155497,-0.03949338287409329,-0.0006117353045626216,-0.05492508739331389,131.0 +0.06713621404157838,0.05068011873981862,-0.006205954135807083,0.06318659722422783,-0.04284754556624487,-0.09588471288665826,0.05232173725423556,-0.0763945037500033,0.059423623484649676,0.05276969239238195,283.0 +-0.06000263174410134,-0.044641636506989144,0.04445121333659049,-0.019441826196154435,-0.009824676969417972,-0.007576846662009428,0.022868634821540033,-0.03949338287409329,-0.02712902329694316,-0.009361911330134878,129.0 +-0.02367724723390713,-0.044641636506989144,-0.06548561819925106,-0.08141314376144114,-0.038719686991641515,-0.05360967054507104,0.059685012862409445,-0.0763945037500033,-0.0371288393600719,-0.042498766648810526,59.0 +0.0344433679824036,0.05068011873981862,0.12528711887765046,0.028758087465735226,-0.05385516843185383,-0.012900370512431508,-0.10230705051741597,0.10811110062954676,0.00027247814860377354,0.027917050903375224,341.0 +0.03081082953138418,-0.044641636506989144,-0.050396249164919873,-0.002227571316908129,-0.04422349842444599,-0.0899348921126571,0.1185912177278005,-0.0763945037500033,-0.018113692315690322,0.0030644094143684884,87.0 +0.016280675727306498,-0.044641636506989144,-0.06332999405148947,-0.057313186930496314,-0.0579830270064572,-0.048912443618228024,0.008142083605192267,-0.03949338287409329,-0.05947118135708968,-0.06735140813781726,65.0 +0.04897352178648128,0.05068011873981862,-0.03099563183506548,-0.04929134415676754,0.04934129593323023,-0.004132213582324539,0.13331776894414826,-0.05351580880693909,0.021311288972396977,0.019632837073706312,102.0 +0.012648137276287077,-0.044641636506989144,0.022894971858974496,0.052858044296680055,0.0080627101871966,-0.02855779360190825,0.0375951860378878,-0.03949338287409329,0.05471997253790904,-0.025930338989472702,265.0 +-0.009147093429829445,-0.044641636506989144,0.011039039046285686,-0.057313186930496314,-0.0249601584096303,-0.04296262284422686,0.030231910429713918,-0.03949338287409329,0.017036071348324546,-0.005219804415300423,276.0 +-0.0018820165277906047,0.05068011873981862,0.07139651518361048,0.09761510698272045,0.08786797596286161,0.07540749571221732,-0.02131101882750326,0.07120997975363674,0.07142887212197009,0.02377494398854077,252.0 +-0.0018820165277906047,0.05068011873981862,0.014272475267928093,-0.07452744180974262,0.0025588987543921156,0.0062016856567301245,-0.01394774321932938,-0.002592261998183278,0.019196469166885697,0.0030644094143684884,90.0 +0.005383060374248237,0.05068011873981862,-0.008361578283568675,0.0218723855140367,0.05484510736603471,0.07321545647969056,-0.024992656631590206,0.03430885887772673,0.012551194864223063,0.09419076154072652,100.0 +-0.09996055470531495,-0.044641636506989144,-0.06764124234701265,-0.10895595156823522,-0.07449446130487065,-0.07271172671423268,0.01550535921336615,-0.03949338287409329,-0.049872451808799324,-0.009361911330134878,55.0 +-0.06000263174410134,0.05068011873981862,-0.010517202431330305,-0.014862834398274925,-0.04972730985725048,-0.02354741821327569,-0.05812739686837268,0.01585829843977173,-0.009918765569334137,-0.03421455281914162,61.0 +0.01991321417832592,-0.044641636506989144,-0.023450947317899894,-0.07108459083389335,0.020446285911006685,-0.010082034356325698,0.1185912177278005,-0.0763945037500033,-0.042570854118219384,0.07348022696655424,92.0 +0.04534098333546186,0.05068011873981862,0.0681630789619681,0.008100981610639655,-0.016704441260423575,0.00463594334778245,-0.07653558588880739,0.07120997975363674,0.03243232415655107,-0.01764612515980379,259.0 +0.027178291080364757,0.05068011873981862,-0.035306880130588664,0.03220093844158448,-0.011200629827619093,0.0015044587298871017,-0.010266105415242439,-0.002592261998183278,-0.014959693812643405,-0.05078298047847944,53.0 +-0.056370093293081916,-0.044641636506989144,-0.011595014505211082,-0.03321323009955148,-0.046975404140848234,-0.047659849771069886,0.0044604458011053266,-0.03949338287409329,-0.007977142213412223,-0.08806194271198954,190.0 +-0.07816532399919843,-0.044641636506989144,-0.07303030271641665,-0.057313186930496314,-0.0841261313122785,-0.07427746902318036,-0.024992656631590206,-0.03949338287409329,-0.018113692315690322,-0.08391983579715509,142.0 +0.06713621404157838,0.05068011873981862,-0.041773752573873474,0.011543832586488917,0.0025588987543921156,0.0058885371949405855,0.04127682384197474,-0.03949338287409329,-0.05947118135708968,-0.021788232074638245,75.0 +-0.04183993948900423,0.05068011873981862,0.014272475267928093,-0.00567042229275739,-0.012576582685820214,0.0062016856567301245,-0.07285394808472044,0.07120997975363674,0.03545870422305857,-0.013504018244969336,142.0 +0.0344433679824036,-0.044641636506989144,-0.007283766209687899,0.014986683562338177,-0.04422349842444599,-0.03732595053201525,-0.002902829807068556,-0.03949338287409329,-0.021395309255276825,0.007206516329202944,155.0 +0.059871137139539544,0.05068011873981862,0.016428099415689682,0.028758087465735226,-0.04147159270804375,-0.02918409052548733,-0.028674294435677143,-0.002592261998183278,-0.002398393416115213,-0.021788232074638245,225.0 +-0.052737554842062495,-0.044641636506989144,-0.00943939035744949,-0.00567042229275739,0.039709625925822375,0.04471894645684291,0.026550272625626974,-0.002592261998183278,-0.018113692315690322,-0.013504018244969336,59.0 +-0.009147093429829445,-0.044641636506989144,-0.015906262800734303,0.07007229917592636,0.01219056876179996,0.022172257207996385,0.01550535921336615,-0.002592261998183278,-0.03324559264822791,0.0486275854775475,104.0 +-0.04910501639104307,-0.044641636506989144,0.02505059600673609,0.008100981610639655,0.020446285911006685,0.017788178742942903,0.05232173725423556,-0.03949338287409329,-0.041176166918895155,0.007206516329202944,182.0 +-0.04183993948900423,-0.044641636506989144,-0.049318437091039065,-0.03665608107540074,-0.007072771253015731,-0.022607972827907094,0.08545647749101803,-0.03949338287409329,-0.06649019536676071,0.007206516329202944,128.0 +-0.04183993948900423,-0.044641636506989144,0.04121777711494808,-0.02632752814785296,-0.0318399227006359,-0.030436684372645465,-0.036037570043851025,0.0029429061332032365,0.033653814906286016,-0.01764612515980379,52.0 +-0.027309785684926546,-0.044641636506989144,-0.06332999405148947,-0.05042748497879779,-0.08962994274508297,-0.1043397213549757,0.05232173725423556,-0.0763945037500033,-0.05615310200706333,-0.06735140813781726,37.0 +0.04170844488444244,-0.044641636506989144,-0.06440780612537028,0.03564378941743375,0.01219056876179996,-0.05799374901012452,0.18117906039727852,-0.0763945037500033,-0.0006117353045626216,-0.05078298047847944,170.0 +0.06350367559055897,0.05068011873981862,-0.02560657146566148,0.011543832586488917,0.06447677737344255,0.048476727998317336,0.030231910429713918,-0.002592261998183278,0.038393928263466416,0.019632837073706312,170.0 +-0.07090024709715959,-0.044641636506989144,-0.004050329988045492,-0.04009893205125,-0.06623874415566393,-0.07866154748823384,0.05232173725423556,-0.0763945037500033,-0.05140387304727299,-0.03421455281914162,61.0 +-0.04183993948900423,0.05068011873981862,0.004572166603000912,-0.05387033595464705,-0.04422349842444599,-0.02730519975475012,-0.08021722369289432,0.07120997975363674,0.03664373256235367,0.019632837073706312,144.0 +-0.027309785684926546,0.05068011873981862,-0.007283766209687899,-0.04009893205125,-0.011200629827619093,-0.013839815897800126,0.059685012862409445,-0.03949338287409329,-0.08237869071592514,-0.025930338989472702,52.0 +-0.03457486258696539,-0.044641636506989144,-0.03746250427835029,-0.060756037906345574,0.020446285911006685,0.043466352609684754,-0.01394774321932938,-0.002592261998183278,-0.030747917533098208,-0.07149351505265171,128.0 +0.06713621404157838,0.05068011873981862,-0.02560657146566148,-0.04009893205125,-0.06348683843926169,-0.05987263978086174,-0.002902829807068556,-0.03949338287409329,-0.019198449026275908,0.0113486232440374,71.0 +-0.045472477940023646,0.05068011873981862,-0.02452875939178067,0.059743746248378575,0.005310804470794357,0.014969842586837095,-0.05444575906428573,0.07120997975363674,0.04234098419358016,0.015490730158871856,163.0 +-0.009147093429829445,0.05068011873981862,-0.018061886948495892,-0.03321323009955148,-0.02083229983502694,0.012151506430731283,-0.07285394808472044,0.07120997975363674,0.00027247814860377354,0.019632837073706312,150.0 +0.04170844488444244,0.05068011873981862,-0.014828450726853487,-0.017135116042335426,-0.005696818394814609,0.008393724889256856,-0.01394774321932938,-0.0018542395806650938,-0.011896851335695978,0.0030644094143684884,97.0 +0.038075906433423026,0.05068011873981862,-0.029917819761184662,-0.04009893205125,-0.033215875558837024,-0.02417371513685477,-0.010266105415242439,-0.002592261998183278,-0.012908683225401873,0.0030644094143684884,160.0 +0.016280675727306498,-0.044641636506989144,-0.04608500086939666,-0.00567042229275739,-0.07587041416307178,-0.06143838208980942,-0.01394774321932938,-0.03949338287409329,-0.05140387304727299,0.019632837073706312,178.0 +-0.0018820165277906047,-0.044641636506989144,-0.06979686649477428,-0.012556124244455912,-0.00019300696201012598,-0.009142588970957101,0.07072992627467027,-0.03949338287409329,-0.06291687914365544,0.040343371647878594,48.0 +-0.0018820165277906047,-0.044641636506989144,0.03367309259778249,0.12515791478951455,0.024574144485610048,0.02624318721126033,-0.010266105415242439,-0.002592261998183278,0.02671684132010462,0.06105390622205087,270.0 +0.06350367559055897,0.05068011873981862,-0.004050329988045492,-0.012556124244455912,0.10300345740307394,0.04878987646010685,0.05600337505832251,-0.002592261998183278,0.08449153066204618,-0.01764612515980379,202.0 +0.012648137276287077,0.05068011873981862,-0.020217511096257485,-0.002227571316908129,0.03833367306762126,0.05317395492516036,-0.006584467611155497,0.03430885887772673,-0.005142189801713891,-0.009361911330134878,111.0 +0.012648137276287077,0.05068011873981862,0.002416542455239321,0.056300895272529315,0.027326050202012293,0.017161881819363848,0.04127682384197474,-0.03949338287409329,0.0037090603325595967,0.07348022696655424,85.0 +-0.009147093429829445,0.05068011873981862,-0.03099563183506548,-0.02632752814785296,-0.011200629827619093,-0.0010007289644291908,-0.02131101882750326,-0.002592261998183278,0.006206735447689297,0.027917050903375224,42.0 +-0.030942324135945967,0.05068011873981862,0.028284032228378497,0.07007229917592636,-0.12678066991651324,-0.10684490904929198,-0.05444575906428573,-0.047980640675552584,-0.030747917533098208,0.015490730158871856,170.0 +-0.09632801625429555,-0.044641636506989144,-0.03638469220446948,-0.07452744180974262,-0.038719686991641515,-0.02761834821653966,0.01550535921336615,-0.03949338287409329,-0.07409260794346935,-0.0010776975004659671,200.0 +0.005383060374248237,-0.044641636506989144,-0.05794093368208547,-0.0228846771720037,-0.06761469701386505,-0.0683276482491792,-0.05444575906428573,-0.002592261998183278,0.04289703595278786,-0.08391983579715509,252.0 +-0.10359309315633439,-0.044641636506989144,-0.03746250427835029,-0.02632752814785296,0.0025588987543921156,0.019980217975469634,0.01182372140927921,-0.002592261998183278,-0.0683315470939731,-0.025930338989472702,113.0 +0.0707687524925978,-0.044641636506989144,0.012116851120166501,0.04252949136913227,0.07135654166444816,0.053487103386949876,0.05232173725423556,-0.002592261998183278,0.025395078941660074,-0.005219804415300423,143.0 +0.012648137276287077,0.05068011873981862,-0.022373135244019075,-0.029770379123702218,0.010814615903598841,0.02843522644378708,-0.02131101882750326,0.03430885887772673,-0.006081096870540014,-0.0010776975004659671,51.0 +-0.016412170331868287,-0.044641636506989144,-0.035306880130588664,-0.02632752814785296,0.03282986163481677,0.017161881819363848,0.10018302870736581,-0.03949338287409329,-0.07020936123162536,-0.07977772888232063,52.0 +-0.03820740103798481,-0.044641636506989144,0.009961226972404908,-0.04698463400294853,-0.05935897986465832,-0.05298337362149199,-0.010266105415242439,-0.03949338287409329,-0.015998872510179042,-0.042498766648810526,210.0 +0.001750521923228816,-0.044641636506989144,-0.039618128426111884,-0.10093410879450646,-0.029088016984233665,-0.03012353591085593,0.04495846164606168,-0.05019470792810719,-0.0683315470939731,-0.1294830118603341,65.0 +0.04534098333546186,-0.044641636506989144,0.07139651518361048,0.0012152796589411327,-0.009824676969417972,-0.0010007289644291908,0.01550535921336615,-0.03949338287409329,-0.041176166918895155,-0.07149351505265171,141.0 +-0.07090024709715959,0.05068011873981862,-0.07518592686417827,-0.04009893205125,-0.051103262715451604,-0.015092409744958261,-0.03971920784793797,-0.002592261998183278,-0.09643494994048712,-0.03421455281914162,55.0 +0.04534098333546186,-0.044641636506989144,-0.006205954135807083,0.011543832586488917,0.06310082451524143,0.01622243643399523,0.09650139090327886,-0.03949338287409329,0.04289703595278786,-0.03835665973397607,134.0 +-0.052737554842062495,0.05068011873981862,-0.040695940499992665,-0.06764173985804409,-0.0318399227006359,-0.037012802070225705,0.0375951860378878,-0.03949338287409329,-0.03452177701362266,0.06933812005171978,42.0 +-0.045472477940023646,-0.044641636506989144,-0.048240625017158284,-0.019441826196154435,-0.00019300696201012598,-0.016031855130326858,0.06704828847058333,-0.03949338287409329,-0.024795429028792802,0.019632837073706312,111.0 +0.012648137276287077,-0.044641636506989144,-0.02560657146566148,-0.04009893205125,-0.030463969842434782,-0.045154662076753616,0.07809320188284416,-0.0763945037500033,-0.07213275338232743,0.0113486232440374,98.0 +0.04534098333546186,-0.044641636506989144,0.05199589785375607,-0.05387033595464705,0.06310082451524143,0.06476044801137316,-0.010266105415242439,0.03430885887772673,0.037236246732001224,0.019632837073706312,164.0 +-0.020044708782887707,-0.044641636506989144,0.004572166603000912,0.09761510698272045,0.005310804470794357,-0.02072908205716988,0.06336665066649638,-0.03949338287409329,0.012551194864223063,0.0113486232440374,48.0 +-0.04910501639104307,-0.044641636506989144,-0.06440780612537028,-0.10207024961653671,-0.0029449126784123676,-0.015405558206747801,0.06336665066649638,-0.047242618258034386,-0.03324559264822791,-0.05492508739331389,96.0 +-0.07816532399919843,-0.044641636506989144,-0.01698407487461508,-0.012556124244455912,-0.00019300696201012598,-0.013526667436010586,0.07072992627467027,-0.03949338287409329,-0.041176166918895155,-0.092204049626824,90.0 +-0.07090024709715959,-0.044641636506989144,-0.05794093368208547,-0.08141314376144114,-0.04559945128264711,-0.02887094206369779,-0.04340084565202491,-0.002592261998183278,0.0011475759991601464,-0.005219804415300423,162.0 +0.056238598688520124,0.05068011873981862,0.009961226972404908,0.049415193320830796,-0.004320865536613489,-0.012274073588852453,-0.04340084565202491,0.03430885887772673,0.060790963876144,0.03205915781820968,150.0 +-0.027309785684926546,-0.044641636506989144,0.08864150836570328,-0.02519138732582271,0.02182223876920781,0.04252690722431616,-0.03235593223976409,0.03430885887772673,0.002861309289833047,0.07762233388138869,279.0 +0.001750521923228816,0.05068011873981862,-0.0051281420619263066,-0.012556124244455912,-0.015328488402222454,-0.013839815897800126,0.008142083605192267,-0.03949338287409329,-0.006081096870540014,-0.06735140813781726,92.0 +-0.0018820165277906047,-0.044641636506989144,-0.06440780612537028,0.011543832586488917,0.027326050202012293,0.03751653183568362,-0.01394774321932938,0.03430885887772673,0.011785484244986226,-0.05492508739331389,83.0 +0.016280675727306498,-0.044641636506989144,0.0175059114895705,-0.0228846771720037,0.06034891879883919,0.04440579799505339,0.030231910429713918,-0.002592261998183278,0.037236246732001224,-0.0010776975004659671,128.0 +0.016280675727306498,0.05068011873981862,-0.04500718879551588,0.06318659722422783,0.010814615903598841,-0.00037443204085011215,0.06336665066649638,-0.03949338287409329,-0.030747917533098208,0.036201264733044136,102.0 +-0.09269547780327612,-0.044641636506989144,0.028284032228378497,-0.015998975220305175,0.03695772020942014,0.02499059336410222,0.05600337505832251,-0.03949338287409329,-0.005142189801713891,-0.0010776975004659671,302.0 +0.059871137139539544,0.05068011873981862,0.04121777711494808,0.011543832586488917,0.041085578784023497,0.0707102687853743,-0.036037570043851025,0.03430885887772673,-0.010903250651210127,-0.03007244590430716,198.0 +-0.027309785684926546,-0.044641636506989144,0.06492964274032566,-0.002227571316908129,-0.0249601584096303,-0.017284448977484993,0.022868634821540033,-0.03949338287409329,-0.061175799045152635,-0.0632093012229828,95.0 +0.02354575262934534,0.05068011873981862,-0.0320734439089463,-0.04009893205125,-0.0318399227006359,-0.0216685274425385,-0.01394774321932938,-0.002592261998183278,-0.010903250651210127,0.019632837073706312,53.0 +-0.09632801625429555,-0.044641636506989144,-0.07626373893805906,-0.04354178302709927,-0.04559945128264711,-0.03482076283769895,0.008142083605192267,-0.03949338287409329,-0.05947118135708968,-0.08391983579715509,134.0 +0.027178291080364757,-0.044641636506989144,0.04984027370599448,-0.0550064767766773,-0.0029449126784123676,0.04064801645357896,-0.05812739686837268,0.05275941931568174,-0.05296264109357657,-0.005219804415300423,144.0 +0.01991321417832592,0.05068011873981862,0.045529025410471304,0.029894228287765473,-0.062110885581060565,-0.0558017097775978,-0.07285394808472044,0.026928634702544724,0.0456043699279467,0.040343371647878594,232.0 +0.038075906433423026,0.05068011873981862,-0.00943939035744949,0.002351420480971383,0.001182945896190995,0.03751653183568362,-0.05444575906428573,0.05017634085436802,-0.025953110560258,0.1066170822852299,81.0 +0.04170844488444244,0.05068011873981862,-0.0320734439089463,-0.0228846771720037,-0.04972730985725048,-0.04014428668812105,0.030231910429713918,-0.03949338287409329,-0.12609712083330468,0.015490730158871856,104.0 +0.01991321417832592,-0.044641636506989144,0.004572166603000912,-0.02632752814785296,0.02319819162740893,0.01027261565999407,0.06704828847058333,-0.03949338287409329,-0.02364686309993755,-0.04664087356364498,59.0 +-0.08543040090123728,-0.044641636506989144,0.020739347711212906,-0.02632752814785296,0.005310804470794357,0.019667069513680118,-0.002902829807068556,-0.002592261998183278,-0.02364686309993755,0.0030644094143684884,246.0 +0.01991321417832592,0.05068011873981862,0.014272475267928093,0.06318659722422783,0.014942474478202204,0.020293366437259194,-0.04708248345611185,0.03430885887772673,0.04666177983070229,0.09004865462589207,297.0 +0.02354575262934534,-0.044641636506989144,0.11019774984331929,0.06318659722422783,0.013566521620001083,-0.03294187206696173,-0.024992656631590206,0.020655444153640023,0.09924057568496533,0.02377494398854077,258.0 +-0.030942324135945967,0.05068011873981862,0.0013387303813585058,-0.00567042229275739,0.06447677737344255,0.04941617338368593,-0.04708248345611185,0.10811110062954676,0.08379874486368903,0.0030644094143684884,229.0 +0.04897352178648128,0.05068011873981862,0.05846277029704089,0.07007229917592636,0.013566521620001083,0.020606514899048713,-0.02131101882750326,0.03430885887772673,0.02200407477075404,0.027917050903375224,275.0 +0.059871137139539544,-0.044641636506989144,-0.0212953231701383,0.08728655405517267,0.045213437358626866,0.031566711061682434,-0.04708248345611185,0.07120997975363674,0.07912244072477838,0.13561183068907107,281.0 +-0.056370093293081916,0.05068011873981862,-0.010517202431330305,0.02531523648988596,0.02319819162740893,0.04002171952999989,-0.03971920784793797,0.03430885887772673,0.02060938757142981,0.05691179930721642,179.0 +0.016280675727306498,-0.044641636506989144,-0.047162812943277475,-0.002227571316908129,-0.019456346976825818,-0.04296262284422686,0.033913548233800855,-0.03949338287409329,0.02736404910541198,0.027917050903375224,200.0 +-0.04910501639104307,-0.044641636506989144,0.004572166603000912,0.011543832586488917,-0.037343734133440394,-0.01853704282464315,-0.01762938102341632,-0.002592261998183278,-0.03980882652740082,-0.021788232074638245,200.0 +0.06350367559055897,-0.044641636506989144,0.0175059114895705,0.0218723855140367,0.0080627101871966,0.02154596028441731,-0.036037570043851025,0.03430885887772673,0.019907486170462722,0.0113486232440374,173.0 +0.04897352178648128,0.05068011873981862,0.08109682384853766,0.0218723855140367,0.04383748450042574,0.06413415108779408,-0.05444575906428573,0.07120997975363674,0.03243232415655107,0.0486275854775475,180.0 +0.005383060374248237,0.05068011873981862,0.03475090467166331,-0.0010914304948778783,0.15253776029831428,0.19878798965729408,-0.06180903467245962,0.18523444326019867,0.015568459328120622,0.07348022696655424,84.0 +-0.005514554978810025,-0.044641636506989144,0.023972783932855315,0.008100981610639655,-0.034591828417038145,-0.038891692840962916,0.022868634821540033,-0.03949338287409329,-0.015998872510179042,-0.013504018244969336,121.0 +-0.005514554978810025,0.05068011873981862,-0.008361578283568675,-0.002227571316908129,-0.033215875558837024,-0.06363042132233616,-0.036037570043851025,-0.002592261998183278,0.0805900527449823,0.007206516329202944,161.0 +-0.0890629393522567,-0.044641636506989144,-0.061174369903727877,-0.02632752814785296,-0.05523112129005496,-0.054549115930439665,0.04127682384197474,-0.0763945037500033,-0.09393727482535742,-0.05492508739331389,99.0 +0.0344433679824036,0.05068011873981862,-0.0018947058402839008,-0.012556124244455912,0.03833367306762126,0.013717248739678958,0.07809320188284416,-0.03949338287409329,0.004547695772676124,-0.09634615654165846,109.0 +-0.052737554842062495,-0.044641636506989144,-0.06225218197760866,-0.02632752814785296,-0.005696818394814609,-0.005071658967693135,0.030231910429713918,-0.03949338287409329,-0.030747917533098208,-0.07149351505265171,115.0 +0.009015598825267658,-0.044641636506989144,0.016428099415689682,0.004658130634790395,0.00943866304539772,0.01058576412178361,-0.028674294435677143,0.03430885887772673,0.03896821122789408,0.11904340302973325,268.0 +-0.06363517019512076,0.05068011873981862,0.09618619288286882,0.10450080893441897,-0.0029449126784123676,-0.0047585105059035964,-0.006584467611155497,-0.002592261998183278,0.022687744966501246,0.07348022696655424,274.0 +-0.09632801625429555,-0.044641636506989144,-0.06979686649477428,-0.06764173985804409,-0.019456346976825818,-0.010708331279904778,0.01550535921336615,-0.03949338287409329,-0.04688253415273158,-0.07977772888232063,158.0 +0.016280675727306498,0.05068011873981862,-0.0212953231701383,-0.009113273268606652,0.0342058144930179,0.047850431074738256,0.0007788079970183853,-0.002592261998183278,-0.012908683225401873,0.02377494398854077,107.0 +-0.04183993948900423,0.05068011873981862,-0.05362968538656229,-0.04009893205125,-0.0841261313122785,-0.07177228132886408,-0.002902829807068556,-0.03949338287409329,-0.07213275338232743,-0.03007244590430716,83.0 +-0.074532785548179,-0.044641636506989144,0.04337340126270967,-0.03321323009955148,0.01219056876179996,0.00025186488272894433,0.06336665066649638,-0.03949338287409329,-0.02712902329694316,-0.04664087356364498,103.0 +-0.005514554978810025,-0.044641636506989144,0.0563071461492793,-0.03665608107540074,-0.04835135699904936,-0.04296262284422686,-0.07285394808472044,0.03799897096531772,0.050782032210405316,0.05691179930721642,272.0 +-0.09269547780327612,-0.044641636506989144,-0.08165279930746305,-0.057313186930496314,-0.060734932722859444,-0.06801449978738965,0.04864009945014862,-0.0763945037500033,-0.06649019536676071,-0.021788232074638245,85.0 +0.005383060374248237,-0.044641636506989144,0.04984027370599448,0.09761510698272045,-0.015328488402222454,-0.016345003592116398,-0.006584467611155497,-0.002592261998183278,0.017036071348324546,-0.013504018244969336,280.0 +0.0344433679824036,0.05068011873981862,0.11127556191720007,0.07695800112762488,-0.0318399227006359,-0.033881317452330355,-0.02131101882750326,-0.002592261998183278,0.02802037249332928,0.07348022696655424,336.0 +0.02354575262934534,-0.044641636506989144,0.061696206518683294,0.052858044296680055,-0.034591828417038145,-0.048912443618228024,-0.028674294435677143,-0.002592261998183278,0.05471997253790904,-0.005219804415300423,281.0 +0.04170844488444244,0.05068011873981862,0.014272475267928093,0.04252949136913227,-0.030463969842434782,-0.0013138774262187302,-0.04340084565202491,-0.002592261998183278,-0.03324559264822791,0.015490730158871856,118.0 +-0.027309785684926546,-0.044641636506989144,0.04768464955823289,-0.04698463400294853,0.0342058144930179,0.057244884928424306,-0.08021722369289432,0.13025177315509276,0.04506654937395887,0.13146972377423663,317.0 +0.04170844488444244,0.05068011873981862,0.012116851120166501,0.03908664039328301,0.05484510736603471,0.04440579799505339,0.0044604458011053266,-0.002592261998183278,0.0456043699279467,-0.0010776975004659671,235.0 +-0.030942324135945967,-0.044641636506989144,0.005649978676881689,-0.009113273268606652,0.019070333052805567,0.006827982580309182,0.07441156407875721,-0.03949338287409329,-0.041176166918895155,-0.042498766648810526,60.0 +0.03081082953138418,0.05068011873981862,0.04660683748435208,-0.015998975220305175,0.020446285911006685,0.05066876723084409,-0.05812739686837268,0.07120997975363674,0.006206735447689297,0.007206516329202944,174.0 +-0.04183993948900423,-0.044641636506989144,0.12852055509929283,0.06318659722422783,-0.033215875558837024,-0.03262872360517222,0.01182372140927921,-0.03949338287409329,-0.015998872510179042,-0.05078298047847944,259.0 +-0.030942324135945967,0.05068011873981862,0.05954058237092167,0.0012152796589411327,0.01219056876179996,0.031566711061682434,-0.04340084565202491,0.03430885887772673,0.014820979914103747,0.007206516329202944,178.0 +-0.056370093293081916,-0.044641636506989144,0.09295275666122646,-0.019441826196154435,0.014942474478202204,0.023424851055154544,-0.028674294435677143,0.02545258986750832,0.026060517932187322,0.040343371647878594,128.0 +-0.06000263174410134,0.05068011873981862,0.015350287341808908,-0.019441826196154435,0.03695772020942014,0.04816357953652778,0.019186997017453092,-0.002592261998183278,-0.030747917533098208,-0.0010776975004659671,96.0 +-0.04910501639104307,0.05068011873981862,-0.0051281420619263066,-0.04698463400294853,-0.02083229983502694,-0.02041593359538034,-0.0691723102806335,0.07120997975363674,0.06123762840403217,-0.03835665973397607,126.0 +0.02354575262934534,-0.044641636506989144,0.07031870310972965,0.02531523648988596,-0.034591828417038145,-0.014466112821379181,-0.03235593223976409,-0.002592261998183278,-0.019198449026275908,-0.009361911330134878,288.0 +0.001750521923228816,-0.044641636506989144,-0.004050329988045492,-0.00567042229275739,-0.008448724111216851,-0.02386056667506523,0.05232173725423556,-0.03949338287409329,-0.00894339609006817,-0.013504018244969336,88.0 +-0.03457486258696539,0.05068011873981862,-0.0008168937664030856,0.07007229917592636,0.039709625925822375,0.06695248724389988,-0.06549067247654655,0.10811110062954676,0.02671684132010462,0.07348022696655424,292.0 +0.04170844488444244,0.05068011873981862,-0.04392937672163507,0.06318659722422783,-0.004320865536613489,0.01622243643399523,-0.01394774321932938,-0.002592261998183278,-0.03452177701362266,0.0113486232440374,71.0 +0.06713621404157838,0.05068011873981862,0.020739347711212906,-0.00567042229275739,0.020446285911006685,0.02624318721126033,-0.002902829807068556,-0.002592261998183278,0.008640601344549246,0.0030644094143684884,197.0 +-0.027309785684926546,0.05068011873981862,0.06061839444480248,0.049415193320830796,0.08511607024645937,0.08636769187485104,-0.002902829807068556,0.03430885887772673,0.037810529696428806,0.0486275854775475,186.0 +-0.016412170331868287,-0.044641636506989144,-0.010517202431330305,0.0012152796589411327,-0.037343734133440394,-0.035760208223067566,0.01182372140927921,-0.03949338287409329,-0.021395309255276825,-0.03421455281914162,25.0 +-0.0018820165277906047,0.05068011873981862,-0.033151255982827074,-0.018305685374124185,0.03145390877661565,0.042840055686105716,-0.01394774321932938,0.019917421736121838,0.010226716198682649,0.027917050903375224,84.0 +-0.012779631880848867,-0.044641636506989144,-0.06548561819925106,-0.0699484500118631,0.001182945896190995,0.016848733357574308,-0.002902829807068556,-0.007020396503292483,-0.030747917533098208,-0.05078298047847944,96.0 +-0.005514554978810025,-0.044641636506989144,0.04337340126270967,0.08728655405517267,0.013566521620001083,0.0071411310420987206,-0.01394774321932938,-0.002592261998183278,0.04234098419358016,-0.01764612515980379,195.0 +-0.009147093429829445,-0.044641636506989144,-0.06225218197760866,-0.07452744180974262,-0.02358420555142918,-0.013213518974221048,0.0044604458011053266,-0.03949338287409329,-0.0358161925842373,-0.04664087356364498,53.0 +-0.045472477940023646,0.05068011873981862,0.06385183066644486,0.07007229917592636,0.1332744202834986,0.1314610703725441,-0.03971920784793797,0.10811110062954676,0.07574055215648227,0.0859065477110576,217.0 +-0.052737554842062495,-0.044641636506989144,0.030439656376140087,-0.07452744180974262,-0.02358420555142918,-0.011334628203483833,-0.002902829807068556,-0.002592261998183278,-0.030747917533098208,-0.0010776975004659671,172.0 +0.016280675727306498,0.05068011873981862,0.0724743272574913,0.07695800112762488,-0.008448724111216851,0.0055753887331510465,-0.006584467611155497,-0.002592261998183278,-0.02364686309993755,0.06105390622205087,131.0 +0.04534098333546186,-0.044641636506989144,-0.01913969902237667,0.0218723855140367,0.027326050202012293,-0.013526667436010586,0.10018302870736581,-0.03949338287409329,0.017765319557121535,-0.013504018244969336,214.0 +-0.04183993948900423,-0.044641636506989144,-0.06656343027313188,-0.04698463400294853,-0.037343734133440394,-0.043275771306016404,0.04864009945014862,-0.03949338287409329,-0.05615310200706333,-0.013504018244969336,59.0 +-0.056370093293081916,0.05068011873981862,-0.06009655782984706,-0.03665608107540074,-0.08825398988688185,-0.07083283594349546,-0.01394774321932938,-0.03949338287409329,-0.07813993550229265,-0.10463037037132736,70.0 +0.0707687524925978,-0.044641636506989144,0.06924089103584885,0.03795049957125276,0.02182223876920781,0.0015044587298871017,-0.036037570043851025,0.03910600459159503,0.07763659749935449,0.1066170822852299,220.0 +0.001750521923228816,0.05068011873981862,0.05954058237092167,-0.002227571316908129,0.06172487165704031,0.0631947057024255,-0.05812739686837268,0.10811110062954676,0.0689858906225002,0.12732761685940217,268.0 +-0.0018820165277906047,-0.044641636506989144,-0.0266843835395423,0.049415193320830796,0.05897296594063807,-0.016031855130326858,-0.04708248345611185,0.07120997975363674,0.13359728192191356,0.019632837073706312,152.0 +0.02354575262934534,0.05068011873981862,-0.020217511096257485,-0.03665608107540074,-0.013952535544021335,-0.015092409744958261,0.059685012862409445,-0.03949338287409329,-0.09643494994048712,-0.01764612515980379,47.0 +-0.020044708782887707,-0.044641636506989144,-0.04608500086939666,-0.09862739864068745,-0.07587041416307178,-0.05987263978086174,-0.01762938102341632,-0.03949338287409329,-0.05140387304727299,-0.04664087356364498,74.0 +0.04170844488444244,0.05068011873981862,0.07139651518361048,0.008100981610639655,0.03833367306762126,0.01590928797220569,-0.01762938102341632,0.03430885887772673,0.07340695788833193,0.0859065477110576,295.0 +-0.06363517019512076,0.05068011873981862,-0.07949717515970146,-0.00567042229275739,-0.07174255558846841,-0.06644875747844198,-0.010266105415242439,-0.03949338287409329,-0.018113692315690322,-0.05492508739331389,101.0 +0.016280675727306498,0.05068011873981862,0.009961226972404908,-0.04354178302709927,-0.09650970703608859,-0.09463211903950011,-0.03971920784793797,-0.03949338287409329,0.017036071348324546,0.007206516329202944,151.0 +0.06713621404157838,-0.044641636506989144,-0.03854031635223107,-0.02632752814785296,-0.0318399227006359,-0.02636575436938152,0.008142083605192267,-0.03949338287409329,-0.02712902329694316,0.0030644094143684884,127.0 +0.04534098333546186,0.05068011873981862,0.01966153563733209,0.03908664039328301,0.020446285911006685,0.025930038749470818,0.008142083605192267,-0.002592261998183278,-0.003300838074501491,0.019632837073706312,237.0 +0.04897352178648128,-0.044641636506989144,0.027206220154497678,-0.02519138732582271,0.02319819162740893,0.018414475666521983,-0.06180903467245962,0.08006624876385515,0.07222192954903682,0.03205915781820968,225.0 +0.04170844488444244,-0.044641636506989144,-0.008361578283568675,-0.02632752814785296,0.024574144485610048,0.01622243643399523,0.07072992627467027,-0.03949338287409329,-0.04835926177554553,-0.03007244590430716,81.0 +-0.02367724723390713,-0.044641636506989144,-0.015906262800734303,-0.012556124244455912,0.020446285911006685,0.04127431337715804,-0.04340084565202491,0.03430885887772673,0.014073500500086794,-0.009361911330134878,151.0 +-0.03820740103798481,0.05068011873981862,0.004572166603000912,0.03564378941743375,-0.011200629827619093,0.0058885371949405855,-0.04708248345611185,0.03430885887772673,0.016306823139527554,-0.0010776975004659671,107.0 +0.04897352178648128,-0.044641636506989144,-0.04285156464775429,-0.05387033595464705,0.045213437358626866,0.05004247030726501,0.033913548233800855,-0.002592261998183278,-0.025953110560258,-0.0632093012229828,64.0 +0.04534098333546186,0.05068011873981862,0.005649978676881689,0.056300895272529315,0.06447677737344255,0.08918602803095686,-0.03971920784793797,0.07120997975363674,0.015568459328120622,-0.009361911330134878,138.0 +0.04534098333546186,0.05068011873981862,-0.035306880130588664,0.06318659722422783,-0.004320865536613489,-0.0016270258880082473,-0.010266105415242439,-0.002592261998183278,0.015568459328120622,0.05691179930721642,185.0 +0.016280675727306498,-0.044641636506989144,0.023972783932855315,-0.0228846771720037,-0.0249601584096303,-0.02605260590759198,-0.03235593223976409,-0.002592261998183278,0.037236246732001224,0.03205915781820968,265.0 +-0.074532785548179,0.05068011873981862,-0.018061886948495892,0.008100981610639655,-0.019456346976825818,-0.02480001206043385,-0.06549067247654655,0.03430885887772673,0.06731773534487707,-0.01764612515980379,101.0 +-0.08179786245021785,0.05068011873981862,0.0422955891888289,-0.019441826196154435,0.039709625925822375,0.05755803339021383,-0.0691723102806335,0.10811110062954676,0.04719048478208009,-0.03835665973397607,137.0 +-0.06726770864614018,-0.044641636506989144,-0.05470749746044306,-0.02632752814785296,-0.07587041416307178,-0.08210618056791873,0.04864009945014862,-0.0763945037500033,-0.08682710478958679,-0.10463037037132736,143.0 +0.005383060374248237,-0.044641636506989144,-0.002972517914164677,0.049415193320830796,0.0741084473808504,0.0707102687853743,0.04495846164606168,-0.002592261998183278,-0.0014959487577289358,-0.009361911330134878,141.0 +-0.0018820165277906047,-0.044641636506989144,-0.06656343027313188,0.0012152796589411327,-0.0029449126784123676,0.003070201038834776,0.01182372140927921,-0.002592261998183278,-0.020292321339471356,-0.025930338989472702,79.0 +0.009015598825267658,-0.044641636506989144,-0.012672826579091896,0.028758087465735226,-0.018080394118624697,-0.005071658967693135,-0.04708248345611185,0.03430885887772673,0.02337141516224845,-0.005219804415300423,292.0 +-0.005514554978810025,0.05068011873981862,-0.041773752573873474,-0.04354178302709927,-0.07999827273767514,-0.07615635979391756,-0.03235593223976409,-0.03949338287409329,0.010226716198682649,-0.009361911330134878,178.0 +0.056238598688520124,0.05068011873981862,-0.03099563183506548,0.008100981610639655,0.019070333052805567,0.02123281182262779,0.033913548233800855,-0.03949338287409329,-0.02952642678336326,-0.05906719430814835,91.0 +0.009015598825267658,0.05068011873981862,-0.0051281420619263066,-0.06419888888219483,0.06998058880624704,0.08386250418053477,-0.03971920784793797,0.07120997975363674,0.03954249419232167,0.019632837073706312,116.0 +-0.06726770864614018,-0.044641636506989144,-0.05901874575596628,0.03220093844158448,-0.051103262715451604,-0.049538740541807104,-0.010266105415242439,-0.03949338287409329,0.0020044426444966374,0.02377494398854077,86.0 +0.027178291080364757,0.05068011873981862,0.02505059600673609,0.014986683562338177,0.02595009734381117,0.048476727998317336,-0.03971920784793797,0.03430885887772673,0.007838428314872565,0.02377494398854077,122.0 +-0.02367724723390713,-0.044641636506989144,-0.04608500086939666,-0.03321323009955148,0.03282986163481677,0.03626393798852546,0.0375951860378878,-0.002592261998183278,-0.03324559264822791,0.0113486232440374,72.0 +0.04897352178648128,0.05068011873981862,0.0034943545291200974,0.07007229917592636,-0.008448724111216851,0.013404100277889418,-0.05444575906428573,0.03430885887772673,0.013316905483459898,0.036201264733044136,129.0 +-0.052737554842062495,-0.044641636506989144,0.05415152200151766,-0.02632752814785296,-0.05523112129005496,-0.033881317452330355,-0.01394774321932938,-0.03949338287409329,-0.07409260794346935,-0.05906719430814835,142.0 +0.04170844488444244,-0.044641636506989144,-0.04500718879551588,0.03450764859540349,0.04383748450042574,-0.015718706668537318,0.0375951860378878,-0.014400620678474476,0.0898970830097539,0.007206516329202944,90.0 +0.056238598688520124,-0.044641636506989144,-0.05794093368208547,-0.0079771324465764,0.05209320164963247,0.049103024921896415,0.05600337505832251,-0.021411833644897377,-0.028323167238848198,0.044485478562713045,158.0 +-0.03457486258696539,0.05068011873981862,-0.05578530953432388,-0.015998975220305175,-0.009824676969417972,-0.007889995123798945,0.0375951860378878,-0.03949338287409329,-0.05296264109357657,0.027917050903375224,39.0 +0.08166636784565606,0.05068011873981862,0.0013387303813585058,0.03564378941743375,0.126394655992493,0.09106491880169407,0.019186997017453092,0.03430885887772673,0.08449153066204618,-0.03007244590430716,196.0 +-0.0018820165277906047,0.05068011873981862,0.030439656376140087,0.052858044296680055,0.039709625925822375,0.056618588004845226,-0.03971920784793797,0.07120997975363674,0.025395078941660074,0.027917050903375224,222.0 +0.11072667545381144,0.05068011873981862,0.006727790750762504,0.028758087465735226,-0.027712064126032544,-0.007263698200219888,-0.04708248345611185,0.03430885887772673,0.0020044426444966374,0.07762233388138869,277.0 +-0.030942324135945967,-0.044641636506989144,0.04660683748435208,0.014986683562338177,-0.016704441260423575,-0.047033552847490806,0.0007788079970183853,-0.002592261998183278,0.06345271983825305,-0.025930338989472702,99.0 +0.001750521923228816,0.05068011873981862,0.026128408080616904,-0.009113273268606652,0.024574144485610048,0.03845597722105221,-0.02131101882750326,0.03430885887772673,0.009433658771615987,0.0030644094143684884,196.0 +0.009015598825267658,-0.044641636506989144,0.045529025410471304,0.028758087465735226,0.01219056876179996,-0.013839815897800126,0.026550272625626974,-0.03949338287409329,0.04613307487932449,0.036201264733044136,202.0 +0.03081082953138418,-0.044641636506989144,0.04013996504106731,0.07695800112762488,0.017694380194604446,0.037829680297473134,-0.028674294435677143,0.03430885887772673,-0.0014959487577289358,0.11904340302973325,155.0 +0.038075906433423026,0.05068011873981862,-0.018061886948495892,0.0666294482000771,-0.051103262715451604,-0.016658152053905938,-0.07653558588880739,0.03430885887772673,-0.011896851335695978,-0.013504018244969336,77.0 +0.009015598825267658,-0.044641636506989144,0.014272475267928093,0.014986683562338177,0.05484510736603471,0.04722413415115918,0.07072992627467027,-0.03949338287409329,-0.03324559264822791,-0.05906719430814835,191.0 +0.09256398319871433,-0.044641636506989144,0.0369065288194249,0.0218723855140367,-0.0249601584096303,-0.016658152053905938,0.0007788079970183853,-0.03949338287409329,-0.022516528376302174,-0.021788232074638245,70.0 +0.06713621404157838,-0.044641636506989144,0.0034943545291200974,0.03564378941743375,0.04934129593323023,0.03125356259989291,0.07072992627467027,-0.03949338287409329,-0.0006117353045626216,0.019632837073706312,73.0 +0.001750521923228816,-0.044641636506989144,-0.07087467856865506,-0.0228846771720037,-0.001568959820211247,-0.0010007289644291908,0.026550272625626974,-0.03949338287409329,-0.022516528376302174,0.007206516329202944,49.0 +0.03081082953138418,-0.044641636506989144,-0.033151255982827074,-0.0228846771720037,-0.046975404140848234,-0.08116673518255012,0.10386466651145274,-0.0763945037500033,-0.03980882652740082,-0.05492508739331389,65.0 +0.027178291080364757,0.05068011873981862,0.09403056873510728,0.09761510698272045,-0.034591828417038145,-0.032002426681593144,-0.04340084565202491,-0.002592261998183278,0.03664373256235367,0.1066170822852299,263.0 +0.012648137276287077,0.05068011873981862,0.03582871674554409,0.049415193320830796,0.053469154507833586,0.07415490186505921,-0.0691723102806335,0.14501222150545676,0.0456043699279467,0.0486275854775475,248.0 +0.07440129094361722,-0.044641636506989144,0.031517468450020895,0.10105795795856971,0.04658939021682799,0.03689023491210454,0.01550535921336615,-0.002592261998183278,0.033653814906286016,0.044485478562713045,296.0 +-0.04183993948900423,-0.044641636506989144,-0.06548561819925106,-0.04009893205125,-0.005696818394814609,0.014343545663258015,-0.04340084565202491,0.03430885887772673,0.007027139682585861,-0.013504018244969336,214.0 +-0.0890629393522567,-0.044641636506989144,-0.041773752573873474,-0.019441826196154435,-0.06623874415566393,-0.07427746902318036,0.008142083605192267,-0.03949338287409329,0.0011475759991601464,-0.03007244590430716,185.0 +0.02354575262934534,0.05068011873981862,-0.039618128426111884,-0.00567042229275739,-0.04835135699904936,-0.033255020528751275,0.01182372140927921,-0.03949338287409329,-0.10163995903077562,-0.06735140813781726,78.0 +-0.045472477940023646,-0.044641636506989144,-0.03854031635223107,-0.02632752814785296,-0.015328488402222454,0.0008781618063080231,-0.03235593223976409,-0.002592261998183278,0.0011475759991601464,-0.03835665973397607,93.0 +-0.02367724723390713,0.05068011873981862,-0.02560657146566148,0.04252949136913227,-0.05385516843185383,-0.047659849771069886,-0.02131101882750326,-0.03949338287409329,0.0011475759991601464,0.019632837073706312,252.0 +-0.09996055470531495,-0.044641636506989144,-0.023450947317899894,-0.06419888888219483,-0.0579830270064572,-0.06018578824265128,0.01182372140927921,-0.03949338287409329,-0.018113692315690322,-0.05078298047847944,150.0 +-0.027309785684926546,-0.044641636506989144,-0.06656343027313188,-0.11239880254408448,-0.04972730985725048,-0.041396880535279186,0.0007788079970183853,-0.03949338287409329,-0.0358161925842373,-0.009361911330134878,77.0 +0.03081082953138418,0.05068011873981862,0.032595280523901676,0.049415193320830796,-0.04009563984984263,-0.04358891976780594,-0.0691723102806335,0.03430885887772673,0.06301517091297482,0.0030644094143684884,208.0 +-0.10359309315633439,0.05068011873981862,-0.04608500086939666,-0.02632752814785296,-0.0249601584096303,-0.02480001206043385,0.030231910429713918,-0.03949338287409329,-0.03980882652740082,-0.05492508739331389,77.0 +0.06713621404157838,0.05068011873981862,-0.029917819761184662,0.05743703609455957,-0.00019300696201012598,-0.015718706668537318,0.07441156407875721,-0.05056371913686628,-0.03845971734112638,0.007206516329202944,108.0 +-0.052737554842062495,-0.044641636506989144,-0.012672826579091896,-0.060756037906345574,-0.00019300696201012598,0.008080576427467316,0.01182372140927921,-0.002592261998183278,-0.02712902329694316,-0.05078298047847944,160.0 +-0.027309785684926546,0.05068011873981862,-0.015906262800734303,-0.029770379123702218,0.003934851612593237,-0.0006875805026396515,0.04127682384197474,-0.03949338287409329,-0.02364686309993755,0.0113486232440374,53.0 +-0.03820740103798481,0.05068011873981862,0.07139651518361048,-0.057313186930496314,0.15391371315651542,0.1558866503921278,0.0007788079970183853,0.07194800217115493,0.050280674066857343,0.06933812005171978,220.0 +0.009015598825267658,-0.044641636506989144,-0.03099563183506548,0.0218723855140367,0.0080627101871966,0.008706873351046395,0.0044604458011053266,-0.002592261998183278,0.009433658771615987,0.0113486232440374,154.0 +0.012648137276287077,0.05068011873981862,0.00026091830747769084,-0.011419983422425662,0.039709625925822375,0.057244884928424306,-0.03971920784793797,0.056080520194513636,0.024055085357995654,0.03205915781820968,259.0 +0.06713621404157838,-0.044641636506989144,0.0369065288194249,-0.05042748497879779,-0.02358420555142918,-0.034507614375909414,0.04864009945014862,-0.03949338287409329,-0.025953110560258,-0.03835665973397607,90.0 +0.04534098333546186,-0.044641636506989144,0.039062152967186486,0.04597234234498153,0.006686757328995478,-0.02417371513685477,0.008142083605192267,-0.012555564634678981,0.06432781768880942,0.05691179930721642,246.0 +0.06713621404157838,0.05068011873981862,-0.014828450726853487,0.05860760542634833,-0.05935897986465832,-0.034507614375909414,-0.06180903467245962,0.012906208769698923,-0.005142189801713891,0.0486275854775475,124.0 +0.027178291080364757,-0.044641636506989144,0.006727790750762504,0.03564378941743375,0.07961225881365488,0.0707102687853743,0.01550535921336615,0.03430885887772673,0.04067282891595704,0.0113486232440374,67.0 +0.056238598688520124,-0.044641636506989144,-0.06871905442089347,-0.06877788068007434,-0.00019300696201012598,-0.0010007289644291908,0.04495846164606168,-0.03764832683029779,-0.04835926177554553,-0.0010776975004659671,72.0 +0.0344433679824036,0.05068011873981862,-0.00943939035744949,0.059743746248378575,-0.035967781275239266,-0.007576846662009428,-0.07653558588880739,0.07120997975363674,0.011010658023139448,-0.021788232074638245,257.0 +0.02354575262934534,-0.044641636506989144,0.01966153563733209,-0.012556124244455912,0.08374011738825825,0.03876912568284173,0.06336665066649638,-0.002592261998183278,0.06605066658209227,0.0486275854775475,262.0 +0.04897352178648128,0.05068011873981862,0.07462995140525285,0.0666294482000771,-0.009824676969417972,-0.002253322811587326,-0.04340084565202491,0.03430885887772673,0.033653814906286016,0.019632837073706312,275.0 +0.03081082953138418,0.05068011873981862,-0.008361578283568675,0.004658130634790395,0.014942474478202204,0.02749578105841849,0.008142083605192267,-0.008127430129569777,-0.02952642678336326,0.05691179930721642,177.0 +-0.10359309315633439,0.05068011873981862,-0.023450947317899894,-0.0228846771720037,-0.08687803702868073,-0.06770135132560012,-0.01762938102341632,-0.03949338287409329,-0.07813993550229265,-0.07149351505265171,71.0 +0.016280675727306498,0.05068011873981862,-0.04608500086939666,0.011543832586488917,-0.033215875558837024,-0.016031855130326858,-0.010266105415242439,-0.002592261998183278,-0.04398377252276359,-0.042498766648810526,47.0 +-0.06000263174410134,0.05068011873981862,0.05415152200151766,-0.019441826196154435,-0.04972730985725048,-0.048912443618228024,0.022868634821540033,-0.03949338287409329,-0.04398377252276359,-0.005219804415300423,187.0 +-0.027309785684926546,-0.044641636506989144,-0.035306880130588664,-0.029770379123702218,-0.05660707414825608,-0.0586200459337036,0.030231910429713918,-0.03949338287409329,-0.049872451808799324,-0.1294830118603341,125.0 +0.04170844488444244,-0.044641636506989144,-0.0320734439089463,-0.06189217872837582,0.07961225881365488,0.05098191569263361,0.05600337505832251,-0.009972486173365287,0.04506654937395887,-0.05906719430814835,78.0 +-0.08179786245021785,-0.044641636506989144,-0.08165279930746305,-0.04009893205125,0.0025588987543921156,-0.01853704282464315,0.07072992627467027,-0.03949338287409329,-0.010903250651210127,-0.092204049626824,51.0 +-0.04183993948900423,-0.044641636506989144,0.04768464955823289,0.059743746248378575,0.1277706088506941,0.1280164372928592,-0.024992656631590206,0.10811110062954676,0.0638902687635312,0.040343371647878594,258.0 +-0.012779631880848867,-0.044641636506989144,0.06061839444480248,0.052858044296680055,0.04796534307502911,0.02937467182915568,-0.01762938102341632,0.03430885887772673,0.07020738137223513,0.007206516329202944,215.0 +0.06713621404157838,-0.044641636506989144,0.0563071461492793,0.07351515015177562,-0.013952535544021335,-0.03920484130275244,-0.03235593223976409,-0.002592261998183278,0.07574055215648227,0.036201264733044136,303.0 +-0.052737554842062495,0.05068011873981862,0.09834181703063047,0.08728655405517267,0.06034891879883919,0.04878987646010685,-0.05812739686837268,0.10811110062954676,0.08449153066204618,0.040343371647878594,243.0 +0.005383060374248237,-0.044641636506989144,0.05954058237092167,-0.05617704610846606,0.024574144485610048,0.052860806463370796,-0.04340084565202491,0.05091436327188625,-0.00422151393810765,-0.03007244590430716,91.0 +0.08166636784565606,-0.044641636506989144,0.03367309259778249,0.008100981610639655,0.05209320164963247,0.056618588004845226,-0.01762938102341632,0.03430885887772673,0.03486619005341102,0.06933812005171978,150.0 +0.03081082953138418,0.05068011873981862,0.0563071461492793,0.07695800112762488,0.04934129593323023,-0.012274073588852453,-0.036037570043851025,0.07120997975363674,0.12005149644350945,0.09004865462589207,310.0 +0.001750521923228816,-0.044641636506989144,-0.06548561819925106,-0.00567042229275739,-0.007072771253015731,-0.019476488210011744,0.04127682384197474,-0.03949338287409329,-0.003300838074501491,0.007206516329202944,153.0 +-0.04910501639104307,-0.044641636506989144,0.16085491731571683,-0.04698463400294853,-0.029088016984233665,-0.019789636671801284,-0.04708248345611185,0.03430885887772673,0.02802037249332928,0.0113486232440374,346.0 +-0.027309785684926546,0.05068011873981862,-0.05578530953432388,0.02531523648988596,-0.007072771253015731,-0.02354741821327569,0.05232173725423556,-0.03949338287409329,-0.005142189801713891,-0.05078298047847944,63.0 +0.07803382939463664,0.05068011873981862,-0.02452875939178067,-0.04240564220506902,0.006686757328995478,0.052860806463370796,-0.0691723102806335,0.08080427118137334,-0.0371288393600719,0.05691179930721642,89.0 +0.012648137276287077,-0.044641636506989144,-0.03638469220446948,0.04252949136913227,-0.013952535544021335,0.01293437758520512,-0.026833475533633678,0.005156973385757823,-0.04398377252276359,0.007206516329202944,50.0 +0.04170844488444244,-0.044641636506989144,-0.008361578283568675,-0.057313186930496314,0.0080627101871966,-0.031376129758014064,0.151725957964583,-0.0763945037500033,-0.08023652410258396,-0.01764612515980379,39.0 +0.04897352178648128,-0.044641636506989144,-0.041773752573873474,0.10450080893441897,0.03558176735121902,-0.02573945744580244,0.17749742259319157,-0.0763945037500033,-0.012908683225401873,0.015490730158871856,103.0 +-0.016412170331868287,0.05068011873981862,0.1274427430254121,0.09761510698272045,0.016318427336403322,0.017475030281153364,-0.02131101882750326,0.03430885887772673,0.03486619005341102,0.0030644094143684884,308.0 +-0.074532785548179,0.05068011873981862,-0.07734155101193986,-0.04698463400294853,-0.046975404140848234,-0.03262872360517222,0.0044604458011053266,-0.03949338287409329,-0.07213275338232743,-0.01764612515980379,116.0 +0.0344433679824036,0.05068011873981862,0.028284032228378497,-0.03321323009955148,-0.04559945128264711,-0.009768885894536158,-0.050764121260198795,-0.002592261998183278,-0.05947118135708968,-0.021788232074638245,145.0 +-0.03457486258696539,0.05068011873981862,-0.02560657146566148,-0.017135116042335426,0.001182945896190995,-0.0028796197351664047,0.008142083605192267,-0.015507654304751785,0.014820979914103747,0.040343371647878594,74.0 +-0.052737554842062495,0.05068011873981862,-0.06225218197760866,0.011543832586488917,-0.008448724111216851,-0.03669965360843617,0.12227285553188745,-0.0763945037500033,-0.08682710478958679,0.0030644094143684884,45.0 +0.059871137139539544,-0.044641636506989144,-0.0008168937664030856,-0.0848559947372904,0.07548440023905152,0.07947842571548126,0.0044604458011053266,0.03430885887772673,0.02337141516224845,0.027917050903375224,115.0 +0.06350367559055897,0.05068011873981862,0.08864150836570328,0.07007229917592636,0.020446285911006685,0.03751653183568362,-0.050764121260198795,0.07120997975363674,0.02929655685872395,0.07348022696655424,264.0 +0.009015598825267658,-0.044641636506989144,-0.0320734439089463,-0.02632752814785296,0.04246153164222462,-0.010395182818115238,0.15908923357275687,-0.0763945037500033,-0.011896851335695978,-0.03835665973397607,87.0 +0.005383060374248237,0.05068011873981862,0.030439656376140087,0.08384370307932341,-0.037343734133440394,-0.04734670130928034,0.01550535921336615,-0.03949338287409329,0.008640601344549246,0.015490730158871856,202.0 +0.038075906433423026,0.05068011873981862,0.008883414898524095,0.04252949136913227,-0.04284754556624487,-0.02104223051895942,-0.03971920784793797,-0.002592261998183278,-0.018113692315690322,0.007206516329202944,127.0 +0.012648137276287077,-0.044641636506989144,0.006727790750762504,-0.05617704610846606,-0.07587041416307178,-0.06644875747844198,-0.02131101882750326,-0.03764832683029779,-0.018113692315690322,-0.092204049626824,182.0 +0.07440129094361722,0.05068011873981862,-0.020217511096257485,0.04597234234498153,0.0741084473808504,0.032819304908840594,-0.036037570043851025,0.07120997975363674,0.10635074572073594,0.036201264733044136,241.0 +0.016280675727306498,-0.044641636506989144,-0.02452875939178067,0.03564378941743375,-0.007072771253015731,-0.003192768196955922,-0.01394774321932938,-0.002592261998183278,0.015568459328120622,0.015490730158871856,66.0 +-0.005514554978810025,0.05068011873981862,-0.011595014505211082,0.011543832586488917,-0.02220825269322806,-0.015405558206747801,-0.02131101882750326,-0.002592261998183278,0.011010658023139448,0.06933812005171978,94.0 +0.012648137276287077,-0.044641636506989144,0.026128408080616904,0.06318659722422783,0.12501870313429186,0.09169121572527314,0.06336665066649638,-0.002592261998183278,0.057573156154827256,-0.021788232074638245,283.0 +-0.03457486258696539,-0.044641636506989144,-0.05901874575596628,0.0012152796589411327,-0.05385516843185383,-0.07803525056465478,0.06704828847058333,-0.0763945037500033,-0.021395309255276825,0.015490730158871856,64.0 +0.06713621404157838,0.05068011873981862,-0.03638469220446948,-0.0848559947372904,-0.007072771253015731,0.019667069513680118,-0.05444575906428573,0.03430885887772673,0.0011475759991601464,0.03205915781820968,102.0 +0.038075906433423026,0.05068011873981862,-0.02452875939178067,0.004658130634790395,-0.026336111267831423,-0.02636575436938152,0.01550535921336615,-0.03949338287409329,-0.015998872510179042,-0.025930338989472702,200.0 +0.009015598825267658,0.05068011873981862,0.018583723563451313,0.03908664039328301,0.017694380194604446,0.01058576412178361,0.019186997017453092,-0.002592261998183278,0.016306823139527554,-0.01764612515980379,265.0 +-0.09269547780327612,0.05068011873981862,-0.09027529589850945,-0.057313186930496314,-0.0249601584096303,-0.030436684372645465,-0.006584467611155497,-0.002592261998183278,0.024055085357995654,0.0030644094143684884,94.0 +0.0707687524925978,-0.044641636506989144,-0.0051281420619263066,-0.00567042229275739,0.08786797596286161,0.10296456034969638,0.01182372140927921,0.03430885887772673,-0.00894339609006817,0.027917050903375224,230.0 +-0.016412170331868287,-0.044641636506989144,-0.05255187331268147,-0.03321323009955148,-0.04422349842444599,-0.036386505146646625,0.019186997017453092,-0.03949338287409329,-0.0683315470939731,-0.03007244590430716,181.0 +0.04170844488444244,0.05068011873981862,-0.022373135244019075,0.028758087465735226,-0.06623874415566393,-0.045154662076753616,-0.06180903467245962,-0.002592261998183278,0.002861309289833047,-0.05492508739331389,156.0 +0.012648137276287077,-0.044641636506989144,-0.020217511096257485,-0.015998975220305175,0.01219056876179996,0.02123281182262779,-0.07653558588880739,0.10811110062954676,0.05987940361514779,-0.021788232074638245,233.0 +-0.03820740103798481,-0.044641636506989144,-0.05470749746044306,-0.07797029278559188,-0.033215875558837024,-0.08649025903297221,0.14068104455232217,-0.0763945037500033,-0.019198449026275908,-0.005219804415300423,60.0 +0.04534098333546186,-0.044641636506989144,-0.006205954135807083,-0.015998975220305175,0.12501870313429186,0.1251981011367534,0.019186997017453092,0.03430885887772673,0.03243232415655107,-0.005219804415300423,219.0 +0.0707687524925978,0.05068011873981862,-0.01698407487461508,0.0218723855140367,0.04383748450042574,0.05630543954305571,0.0375951860378878,-0.002592261998183278,-0.07020936123162536,-0.01764612515980379,80.0 +-0.074532785548179,0.05068011873981862,0.055229334075398484,-0.04009893205125,0.053469154507833586,0.05317395492516036,-0.04340084565202491,0.07120997975363674,0.06123762840403217,-0.03421455281914162,68.0 +0.059871137139539544,0.05068011873981862,0.07678557555301448,0.02531523648988596,0.001182945896190995,0.016848733357574308,-0.05444575906428573,0.03430885887772673,0.02993464904142137,0.044485478562713045,332.0 +0.07440129094361722,-0.044641636506989144,0.018583723563451313,0.06318659722422783,0.06172487165704031,0.042840055686105716,0.008142083605192267,-0.002592261998183278,0.05803805188793539,-0.05906719430814835,248.0 +0.009015598825267658,-0.044641636506989144,-0.022373135244019075,-0.03207708927752123,-0.04972730985725048,-0.06864079671096873,0.07809320188284416,-0.0708593356186168,-0.06291687914365544,-0.03835665973397607,84.0 +-0.07090024709715959,-0.044641636506989144,0.09295275666122646,0.012679973408519169,0.020446285911006685,0.04252690722431616,0.0007788079970183853,0.0003598276718895252,-0.05453964034510003,-0.0010776975004659671,200.0 +0.02354575262934534,0.05068011873981862,-0.03099563183506548,-0.00567042229275739,-0.016704441260423575,0.017788178742942903,-0.03235593223976409,-0.002592261998183278,-0.07409260794346935,-0.03421455281914162,55.0 +-0.052737554842062495,0.05068011873981862,0.039062152967186486,-0.04009893205125,-0.005696818394814609,-0.012900370512431508,0.01182372140927921,-0.03949338287409329,0.016306823139527554,0.0030644094143684884,85.0 +0.06713621404157838,-0.044641636506989144,-0.061174369903727877,-0.04009893205125,-0.026336111267831423,-0.02448686359864431,0.033913548233800855,-0.03949338287409329,-0.05615310200706333,-0.05906719430814835,89.0 +0.001750521923228816,-0.044641636506989144,-0.008361578283568675,-0.06419888888219483,-0.038719686991641515,-0.02448686359864431,0.0044604458011053266,-0.03949338287409329,-0.06468530604998815,-0.05492508739331389,31.0 +0.02354575262934534,0.05068011873981862,-0.03746250427835029,-0.04698463400294853,-0.0910058956032841,-0.07553006287033849,-0.03235593223976409,-0.03949338287409329,-0.030747917533098208,-0.013504018244969336,129.0 +0.038075906433423026,0.05068011873981862,-0.013750638652972673,-0.015998975220305175,-0.035967781275239266,-0.021981675904328014,-0.01394774321932938,-0.002592261998183278,-0.025953110560258,-0.0010776975004659671,83.0 +0.016280675727306498,-0.044641636506989144,0.0735521393313721,-0.04123507287328025,-0.004320865536613489,-0.013526667436010586,-0.01394774321932938,-0.0011162171631468765,0.04289703595278786,0.044485478562713045,275.0 +-0.0018820165277906047,0.05068011873981862,-0.02452875939178067,0.052858044296680055,0.027326050202012293,0.03000096875273476,0.030231910429713918,-0.002592261998183278,-0.021395309255276825,0.036201264733044136,65.0 +0.012648137276287077,-0.044641636506989144,0.03367309259778249,0.03333707926361473,0.030077955918414535,0.02718263259662897,-0.002902829807068556,0.00884708547334881,0.031192602201596156,0.027917050903375224,198.0 +0.07440129094361722,-0.044641636506989144,0.03475090467166331,0.0941722560068712,0.05759701308243695,0.020293366437259194,0.022868634821540033,-0.002592261998183278,0.07379892880056028,-0.021788232074638245,236.0 +0.04170844488444244,0.05068011873981862,-0.03854031635223107,0.052858044296680055,0.07686035309725264,0.11642994420664642,-0.03971920784793797,0.07120997975363674,-0.022516528376302174,-0.013504018244969336,253.0 +-0.009147093429829445,0.05068011873981862,-0.039618128426111884,-0.04009893205125,-0.008448724111216851,0.01622243643399523,-0.06549067247654655,0.07120997975363674,0.017765319557121535,-0.06735140813781726,124.0 +0.009015598825267658,0.05068011873981862,-0.0018947058402839008,0.0218723855140367,-0.038719686991641515,-0.02480001206043385,-0.006584467611155497,-0.03949338287409329,-0.03980882652740082,-0.013504018244969336,44.0 +0.06713621404157838,0.05068011873981862,-0.03099563183506548,0.004658130634790395,0.024574144485610048,0.03563764106494638,-0.028674294435677143,0.03430885887772673,0.02337141516224845,0.08176444079622315,172.0 +0.001750521923228816,-0.044641636506989144,-0.04608500086939666,-0.03321323009955148,-0.07311850844666953,-0.08147988364433965,0.04495846164606168,-0.06938329078358041,-0.061175799045152635,-0.07977772888232063,114.0 +-0.009147093429829445,0.05068011873981862,0.0013387303813585058,-0.002227571316908129,0.07961225881365488,0.07008397186179521,0.033913548233800855,-0.002592261998183278,0.02671684132010462,0.08176444079622315,142.0 +-0.005514554978810025,-0.044641636506989144,0.06492964274032566,0.03564378941743375,-0.001568959820211247,0.014969842586837095,-0.01394774321932938,0.0007288388806486174,-0.018113692315690322,0.03205915781820968,109.0 +0.09619652164973376,-0.044641636506989144,0.04013996504106731,-0.057313186930496314,0.045213437358626866,0.06068951800810917,-0.02131101882750326,0.03615391492152222,0.012551194864223063,0.02377494398854077,180.0 +-0.074532785548179,-0.044641636506989144,-0.023450947317899894,-0.00567042229275739,-0.02083229983502694,-0.014152964359589643,0.01550535921336615,-0.03949338287409329,-0.03845971734112638,-0.03007244590430716,144.0 +0.059871137139539544,0.05068011873981862,0.053073709927636895,0.052858044296680055,0.03282986163481677,0.019667069513680118,-0.010266105415242439,0.03430885887772673,0.055203099476237055,-0.0010776975004659671,163.0 +-0.02367724723390713,-0.044641636506989144,0.04013996504106731,-0.012556124244455912,-0.009824676969417972,-0.0010007289644291908,-0.002902829807068556,-0.002592261998183278,-0.011896851335695978,-0.03835665973397607,147.0 +0.009015598825267658,-0.044641636506989144,-0.020217511096257485,-0.05387033595464705,0.03145390877661565,0.020606514899048713,0.05600337505832251,-0.03949338287409329,-0.010903250651210127,-0.0010776975004659671,97.0 +0.016280675727306498,0.05068011873981862,0.014272475267928093,0.0012152796589411327,0.001182945896190995,-0.02135537898074896,-0.03235593223976409,0.03430885887772673,0.0749657259346355,0.040343371647878594,220.0 +0.01991321417832592,-0.044641636506989144,-0.03422906805670789,0.05516475445049906,0.0672286830898448,0.07415490186505921,-0.006584467611155497,0.032832814042690325,0.024729639951132837,0.06933812005171978,190.0 +0.0889314447476949,-0.044641636506989144,0.006727790750762504,0.02531523648988596,0.030077955918414535,0.008706873351046395,0.06336665066649638,-0.03949338287409329,0.009433658771615987,0.03205915781820968,109.0 +0.01991321417832592,-0.044641636506989144,0.004572166603000912,0.04597234234498153,-0.018080394118624697,-0.054549115930439665,0.06336665066649638,-0.03949338287409329,0.028658464676026615,0.06105390622205087,191.0 +-0.02367724723390713,-0.044641636506989144,0.030439656376140087,-0.00567042229275739,0.08236416453005713,0.09200436418706266,-0.01762938102341632,0.07120997975363674,0.0330430695314185,0.0030644094143684884,122.0 +0.09619652164973376,-0.044641636506989144,0.05199589785375607,0.0792647112814439,0.05484510736603471,0.036577086450315016,-0.07653558588880739,0.14132210941786577,0.0986480615153178,0.06105390622205087,230.0 +0.02354575262934534,0.05068011873981862,0.061696206518683294,0.06205045640219759,0.024574144485610048,-0.03607335668485709,-0.09126213710515514,0.15534453535071155,0.13339673866449434,0.08176444079622315,242.0 +0.0707687524925978,0.05068011873981862,-0.007283766209687899,0.049415193320830796,0.06034891879883919,-0.004445362044114079,-0.05444575906428573,0.10811110062954676,0.12902124941171242,0.05691179930721642,248.0 +0.03081082953138418,-0.044641636506989144,0.005649978676881689,0.011543832586488917,0.07823630595545376,0.0779126834065336,-0.04340084565202491,0.10811110062954676,0.06605066658209227,0.019632837073706312,249.0 +-0.0018820165277906047,-0.044641636506989144,0.05415152200151766,-0.06650559903601384,0.07273249452264928,0.056618588004845226,-0.04340084565202491,0.08486339447772344,0.08449153066204618,0.0486275854775475,192.0 +0.04534098333546186,0.05068011873981862,-0.008361578283568675,-0.03321323009955148,-0.007072771253015731,0.0011913102680975625,-0.03971920784793797,0.03430885887772673,0.02993464904142137,0.027917050903375224,131.0 +0.07440129094361722,-0.044641636506989144,0.11450899813884247,0.028758087465735226,0.024574144485610048,0.02499059336410222,0.019186997017453092,-0.002592261998183278,-0.0006117353045626216,-0.005219804415300423,237.0 +-0.03820740103798481,-0.044641636506989144,0.0670852668880873,-0.060756037906345574,-0.029088016984233665,-0.023234269751486174,-0.010266105415242439,-0.002592261998183278,-0.0014959487577289358,0.019632837073706312,78.0 +-0.012779631880848867,0.05068011873981862,-0.05578530953432388,-0.002227571316908129,-0.027712064126032544,-0.02918409052548733,0.019186997017453092,-0.03949338287409329,-0.017056282412934727,0.044485478562713045,135.0 +0.009015598825267658,0.05068011873981862,0.030439656376140087,0.04252949136913227,-0.0029449126784123676,0.03689023491210454,-0.06549067247654655,0.07120997975363674,-0.02364686309993755,0.015490730158871856,244.0 +0.08166636784565606,0.05068011873981862,-0.02560657146566148,-0.03665608107540074,-0.07036660273026729,-0.046407255923911754,-0.03971920784793797,-0.002592261998183278,-0.041176166918895155,-0.005219804415300423,199.0 +0.03081082953138418,-0.044641636506989144,0.10480868947391528,0.07695800112762488,-0.011200629827619093,-0.011334628203483833,-0.05812739686837268,0.03430885887772673,0.0571082604217192,0.036201264733044136,270.0 +0.027178291080364757,0.05068011873981862,-0.006205954135807083,0.028758087465735226,-0.016704441260423575,-0.0016270258880082473,-0.05812739686837268,0.03430885887772673,0.02929655685872395,0.03205915781820968,164.0 +-0.06000263174410134,0.05068011873981862,-0.047162812943277475,-0.0228846771720037,-0.07174255558846841,-0.05768060054833501,-0.006584467611155497,-0.03949338287409329,-0.06291687914365544,-0.05492508739331389,72.0 +0.005383060374248237,-0.044641636506989144,-0.048240625017158284,-0.012556124244455912,0.001182945896190995,-0.0066374012766408095,0.06336665066649638,-0.03949338287409329,-0.05140387304727299,-0.05906719430814835,96.0 +-0.020044708782887707,-0.044641636506989144,0.08540807214406083,-0.03665608107540074,0.09199583453746497,0.08949917649274639,-0.06180903467245962,0.14501222150545676,0.08094556124677081,0.05276969239238195,306.0 +0.01991321417832592,0.05068011873981862,-0.012672826579091896,0.07007229917592636,-0.011200629827619093,0.0071411310420987206,-0.03971920784793797,0.03430885887772673,0.005386331212792652,0.0030644094143684884,91.0 +-0.06363517019512076,-0.044641636506989144,-0.033151255982827074,-0.03321323009955148,0.001182945896190995,0.024051147978733624,-0.024992656631590206,-0.002592261998183278,-0.022516528376302174,-0.05906719430814835,214.0 +0.027178291080364757,-0.044641636506989144,-0.007283766209687899,-0.05042748497879779,0.07548440023905152,0.056618588004845226,0.033913548233800855,-0.002592261998183278,0.04344397210938562,0.015490730158871856,95.0 +-0.016412170331868287,-0.044641636506989144,-0.013750638652972673,0.13204361674121307,-0.009824676969417972,-0.0038190651205350003,0.019186997017453092,-0.03949338287409329,-0.0358161925842373,-0.03007244590430716,216.0 +0.03081082953138418,0.05068011873981862,0.05954058237092167,0.056300895272529315,-0.02220825269322806,0.0011913102680975625,-0.03235593223976409,-0.002592261998183278,-0.024795429028792802,-0.01764612515980379,263.0 +0.056238598688520124,0.05068011873981862,0.021817159785093684,0.056300895272529315,-0.007072771253015731,0.018101327204732443,-0.03235593223976409,-0.002592261998183278,-0.02364686309993755,0.02377494398854077,178.0 +-0.020044708782887707,-0.044641636506989144,0.018583723563451313,0.09072940503102193,0.003934851612593237,0.008706873351046395,0.0375951860378878,-0.03949338287409329,-0.05780302607946657,0.007206516329202944,113.0 +-0.1072256316073538,-0.044641636506989144,-0.011595014505211082,-0.04009893205125,0.04934129593323023,0.0644472995495836,-0.01394774321932938,0.03430885887772673,0.007027139682585861,-0.03007244590430716,200.0 +0.08166636784565606,0.05068011873981862,-0.002972517914164677,-0.03321323009955148,0.04246153164222462,0.057871181852003385,-0.010266105415242439,0.03430885887772673,-0.0006117353045626216,-0.0010776975004659671,139.0 +0.005383060374248237,0.05068011873981862,0.0175059114895705,0.03220093844158448,0.1277706088506941,0.12739014036928015,-0.02131101882750326,0.07120997975363674,0.06257762198769659,0.015490730158871856,139.0 +0.038075906433423026,0.05068011873981862,-0.029917819761184662,-0.07452744180974262,-0.012576582685820214,-0.012587222050641968,0.0044604458011053266,-0.002592261998183278,0.0037090603325595967,-0.03007244590430716,88.0 +0.03081082953138418,-0.044641636506989144,-0.020217511096257485,-0.00567042229275739,-0.004320865536613489,-0.02949723898727687,0.07809320188284416,-0.03949338287409329,-0.010903250651210127,-0.0010776975004659671,148.0 +0.001750521923228816,0.05068011873981862,-0.05794093368208547,-0.04354178302709927,-0.09650970703608859,-0.047033552847490806,-0.09862541271332903,0.03430885887772673,-0.061175799045152635,-0.07149351505265171,88.0 +-0.027309785684926546,0.05068011873981862,0.06061839444480248,0.10794365991026823,0.01219056876179996,-0.017597597439274533,-0.002902829807068556,-0.002592261998183278,0.07020738137223513,0.13561183068907107,243.0 +-0.08543040090123728,0.05068011873981862,-0.040695940499992665,-0.03321323009955148,-0.08137422559587626,-0.06958024209633733,-0.006584467611155497,-0.03949338287409329,-0.05780302607946657,-0.042498766648810526,71.0 +0.012648137276287077,0.05068011873981862,-0.07195249064253588,-0.04698463400294853,-0.051103262715451604,-0.09713730673381639,0.1185912177278005,-0.0763945037500033,-0.020292321339471356,-0.03835665973397607,77.0 +-0.052737554842062495,-0.044641636506989144,-0.05578530953432388,-0.03665608107540074,0.08924392882106273,-0.003192768196955922,0.008142083605192267,0.03430885887772673,0.1323757911721786,0.0030644094143684884,109.0 +-0.02367724723390713,0.05068011873981862,0.045529025410471304,0.0218723855140367,0.10988322169407955,0.08887287956916731,0.0007788079970183853,0.03430885887772673,0.07419089971278872,0.06105390622205087,272.0 +-0.074532785548179,0.05068011873981862,-0.00943939035744949,0.014986683562338177,-0.037343734133440394,-0.0216685274425385,-0.01394774321932938,-0.002592261998183278,-0.03324559264822791,0.0113486232440374,60.0 +-0.005514554978810025,0.05068011873981862,-0.033151255982827074,-0.015998975220305175,0.0080627101871966,0.01622243643399523,0.01550535921336615,-0.002592261998183278,-0.028323167238848198,-0.07563562196748617,54.0 +-0.06000263174410134,0.05068011873981862,0.04984027370599448,0.01842953453818744,-0.016704441260423575,-0.03012353591085593,-0.01762938102341632,-0.002592261998183278,0.04977020032069951,-0.05906719430814835,221.0 +-0.020044708782887707,-0.044641636506989144,-0.08488623552910546,-0.02632752814785296,-0.035967781275239266,-0.03419446591411989,0.04127682384197474,-0.05167075276314359,-0.08237869071592514,-0.04664087356364498,90.0 +0.038075906433423026,0.05068011873981862,0.005649978676881689,0.03220093844158448,0.006686757328995478,0.017475030281153364,-0.024992656631590206,0.03430885887772673,0.014820979914103747,0.06105390622205087,311.0 +0.016280675727306498,-0.044641636506989144,0.020739347711212906,0.0218723855140367,-0.013952535544021335,-0.013213518974221048,-0.006584467611155497,-0.002592261998183278,0.013316905483459898,0.040343371647878594,281.0 +0.04170844488444244,-0.044641636506989144,-0.007283766209687899,0.028758087465735226,-0.04284754556624487,-0.048286146694648965,0.05232173725423556,-0.0763945037500033,-0.07213275338232743,0.02377494398854077,182.0 +0.01991321417832592,0.05068011873981862,0.10480868947391528,0.07007229917592636,-0.035967781275239266,-0.02667890283117104,-0.024992656631590206,-0.002592261998183278,0.0037090603325595967,0.040343371647878594,321.0 +-0.04910501639104307,0.05068011873981862,-0.02452875939178067,7.913883691088233e-05,-0.046975404140848234,-0.02824464514011871,-0.06549067247654655,0.028404679537581124,0.019196469166885697,0.0113486232440374,58.0 +0.001750521923228816,0.05068011873981862,-0.006205954135807083,-0.019441826196154435,-0.009824676969417972,0.004949091809571968,-0.03971920784793797,0.03430885887772673,0.014820979914103747,0.09833286845556097,262.0 +0.0344433679824036,-0.044641636506989144,-0.03854031635223107,-0.012556124244455912,0.00943866304539772,0.0052622402713615075,-0.006584467611155497,-0.002592261998183278,0.031192602201596156,0.09833286845556097,206.0 +-0.045472477940023646,0.05068011873981862,0.13714305169033927,-0.015998975220305175,0.041085578784023497,0.03187985952347199,-0.04340084565202491,0.07120997975363674,0.07101867000452176,0.0486275854775475,233.0 +-0.009147093429829445,0.05068011873981862,0.17055522598064407,0.014986683562338177,0.030077955918414535,0.03375875029420919,-0.02131101882750326,0.03430885887772673,0.033653814906286016,0.03205915781820968,242.0 +-0.016412170331868287,0.05068011873981862,0.002416542455239321,0.014986683562338177,0.02182223876920781,-0.010082034356325698,-0.024992656631590206,0.03430885887772673,0.08553070935958189,0.08176444079622315,123.0 +-0.009147093429829445,-0.044641636506989144,0.03798434089330568,-0.04009893205125,-0.0249601584096303,-0.0038190651205350003,-0.04340084565202491,0.01585829843977173,-0.005142189801713891,0.027917050903375224,167.0 +0.01991321417832592,-0.044641636506989144,-0.05794093368208547,-0.057313186930496314,-0.001568959820211247,-0.012587222050641968,0.07441156407875721,-0.03949338287409329,-0.061175799045152635,-0.07563562196748617,63.0 +0.0526060602375007,0.05068011873981862,-0.00943939035744949,0.049415193320830796,0.05071724879143135,-0.019163339748222204,-0.01394774321932938,0.03430885887772673,0.11934047943993234,-0.01764612515980379,197.0 +-0.027309785684926546,0.05068011873981862,-0.023450947317899894,-0.015998975220305175,0.013566521620001083,0.012777803354310339,0.026550272625626974,-0.002592261998183278,-0.010903250651210127,-0.021788232074638245,71.0 +-0.074532785548179,-0.044641636506989144,-0.010517202431330305,-0.00567042229275739,-0.06623874415566393,-0.05705430362475593,-0.002902829807068556,-0.03949338287409329,-0.042570854118219384,-0.0010776975004659671,168.0 +-0.1072256316073538,-0.044641636506989144,-0.03422906805670789,-0.06764173985804409,-0.06348683843926169,-0.07051968748170592,0.008142083605192267,-0.03949338287409329,-0.0006117353045626216,-0.07977772888232063,140.0 +0.04534098333546186,0.05068011873981862,-0.002972517914164677,0.10794365991026823,0.03558176735121902,0.02248540566978595,0.026550272625626974,-0.002592261998183278,0.02802037249332928,0.019632837073706312,217.0 +-0.0018820165277906047,-0.044641636506989144,0.0681630789619681,-0.00567042229275739,0.11951489170148738,0.13020847652538595,-0.024992656631590206,0.08670845052151895,0.04613307487932449,-0.0010776975004659671,121.0 +0.01991321417832592,0.05068011873981862,0.009961226972404908,0.01842953453818744,0.014942474478202204,0.04471894645684291,-0.06180903467245962,0.07120997975363674,0.009433658771615987,-0.0632093012229828,235.0 +0.016280675727306498,0.05068011873981862,0.002416542455239321,-0.00567042229275739,-0.005696818394814609,0.010898912583573148,-0.050764121260198795,0.03430885887772673,0.022687744966501246,-0.03835665973397607,245.0 +-0.0018820165277906047,-0.044641636506989144,-0.03854031635223107,0.0218723855140367,-0.10889328275989867,-0.11561306597939897,0.022868634821540033,-0.0763945037500033,-0.04688253415273158,0.02377494398854077,40.0 +0.016280675727306498,-0.044641636506989144,0.026128408080616904,0.05860760542634833,-0.060734932722859444,-0.044215216691385,-0.01394774321932938,-0.03395821474270679,-0.05140387304727299,-0.025930338989472702,52.0 +-0.07090024709715959,0.05068011873981862,-0.08919748382462865,-0.07452744180974262,-0.04284754556624487,-0.02573945744580244,-0.03235593223976409,-0.002592261998183278,-0.012908683225401873,-0.05492508739331389,104.0 +0.04897352178648128,-0.044641636506989144,0.06061839444480248,-0.0228846771720037,-0.02358420555142918,-0.07271172671423268,-0.04340084565202491,-0.002592261998183278,0.10413565428651514,0.036201264733044136,132.0 +0.005383060374248237,0.05068011873981862,-0.028840007687303888,-0.009113273268606652,-0.0318399227006359,-0.02887094206369779,0.008142083605192267,-0.03949338287409329,-0.018113692315690322,0.007206516329202944,88.0 +0.0344433679824036,0.05068011873981862,-0.029917819761184662,0.004658130634790395,0.0933717873956661,0.08699398879843012,0.033913548233800855,-0.002592261998183278,0.024055085357995654,-0.03835665973397607,69.0 +0.02354575262934534,0.05068011873981862,-0.01913969902237667,0.049415193320830796,-0.06348683843926169,-0.06112523362801988,0.0044604458011053266,-0.03949338287409329,-0.025953110560258,-0.013504018244969336,219.0 +0.01991321417832592,-0.044641636506989144,-0.040695940499992665,-0.015998975220305175,-0.008448724111216851,-0.017597597439274533,0.05232173725423556,-0.03949338287409329,-0.030747917533098208,0.0030644094143684884,72.0 +-0.045472477940023646,-0.044641636506989144,0.015350287341808908,-0.07452744180974262,-0.04972730985725048,-0.017284448977484993,-0.028674294435677143,-0.002592261998183278,-0.10436552421115437,-0.07563562196748617,201.0 +0.0526060602375007,0.05068011873981862,-0.02452875939178067,0.056300895272529315,-0.007072771253015731,-0.005071658967693135,-0.02131101882750326,-0.002592261998183278,0.02671684132010462,-0.03835665973397607,110.0 +-0.005514554978810025,0.05068011873981862,0.0013387303813585058,-0.0848559947372904,-0.011200629827619093,-0.016658152053905938,0.04864009945014862,-0.03949338287409329,-0.041176166918895155,-0.08806194271198954,51.0 +0.009015598825267658,0.05068011873981862,0.06924089103584885,0.059743746248378575,0.017694380194604446,-0.023234269751486174,-0.04708248345611185,0.03430885887772673,0.10329701884639861,0.07348022696655424,277.0 +-0.02367724723390713,-0.044641636506989144,-0.06979686649477428,-0.06419888888219483,-0.05935897986465832,-0.05047818592717569,0.019186997017453092,-0.03949338287409329,-0.08913335224990723,-0.05078298047847944,63.0 +-0.04183993948900423,0.05068011873981862,-0.029917819761184662,-0.002227571316908129,0.02182223876920781,0.036577086450315016,0.01182372140927921,-0.002592261998183278,-0.041176166918895155,0.06519601313688532,118.0 +-0.074532785548179,-0.044641636506989144,-0.04608500086939666,-0.04354178302709927,-0.029088016984233665,-0.023234269751486174,0.01550535921336615,-0.03949338287409329,-0.03980882652740082,-0.021788232074638245,69.0 +0.0344433679824036,-0.044641636506989144,0.018583723563451313,0.056300895272529315,0.01219056876179996,-0.054549115930439665,-0.0691723102806335,0.07120997975363674,0.13007865931446802,0.007206516329202944,273.0 +-0.06000263174410134,-0.044641636506989144,0.0013387303813585058,-0.029770379123702218,-0.007072771253015731,-0.0216685274425385,0.01182372140927921,-0.002592261998183278,0.031812463179073616,-0.05492508739331389,258.0 +-0.08543040090123728,0.05068011873981862,-0.03099563183506548,-0.0228846771720037,-0.06348683843926169,-0.05423596746865012,0.019186997017453092,-0.03949338287409329,-0.09643494994048712,-0.03421455281914162,43.0 +0.0526060602375007,-0.044641636506989144,-0.004050329988045492,-0.03090651994573247,-0.046975404140848234,-0.05830689747191407,-0.01394774321932938,-0.02583996815000658,0.036060333995316066,0.02377494398854077,198.0 +0.012648137276287077,-0.044641636506989144,0.015350287341808908,-0.03321323009955148,0.041085578784023497,0.032193007985261514,-0.002902829807068556,-0.002592261998183278,0.04506654937395887,-0.06735140813781726,242.0 +0.059871137139539544,0.05068011873981862,0.022894971858974496,0.049415193320830796,0.016318427336403322,0.011838357968941744,-0.01394774321932938,-0.002592261998183278,0.03954249419232167,0.019632837073706312,232.0 +-0.02367724723390713,-0.044641636506989144,0.045529025410471304,0.09072940503102193,-0.018080394118624697,-0.03544705976127803,0.07072992627467027,-0.03949338287409329,-0.03452177701362266,-0.009361911330134878,175.0 +0.016280675727306498,-0.044641636506989144,-0.04500718879551588,-0.057313186930496314,-0.034591828417038145,-0.053922819006860585,0.07441156407875721,-0.0763945037500033,-0.042570854118219384,0.040343371647878594,93.0 +0.11072667545381144,0.05068011873981862,-0.033151255982827074,-0.0228846771720037,-0.004320865536613489,0.020293366437259194,-0.06180903467245962,0.07120997975363674,0.015568459328120622,0.044485478562713045,168.0 +-0.020044708782887707,-0.044641636506989144,0.09726400495674965,-0.00567042229275739,-0.005696818394814609,-0.02386056667506523,-0.02131101882750326,-0.002592261998183278,0.06168429293192034,0.040343371647878594,275.0 +-0.016412170331868287,-0.044641636506989144,0.05415152200151766,0.07007229917592636,-0.033215875558837024,-0.0279314966783292,0.008142083605192267,-0.03949338287409329,-0.02712902329694316,-0.009361911330134878,293.0 +0.04897352178648128,0.05068011873981862,0.12313149472988882,0.08384370307932341,-0.10476542418529532,-0.10089508827529081,-0.0691723102806335,-0.002592261998183278,0.03664373256235367,-0.03007244590430716,281.0 +-0.056370093293081916,-0.044641636506989144,-0.08057498723358228,-0.0848559947372904,-0.037343734133440394,-0.037012802070225705,0.033913548233800855,-0.03949338287409329,-0.05615310200706333,-0.13776722569000302,72.0 +0.027178291080364757,-0.044641636506989144,0.09295275666122646,-0.0527341951326168,0.0080627101871966,0.039708571068210366,-0.028674294435677143,0.021024455362399115,-0.04835926177554553,0.019632837073706312,140.0 +0.06350367559055897,-0.044641636506989144,-0.050396249164919873,0.10794365991026823,0.03145390877661565,0.019353921051890578,-0.01762938102341632,0.02360753382371283,0.05803805188793539,0.040343371647878594,189.0 +-0.052737554842062495,0.05068011873981862,-0.011595014505211082,0.056300895272529315,0.05622106022423583,0.07290230801790105,-0.03971920784793797,0.07120997975363674,0.030563625621508765,-0.005219804415300423,181.0 +-0.009147093429829445,0.05068011873981862,-0.027762195613423073,0.008100981610639655,0.04796534307502911,0.037203383373894054,-0.028674294435677143,0.03430885887772673,0.06605066658209227,-0.042498766648810526,209.0 +0.005383060374248237,-0.044641636506989144,0.05846277029704089,-0.04354178302709927,-0.07311850844666953,-0.07239857825244314,0.019186997017453092,-0.0763945037500033,-0.05140387304727299,-0.025930338989472702,136.0 +0.07440129094361722,-0.044641636506989144,0.08540807214406083,0.06318659722422783,0.014942474478202204,0.013090951816099879,0.01550535921336615,-0.002592261998183278,0.006206735447689297,0.0859065477110576,261.0 +-0.052737554842062495,-0.044641636506989144,-0.0008168937664030856,-0.02632752814785296,0.010814615903598841,0.0071411310420987206,0.04864009945014862,-0.03949338287409329,-0.0358161925842373,0.019632837073706312,113.0 +0.08166636784565606,0.05068011873981862,0.006727790750762504,-0.00453428147072714,0.10988322169407955,0.11705624113022545,-0.03235593223976409,0.09187460744414634,0.05471997253790904,0.007206516329202944,131.0 +-0.005514554978810025,-0.044641636506989144,0.008883414898524095,-0.05042748497879779,0.02595009734381117,0.04722413415115918,-0.04340084565202491,0.07120997975363674,0.014820979914103747,0.0030644094143684884,174.0 +-0.027309785684926546,-0.044641636506989144,0.08001901177465684,0.0987512478047507,-0.0029449126784123676,0.018101327204732443,-0.01762938102341632,0.003311917341962329,-0.02952642678336326,0.036201264733044136,257.0 +-0.052737554842062495,-0.044641636506989144,0.07139651518361048,-0.07452744180974262,-0.015328488402222454,-0.0013138774262187302,0.0044604458011053266,-0.021411833644897377,-0.04688253415273158,0.0030644094143684884,55.0 +0.009015598825267658,-0.044641636506989144,-0.02452875939178067,-0.02632752814785296,0.09887559882847057,0.0941964034195894,0.07072992627467027,-0.002592261998183278,-0.021395309255276825,0.007206516329202944,84.0 +-0.020044708782887707,-0.044641636506989144,-0.05470749746044306,-0.05387033595464705,-0.06623874415566393,-0.05736745208654548,0.01182372140927921,-0.03949338287409329,-0.07409260794346935,-0.005219804415300423,42.0 +0.02354575262934534,-0.044641636506989144,-0.03638469220446948,7.913883691088233e-05,0.001182945896190995,0.034698195679577784,-0.04340084565202491,0.03430885887772673,-0.03324559264822791,0.06105390622205087,146.0 +0.038075906433423026,0.05068011873981862,0.016428099415689682,0.0218723855140367,0.039709625925822375,0.04503209491863242,-0.04340084565202491,0.07120997975363674,0.04977020032069951,0.015490730158871856,212.0 +-0.07816532399919843,0.05068011873981862,0.07786338762689529,0.052858044296680055,0.07823630595545376,0.0644472995495836,0.026550272625626974,-0.002592261998183278,0.04067282891595704,-0.009361911330134878,233.0 +0.009015598825267658,0.05068011873981862,-0.039618128426111884,0.028758087465735226,0.03833367306762126,0.07352860494148013,-0.07285394808472044,0.10811110062954676,0.015568459328120622,-0.04664087356364498,91.0 +0.001750521923228816,0.05068011873981862,0.011039039046285686,-0.019441826196154435,-0.016704441260423575,-0.0038190651205350003,-0.04708248345611185,0.03430885887772673,0.024055085357995654,0.02377494398854077,111.0 +-0.07816532399919843,-0.044641636506989144,-0.040695940499992665,-0.08141314376144114,-0.10063756561069194,-0.11279472982329314,0.022868634821540033,-0.0763945037500033,-0.020292321339471356,-0.05078298047847944,152.0 +0.03081082953138418,0.05068011873981862,-0.03422906805670789,0.04366563219116252,0.05759701308243695,0.0688313780146371,-0.03235593223976409,0.05755656502955003,0.03545870422305857,0.0859065477110576,120.0 +-0.03457486258696539,0.05068011873981862,0.005649978676881689,-0.00567042229275739,-0.07311850844666953,-0.06269097593696756,-0.006584467611155497,-0.03949338287409329,-0.04542403773513769,0.03205915781820968,67.0 +0.04897352178648128,0.05068011873981862,0.08864150836570328,0.08728655405517267,0.03558176735121902,0.02154596028441731,-0.024992656631590206,0.03430885887772673,0.06605066658209227,0.13146972377423663,310.0 +-0.04183993948900423,-0.044641636506989144,-0.033151255982827074,-0.0228846771720037,0.04658939021682799,0.041587461838947556,0.05600337505832251,-0.024732934523729287,-0.025953110560258,-0.03835665973397607,94.0 +-0.009147093429829445,-0.044641636506989144,-0.05686312160820465,-0.05042748497879779,0.02182223876920781,0.04534524338042199,-0.028674294435677143,0.03430885887772673,-0.009918765569334137,-0.01764612515980379,183.0 +0.0707687524925978,0.05068011873981862,-0.03099563183506548,0.0218723855140367,-0.037343734133440394,-0.047033552847490806,0.033913548233800855,-0.03949338287409329,-0.014959693812643405,-0.0010776975004659671,66.0 +0.009015598825267658,-0.044641636506989144,0.055229334075398484,-0.00567042229275739,0.05759701308243695,0.04471894645684291,-0.002902829807068556,0.023238522614953735,0.05568622641456506,0.1066170822852299,173.0 +-0.027309785684926546,-0.044641636506989144,-0.06009655782984706,-0.029770379123702218,0.04658939021682799,0.019980217975469634,0.12227285553188745,-0.03949338287409329,-0.05140387304727299,-0.009361911330134878,72.0 +0.016280675727306498,-0.044641636506989144,0.0013387303813585058,0.008100981610639655,0.005310804470794357,0.010898912583573148,0.030231910429713918,-0.03949338287409329,-0.04542403773513769,0.03205915781820968,49.0 +-0.012779631880848867,-0.044641636506989144,-0.023450947317899894,-0.04009893205125,-0.016704441260423575,0.00463594334778245,-0.01762938102341632,-0.002592261998183278,-0.03845971734112638,-0.03835665973397607,64.0 +-0.056370093293081916,-0.044641636506989144,-0.07410811479029747,-0.05042748497879779,-0.0249601584096303,-0.047033552847490806,0.09281975309919192,-0.0763945037500033,-0.061175799045152635,-0.04664087356364498,48.0 +0.04170844488444244,0.05068011873981862,0.01966153563733209,0.059743746248378575,-0.005696818394814609,-0.0025664712733768653,-0.028674294435677143,-0.002592261998183278,0.031192602201596156,0.007206516329202944,178.0 +-0.005514554978810025,0.05068011873981862,-0.015906262800734303,-0.06764173985804409,0.04934129593323023,0.07916527725369175,-0.028674294435677143,0.03430885887772673,-0.018113692315690322,0.044485478562713045,104.0 +0.04170844488444244,0.05068011873981862,-0.015906262800734303,0.01729339371615719,-0.037343734133440394,-0.013839815897800126,-0.024992656631590206,-0.011079519799642579,-0.04688253415273158,0.015490730158871856,132.0 +-0.045472477940023646,-0.044641636506989144,0.039062152967186486,0.0012152796589411327,0.016318427336403322,0.015282991048626631,-0.028674294435677143,0.02655962349378563,0.04452872881997113,-0.025930338989472702,220.0 +-0.045472477940023646,-0.044641636506989144,-0.07303030271641665,-0.08141314376144114,0.08374011738825825,0.027808929520208008,0.17381578478910462,-0.03949338287409329,-0.00422151393810765,0.0030644094143684884,57.0 diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,7 +1,6 @@ cabal-version: 3.0 name: dataframe -version: 2.1.0.3 - +version: 2.2.0.0 synopsis: A fast, safe, and intuitive DataFrame library. description: A fast, safe, and intuitive DataFrame library for exploratory data analysis. @@ -21,6 +20,8 @@ tests/data/typing/texts_with_empties.txt tests/data/typing/texts_with_empties_and_nullish.txt data/titanic/*.csv + data/ml/*.csv + data/ml/golden.json data/sharded/*.parquet tests/data/*.csv tests/data/*.tsv @@ -120,16 +121,16 @@ DataFrame.Typed.Record, DataFrame.Typed.Generic build-depends: base >= 4 && <5, - dataframe-core ^>= 1.0.2, + dataframe-core ^>= 1.1, dataframe-json ^>= 1.0, - dataframe-operations ^>= 1.1.0.2, - dataframe-parsing ^>= 1.0, - dataframe-viz ^>= 1.0, - dataframe-learn ^>= 1.0 + dataframe-operations >= 1.1.1 && < 1.2, + dataframe-parsing ^>= 1.0.2, + dataframe-viz ^>= 1.0.3, + dataframe-learn ^>= 1.1 if !flag(no-csv) reexported-modules: DataFrame.IO.CSV - build-depends: dataframe-csv ^>= 1.0 + build-depends: dataframe-csv ^>= 1.0.2 cpp-options: -DWITH_CSV if !flag(no-parquet) @@ -161,7 +162,7 @@ DataFrame.Lazy.Internal.Optimizer, DataFrame.Lazy.Internal.Executor, DataFrame.Typed.Lazy - build-depends: dataframe-lazy ^>= 1.0 + build-depends: dataframe-lazy ^>= 1.0.2 cpp-options: -DWITH_LAZY if !flag(no-th) @@ -194,13 +195,13 @@ buildable: False build-depends: base >= 4 && < 5, - dataframe-core ^>= 1.0.2, - dataframe-csv ^>= 1.0, + dataframe-core ^>= 1.1, + dataframe-csv ^>= 1.0.2, dataframe-json ^>= 1.0, - dataframe-lazy ^>= 1.0, - dataframe-operations ^>= 1.1.0.2, + dataframe-lazy ^>= 1.0.2, + dataframe-operations >= 1.1.1 && < 1.2, dataframe-parquet ^>= 1.0.1.1, - dataframe-parsing ^>= 1.0, + dataframe-parsing ^>= 1.0.2, text >= 2.0 && < 3, aeson >= 0.11 && < 3, bytestring >= 0.11 && < 0.13, @@ -214,7 +215,7 @@ main-is: Benchmark.hs build-depends: base >= 4 && < 5, dataframe >= 1 && < 3, - dataframe-operations ^>= 1.1, + dataframe-operations >= 1.1.1 && < 1.2, random >= 1 && < 2, time >= 1.12 && < 2, vector ^>= 0.13, @@ -227,9 +228,9 @@ main-is: Synthesis.hs build-depends: base >= 4 && < 5, dataframe >= 1 && < 3, - dataframe-core ^>= 1.0, - dataframe-learn ^>= 1.0, - dataframe-operations ^>= 1.1, + dataframe-core ^>= 1.1, + dataframe-learn ^>= 1.1, + dataframe-operations >= 1.1.1 && < 1.2, random >= 1 && < 2, text >= 2.0 && < 3 hs-source-dirs: app @@ -261,9 +262,9 @@ bytestring >= 0.11 && < 0.13, containers >= 0.6.7 && < 0.9, dataframe >= 1 && < 3, - dataframe-core ^>= 1.0, - dataframe-lazy ^>= 1.0, - dataframe-parsing ^>= 1.0, + dataframe-core ^>= 1.1, + dataframe-lazy ^>= 1.0.2, + dataframe-parsing ^>= 1.0.2, directory >= 1.3.0.0 && < 2, random >= 1 && < 2, text >= 2.0 && < 3, @@ -293,6 +294,9 @@ import: warnings type: exitcode-stdio-1.0 main-is: Main.hs + -- Threaded RTS with -N so the parallel-grouping parity test exercises the + -- real multi-capability fork path rather than the sequential fallback. + ghc-options: -threaded -rtsopts -with-rtsopts=-N -- The test runner imports CSV, JSON, Parquet, Lazy, and TH-derived -- schema modules. All features must be enabled for it to compile. if flag(no-csv) || flag(no-parquet) || flag(no-th) @@ -302,8 +306,24 @@ DecisionTree, Functions, GenDataFrame, + Internal.ColumnBuilder, + Internal.DictEncode, + Internal.PackedText, Internal.Parsing, + PackedTextMigration, + Learn.Numerics, + Learn.Denotation, + Learn.Models, + Learn.Ensembles, + Learn.Symbolic, + Learn.SklearnParity, + Learn.Synthesis, + Learn.MetricsTests, + Learn.Metamorphic, + Learn.EdgeCases, + Learn.NumericalRigor, IO.CSV, + IO.CsvGolden, IO.JSON, LinearSolver, Operations.Aggregations, @@ -312,6 +332,9 @@ Operations.Derive, Operations.Filter, Operations.GroupBy, + Operations.ParallelGroupBy, + Operations.ParallelJoin, + Operations.Inference, Operations.InsertColumn, Operations.Join, Operations.Merge, @@ -328,8 +351,10 @@ Operations.Statistics, Operations.Take, Operations.Typing, + Operations.VectorKernel, Operations.Record, LazyParquet, + LazyParity, Parquet, ParquetTestData, Plotting, @@ -344,14 +369,14 @@ aeson >= 0.11.0.0 && < 3, bytestring >= 0.11 && < 0.13, dataframe >= 1 && < 3, - dataframe-core ^>= 1.0, - dataframe-csv ^>= 1.0, + dataframe-core ^>= 1.1, + dataframe-csv ^>= 1.0.2, dataframe-json ^>= 1.0, - dataframe-lazy ^>= 1.0, - dataframe-learn ^>= 1.0, - dataframe-operations ^>= 1.1, + dataframe-lazy ^>= 1.0.2, + dataframe-learn ^>= 1.1, + dataframe-operations >= 1.1.1 && < 1.2, dataframe-parquet ^>= 1.0, - dataframe-parsing ^>= 1.0, + dataframe-parsing ^>= 1.0.2, HUnit ^>= 1.6, QuickCheck >= 2 && < 3, random-shuffle >= 0.0.4 && < 1, @@ -359,6 +384,28 @@ text >= 2.0 && < 3, time >= 1.12 && < 2, vector ^>= 0.13, + temporary >= 1.3 && < 1.4, + directory >= 1.3 && < 1.4, containers >= 0.6.7 && < 0.9 + hs-source-dirs: tests + default-language: Haskell2010 + +-- Focused oracle for the packed-text column variant. Kept as its own suite so +-- it can run independently of the (heavier) `tests` runner. +test-suite packed-text + import: warnings + type: exitcode-stdio-1.0 + main-is: PackedTextMain.hs + if flag(no-csv) || flag(no-parquet) || flag(no-th) + buildable: False + other-modules: Internal.PackedText + build-depends: base >= 4 && < 5, + bytestring >= 0.11 && < 0.13, + dataframe >= 1 && < 3, + dataframe-core ^>= 1.1, + dataframe-operations >= 1.1.1 && < 1.2, + HUnit ^>= 1.6, + text >= 2.0 && < 3, + vector ^>= 0.13 hs-source-dirs: tests default-language: Haskell2010 diff --git a/ffi/DataFrame/IR.hs b/ffi/DataFrame/IR.hs --- a/ffi/DataFrame/IR.hs +++ b/ffi/DataFrame/IR.hs @@ -258,6 +258,7 @@ columnTypeRep :: Column -> SomeTypeRep columnTypeRep (UnboxedColumn _ (_ :: VU.Vector a)) = SomeTypeRep (typeRep @a) columnTypeRep (BoxedColumn _ (_ :: V.Vector a)) = SomeTypeRep (typeRep @a) + columnTypeRep (PackedText _ _) = SomeTypeRep (typeRep @T.Text) mk :: (Columnable a, Ord a) => Expr a -> SortOrder mk = if isAsc then Asc else Desc dispatchType (SomeTypeRep tr) @@ -335,6 +336,7 @@ columnTypeRep :: Column -> SomeTypeRep columnTypeRep (UnboxedColumn _ (_ :: VU.Vector a)) = SomeTypeRep (typeRep @a) columnTypeRep (BoxedColumn _ (_ :: V.Vector a)) = SomeTypeRep (typeRep @a) + columnTypeRep (PackedText _ _) = SomeTypeRep (typeRep @T.Text) fr :: forall a. (Columnable a, Ord a) => IO DataFrame fr = return $ Stats.frequencies (Col @a colName) df @@ -369,6 +371,8 @@ countExpr name colName (UnboxedColumn (Just _) (_ :: VU.Vector a)) = return $ name .= count (Col @(Maybe a) colName) countExpr name colName (BoxedColumn Nothing (_ :: V.Vector a)) = return $ name .= count (Col @a colName) countExpr name colName (BoxedColumn (Just _) (_ :: V.Vector a)) = return $ name .= count (Col @(Maybe a) colName) +countExpr name colName (PackedText Nothing _) = return $ name .= count (Col @T.Text colName) +countExpr name colName (PackedText (Just _) _) = return $ name .= count (Col @(Maybe T.Text) colName) sumExpr :: T.Text -> T.Text -> Column -> IO NamedExpr sumExpr name colName (UnboxedColumn Nothing (_ :: VU.Vector a)) diff --git a/tests/IO/CsvGolden.hs b/tests/IO/CsvGolden.hs new file mode 100644 --- /dev/null +++ b/tests/IO/CsvGolden.hs @@ -0,0 +1,560 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +{- | Golden semantics for the default CSV reader, pinned against the +cassava-based implementation (oracle runs of 2026-06-12) before the +strict-scanner rewrite. Ragged-row cases encode the new pad-with-null +behavior (audit D6) — the one intentional change. +-} +module IO.CsvGolden (tests) where + +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as BL +import qualified Data.Map.Strict as M +import qualified Data.Text as T + +import Control.Exception (SomeException, evaluate, try) +import Data.List (isInfixOf) +import Data.Time.Calendar (Day, fromGregorian) +import DataFrame.IO.CSV +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.DataFrame ( + columnIndices, + dataframeDimensions, + forceDataFrame, + getColumn, + ) +import DataFrame.Internal.Schema (schemaType) +import DataFrame.Operations.Typing (SafeReadMode (..)) +import Test.HUnit + +data Expect + = Cols (Int, Int) [(T.Text, DI.Column)] + | Err String + +ints :: [Int] -> DI.Column +ints = DI.fromList +mints :: [Maybe Int] -> DI.Column +mints = DI.fromList +dbls :: [Double] -> DI.Column +dbls = DI.fromList +texts :: [T.Text] -> DI.Column +texts = DI.fromList +mtexts :: [Maybe T.Text] -> DI.Column +mtexts = DI.fromList +eti :: [Either T.Text Int] -> DI.Column +eti = DI.fromList +ett :: [Either T.Text T.Text] -> DI.Column +ett = DI.fromList +days :: [Day] -> DI.Column +days = DI.fromList +boolsC :: [Bool] -> DI.Column +boolsC = DI.fromList + +sample :: Int -> ReadOptions +sample n = defaultReadOptions{typeSpec = InferFromSample n} + +goldenCase :: (String, ReadOptions, BS.ByteString, Expect) -> Test +goldenCase (label, opts, input, expect) = TestLabel label $ TestCase $ do + r <- try $ do + df <- decodeSeparated opts (BL.fromStrict input) + evaluate (forceDataFrame df) + case (expect, r) of + (Err sub, Left (e :: SomeException)) -> + assertBool + (label <> ": error should mention " <> show sub <> ", got " <> show e) + (sub `isInfixOf` show e) + (Err _, Right _) -> assertFailure (label <> ": expected an error") + (Cols _ _, Left (e :: SomeException)) -> + assertFailure (label <> ": unexpected error " <> show e) + (Cols dims cols, Right df) -> do + assertEqual (label <> ": dims") dims (dataframeDimensions df) + mapM_ + ( \(name, expected) -> case getColumn name df of + Nothing -> assertFailure (label <> ": missing column " <> show name) + Just actual -> + assertEqual (label <> ": column " <> show name) expected actual + ) + cols + +goldenCases :: [(String, ReadOptions, BS.ByteString, Expect)] +goldenCases = + [ + ( "basic" + , defaultReadOptions + , "a,b,c\n1,2.5,x\n2,3.5,y\n" + , Cols + (2, 3) + [("a", ints [1, 2]), ("b", dbls [2.5, 3.5]), ("c", texts ["x", "y"])] + ) + , + ( "quoted_doubled" + , defaultReadOptions + , "a,b\n\"x,1\",\"he said \"\"hi\"\"\"\n\"multi\nline\",plain\n" + , Cols + (2, 2) + [("a", texts ["x,1", "multi\nline"]), ("b", texts ["he said \"hi\"", "plain"])] + ) + , + ( "crlf" + , defaultReadOptions + , "a,b\r\n1,x\r\n2,y\r\n" + , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])] + ) + , + ( "lone_cr" + , defaultReadOptions + , "a,b\r1,x\r2,y" + , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])] + ) + , + ( "blank_lines" + , defaultReadOptions + , "a,b\n\n1,x\n\n\n2,y\n\n" + , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])] + ) + , + ( "lone_cr_line" + , defaultReadOptions + , "a,b\r\n\r1,x\r\n" + , Cols (1, 2) [("a", ints [1]), ("b", texts ["x"])] + ) + , + ( "no_eof_newline" + , defaultReadOptions + , "a,b\n1,x" + , Cols (1, 2) [("a", ints [1]), ("b", texts ["x"])] + ) + , + ( "missing_numeric" + , defaultReadOptions + , "a,b\n1,2\nNA,3\n,4\nnan,5\n" + , Cols + (4, 2) + [("a", mints [Just 1, Nothing, Nothing, Nothing]), ("b", ints [2, 3, 4, 5])] + ) + , + ( "missing_text" + , defaultReadOptions + , "t,u\nfoo,1\nNA,2\n ,3\nbar,4\n" + , Cols + (4, 2) + [ ("t", mtexts [Just "foo", Nothing, Nothing, Just "bar"]) + , ("u", ints [1, 2, 3, 4]) + ] + ) + , + ( "ws_padding" + , defaultReadOptions + , "a,b\n 1 , x \n2,y\n" + , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])] + ) + , + ( "tabs_strip" + , defaultReadOptions + , "a,b\n\t1\t,\tx\t\n2,y\n" + , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])] + ) + , -- D6: short rows now pad trailing columns with null (was: misaligned). + + ( "ragged_short_D6" + , defaultReadOptions + , "a,b,c\n1,2,3\n4,5\n6,7,8\n" + , Cols + (3, 3) + [ ("a", ints [1, 4, 6]) + , ("b", ints [2, 5, 7]) + , ("c", mints [Just 3, Nothing, Just 8]) + ] + ) + , + ( "ragged_long" + , defaultReadOptions + , "a,b\n1,2\n3,4,5\n6,7\n" + , Cols (3, 2) [("a", ints [1, 3, 6]), ("b", ints [2, 4, 7])] + ) + , + ( "all_rows_ragged_short_D6" + , defaultReadOptions + , "a,b,c\n1,2\n3,4\n" + , Cols + (2, 3) + [("a", ints [1, 3]), ("b", ints [2, 4]), ("c", mtexts [Nothing, Nothing])] + ) + , + ( "maybe_read_ragged_D6" + , defaultReadOptions{safeRead = MaybeRead} + , "a,b\n1,2\n3\n" + , Cols (2, 2) [("a", mints [Just 1, Just 3]), ("b", mints [Just 2, Nothing])] + ) + , + ( "either_read_ragged_D6" + , defaultReadOptions{safeRead = EitherRead} + , "a,b\n1,2\n3\n" + , Cols (2, 2) [("a", eti [Right 1, Right 3]), ("b", eti [Right 2, Left ""])] + ) + , ("stray_quote_mid", defaultReadOptions, "a,b\nx\"y,2\n", Err "") + , ("quote_garbage", defaultReadOptions, "a,b\n\"x\"y,2\n", Err "") + , ("space_then_quote", defaultReadOptions, "a,b\n \"x\",2\n", Err "") + , ("quote_space_garbage", defaultReadOptions, "a,b\n\"x\" ,2\n", Err "") + , ("quote_at_field_end", defaultReadOptions, "a,b\n1,x\"\n", Err "") + , ("doubled_then_garbage", defaultReadOptions, "a,b\n\"\"x,2\n", Err "") + , + ( "quoted_empty" + , defaultReadOptions + , "a,b\n\"\",2\n" + , Cols (1, 2) [("a", mtexts [Nothing]), ("b", ints [2])] + ) + , + ( "quoted_empty_row_skipped" + , defaultReadOptions + , "a\nx\n\"\"\ny\n" + , Cols (2, 1) [("a", texts ["x", "y"])] + ) + , + ( "noheader" + , defaultReadOptions{headerSpec = NoHeader} + , "1,2\n3,4\n" + , Cols (2, 2) [("0", ints [1, 3]), ("1", ints [2, 4])] + ) + , + ( "providenames_fewer" + , defaultReadOptions{headerSpec = ProvideNames ["x"]} + , "1,2\n3,4\n" + , Cols (2, 2) [("x", ints [1, 3]), ("1", ints [2, 4])] + ) + , -- D6: the always-short padded column is now row-aligned (all null). + + ( "providenames_more_D6" + , defaultReadOptions{headerSpec = ProvideNames ["x", "y", "z"]} + , "1,2\n3,4\n" + , Cols + (2, 3) + [("x", ints [1, 3]), ("y", ints [2, 4]), ("z", mtexts [Nothing, Nothing])] + ) + , + ( "either_read" + , (sample 2){safeRead = EitherRead} + , "a,b\n1,x\n2,y\nz,\n" + , Cols + (3, 2) + [ ("a", eti [Right 1, Right 2, Left "z"]) + , ("b", ett [Right "x", Right "y", Left ""]) + ] + ) + , + ( "maybe_read" + , defaultReadOptions{safeRead = MaybeRead} + , "a,b\n1,x\nNA,y\n" + , Cols (2, 2) [("a", mints [Just 1, Nothing]), ("b", mtexts [Just "x", Just "y"])] + ) + , + ( "schema_int_bad" + , defaultReadOptions{typeSpec = SpecifyTypes [("a", schemaTypeInt)] NoInference} + , "a,b\n1,p\nx,q\n3,r\n" + , Cols + (3, 2) + [("a", mints [Just 1, Nothing, Just 3]), ("b", texts ["p", "q", "r"])] + ) + , + ( "schema_int_missing" + , defaultReadOptions{typeSpec = SpecifyTypes [("a", schemaTypeInt)] NoInference} + , "a\n1\nNA\n\n3\n" + , Cols (3, 1) [("a", mints [Just 1, Nothing, Just 3])] + ) + , + ( "schema_text_missing" + , defaultReadOptions{typeSpec = SpecifyTypes [("a", schemaTypeText)] NoInference} + , "a\nx\nNA\n\nz\n" + , Cols (3, 1) [("a", mtexts [Just "x", Nothing, Just "z"])] + ) + , + ( "dates" + , defaultReadOptions + , "d\n2024-01-02\n2024-02-03\n" + , Cols (2, 1) [("d", days [fromGregorian 2024 1 2, fromGregorian 2024 2 3])] + ) + , + ( "dates_custom" + , defaultReadOptions{dateFormat = "%d/%m/%Y"} + , "d\n02/01/2024\n03/02/2024\n" + , Cols (2, 1) [("d", days [fromGregorian 2024 1 2, fromGregorian 2024 2 3])] + ) + , + ( "custom_sep" + , defaultReadOptions{columnSeparator = ';'} + , "a;b\n1;x\n2;y\n" + , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])] + ) + , + ( "custom_missing" + , defaultReadOptions{missingIndicators = ["foo"]} + , "a,b\n1,foo\n2,bar\n" + , Cols (2, 2) [("a", ints [1, 2]), ("b", mtexts [Nothing, Just "bar"])] + ) + , + ( "bools" + , defaultReadOptions + , "f\nTrue\nFalse\n" + , Cols (2, 1) [("f", boolsC [True, False])] + ) + , + ( "bool_ws" + , defaultReadOptions + , "f\n True\nFalse\n" + , Cols (2, 1) [("f", boolsC [True, False])] + ) + , + ( "single_space_field" + , defaultReadOptions + , "a,b\nx, \ny,z\n" + , Cols (2, 2) [("a", texts ["x", "y"]), ("b", mtexts [Nothing, Just "z"])] + ) + , + ( "row_cap" + , defaultReadOptions{numColumns = Just 2} + , "a\n1\n2\n3\n4\n" + , Cols (2, 1) [("a", ints [1, 2])] + ) + , + ( "row_cap_zero" + , defaultReadOptions{numColumns = Just 0} + , "a,b\n1,2\n" + , Cols (0, 2) [("a", mtexts []), ("b", mtexts [])] + ) + , + ( "trailing_sep" + , defaultReadOptions + , "a,b\n1,\n2,3\n" + , Cols (2, 2) [("a", ints [1, 2]), ("b", mints [Nothing, Just 3])] + ) + , ("header_only", defaultReadOptions, "a,b\n", Err "Empty CSV file") + , ("empty", defaultReadOptions, "", Err "Empty CSV file") + , ("only_blank_lines", defaultReadOptions, "\n\n\n", Err "Empty CSV file") + , + ( "all_null_col" + , defaultReadOptions + , "a,b\n,1\n,2\n" + , Cols (2, 2) [("a", mtexts [Nothing, Nothing]), ("b", ints [1, 2])] + ) + , + ( "int_overflow" + , defaultReadOptions + , "a\n9223372036854775808\n1\n" + , Cols (2, 1) [("a", dbls [9.223372036854776e18, 1.0])] + ) + , + ( "mixed_int_double" + , defaultReadOptions + , "a\n1\n2.5\n" + , Cols (2, 1) [("a", dbls [1.0, 2.5])] + ) + , + ( "int_then_text" + , sample 2 + , "a\n1\n2\nx\n" + , Cols (3, 1) [("a", texts ["1", "2", "x"])] + ) + , + ( "int_then_double" + , sample 2 + , "a\n1\n2\n3.5\n" + , Cols (3, 1) [("a", dbls [1.0, 2.0, 3.5])] + ) + , + ( "sample_all_null_then_data" + , sample 2 + , "a\n\n\n5\n7\n" + , Cols (2, 1) [("a", ints [5, 7])] + ) + , + ( "quoted_number" + , defaultReadOptions + , "a\n\"1\"\n\"2\"\n" + , Cols (2, 1) [("a", ints [1, 2])] + ) + , + ( "quoted_padded_number" + , defaultReadOptions + , "a\n\" 1 \"\n\"2\"\n" + , Cols (2, 1) [("a", ints [1, 2])] + ) + , + ( "quoted_header" + , defaultReadOptions + , "\"a b\",c\n1,2\n" + , Cols (1, 2) [("a b", ints [1]), ("c", ints [2])] + ) + , + ( "header_quoted_doubled" + , defaultReadOptions + , "\"a\"\"b\",c\n1,2\n" + , Cols (1, 2) [("a\"b", ints [1]), ("c", ints [2])] + ) + , + ( "quoted_crlf_field" + , defaultReadOptions + , "a,b\r\n\"x\r\ny\",2\r\n" + , Cols (1, 2) [("a", texts ["x\r\ny"]), ("b", ints [2])] + ) + , + ( "quoted_field_with_cr_alone" + , defaultReadOptions + , "a,b\n\"x\ry\",2\n" + , Cols (1, 2) [("a", texts ["x\ry"]), ("b", ints [2])] + ) + , + ( "tab_sep_quoted" + , defaultReadOptions{columnSeparator = '\t'} + , "a\tb\n\"x\t1\"\ty\n" + , Cols (1, 2) [("a", texts ["x\t1"]), ("b", texts ["y"])] + ) + , + ( "either_disables_missing" + , defaultReadOptions{safeReadOverrides = [("b", EitherRead)]} + , "a,b\nNA,1\n2,2\n" + , Cols (2, 2) [("a", texts ["NA", "2"]), ("b", eti [Right 1, Right 2])] + ) + , -- D6: the column after the EOF-truncated quote field is now padded. + + ( "unclosed_quote_D6" + , defaultReadOptions + , "a,b\n\"x,2\n" + , Cols (1, 2) [("a", texts ["x,2"]), ("b", mtexts [Nothing])] + ) + , + ( "unclosed_trailing_doubled_D6" + , defaultReadOptions + , "a,b\n\"x\"\"\n" + , Cols (1, 2) [("a", texts ["x\""]), ("b", mtexts [Nothing])] + ) + , + ( "closed_after_doubled_D6" + , defaultReadOptions + , "a,b\n\"x\"\"\"\n" + , Cols (1, 2) [("a", texts ["x\""]), ("b", mtexts [Nothing])] + ) + , + ( "multi_doubled" + , defaultReadOptions + , "a\n\"x\"\"y\"\"z\"\nw\n" + , Cols (2, 1) [("a", texts ["x\"y\"z", "w"])] + ) + , + ( "only_doubled" + , defaultReadOptions + , "a\n\"\"\"\"\nw\n" + , Cols (2, 1) [("a", texts ["\"", "w"])] + ) + , + ( "doubled_at_open" + , defaultReadOptions + , "a,b\n\"\"\"x\",2\n" + , Cols (1, 2) [("a", texts ["\"x"]), ("b", ints [2])] + ) + , + ( "interior_doubled_inferred" + , defaultReadOptions + , "a\n\"x\"\"y\"\nz\n" + , Cols (2, 1) [("a", texts ["x\"y", "z"])] + ) + , + ( "trailing_cr_eof" + , defaultReadOptions + , "a,b\n1,x\r" + , Cols (1, 2) [("a", ints [1]), ("b", texts ["x"])] + ) + , + ( "single_col_blank" + , defaultReadOptions + , "a\n1\n\n2\n" + , Cols (2, 1) [("a", ints [1, 2])] + ) + , + ( "single_col_quoted_empty" + , defaultReadOptions + , "a\n1\n\"\"\n2\n" + , Cols (2, 1) [("a", ints [1, 2])] + ) + , + ( "row_only_sep" + , defaultReadOptions + , "a,b\n,\n1,2\n" + , Cols (2, 2) [("a", mints [Nothing, Just 1]), ("b", mints [Nothing, Just 2])] + ) + , + ( "sep_at_eof" + , defaultReadOptions + , "a,b\n1,2\n3," + , Cols (2, 2) [("a", ints [1, 3]), ("b", mints [Just 2, Nothing])] + ) + , + ( "noheader_ragged_grow" + , defaultReadOptions{headerSpec = NoHeader} + , "1,2\n3,4,5\n" + , Cols (2, 2) [("0", ints [1, 3]), ("1", ints [2, 4])] + ) + , + ( "quoted_last_no_nl" + , defaultReadOptions + , "a,b\n1,2\n\"x\",3" + , Cols (2, 2) [("a", texts ["1", "x"]), ("b", ints [2, 3])] + ) + , + ( "empty_quoted_middle_col" + , defaultReadOptions + , "a,b,c\n1,\"\",3\n4,5,6\n" + , Cols + (2, 3) + [("a", ints [1, 4]), ("b", mints [Nothing, Just 5]), ("c", ints [3, 6])] + ) + , + ( "exp_double" + , defaultReadOptions + , "a\n1e3\n1E-3\n1e999\n" + , Cols (3, 1) [("a", dbls [1000.0, 1.0e-3, 1 / 0])] + ) + , + ( "five_field_double" + , defaultReadOptions + , "a\n+.5\n5.\n" + , Cols (2, 1) [("a", texts ["+.5", "5."])] + ) + , + ( "raw_invalid_utf8" + , defaultReadOptions + , BS.pack [97, 10, 255, 10, 98, 10] + , Cols (2, 1) [("a", texts ["\65533", "b"])] + ) + , + ( "raw_nbsp_edges_stripped" + , defaultReadOptions + , BS.pack [97, 10, 160, 120, 160, 10, 98, 10] + , Cols (2, 1) [("a", texts ["x", "b"])] + ) + ] + where + schemaTypeInt = schemaType @Int + schemaTypeText = schemaType @T.Text + +-- Duplicate header names: last index wins in the map; both columns kept. +dupHeaders :: Test +dupHeaders = TestLabel "dup_headers" $ TestCase $ do + df <- decodeSeparated defaultReadOptions (BL.fromStrict "a,a\n1,2\n3,4\n") + assertEqual "dims" (2, 2) (dataframeDimensions df) + assertEqual "index map" [("a", 1)] (M.toList (columnIndices df)) + +-- Ingest pin: a string column freezes to a shared-buffer 'PackedText' +-- (no per-row boxed Text header), while values stay byte-identical. +textFreezesPacked :: Test +textFreezesPacked = TestLabel "text_freezes_packed" $ TestCase $ do + df <- decodeSeparated defaultReadOptions (BL.fromStrict "a,b\n1,x\n2,y\n") + case getColumn "b" df of + Nothing -> assertFailure "missing text column" + Just col -> do + assertBool "text column is PackedText" (DI.isPackedText col) + assertEqual "values byte-identical" (texts ["x", "y"]) col + +tests :: [Test] +tests = dupHeaders : textFreezesPacked : map goldenCase goldenCases diff --git a/tests/Internal/ColumnBuilder.hs b/tests/Internal/ColumnBuilder.hs new file mode 100644 --- /dev/null +++ b/tests/Internal/ColumnBuilder.hs @@ -0,0 +1,298 @@ +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +module Internal.ColumnBuilder (tests, props) where + +import qualified Data.ByteString as B +import qualified Data.ByteString.Unsafe as BU +import qualified Data.Text as T +import qualified Data.Text.Array as A +import qualified Data.Vector as VB +import qualified Data.Vector.Unboxed as VU + +import Control.Monad (forM_, zipWithM_) +import Control.Monad.ST (runST, stToIO) +import Data.Maybe (catMaybes, isNothing) +import Data.Text.Encoding (decodeUtf8Lenient, encodeUtf8) +import Data.Type.Equality (testEquality, (:~:) (Refl)) +import Data.Word (Word8) +import DataFrame.Internal.Column hiding (mergeColumns) +import DataFrame.Internal.ColumnBuilder +import Foreign.Ptr (castPtr) +import Test.HUnit +import Test.QuickCheck +import Type.Reflection (typeRep) + +-- Build a column by appending each list element (Nothing => appendNull). +buildIntColumn :: [Maybe Int] -> Column +buildIntColumn xs = runST $ do + b <- newIntBuilder (length xs) + mapM_ (maybe (appendNull b) (appendInt b)) xs + freezeBuilder b + +buildDoubleColumn :: [Maybe Double] -> Column +buildDoubleColumn xs = runST $ do + b <- newDoubleBuilder (length xs) + mapM_ (maybe (appendNull b) (appendDouble b)) xs + freezeBuilder b + +buildTextColumn :: [Maybe T.Text] -> Column +buildTextColumn xs = runST $ do + b <- newTextBuilder (length xs) 16 + mapM_ (maybe (appendNull b) (appendText b)) xs + freezeBuilder b + +-- Build a text column from raw byte fields via appendTextSlice. +buildTextColumnFromBytes :: [[Word8]] -> Column +buildTextColumnFromBytes fields = runST $ do + b <- newTextBuilder (length fields) 16 + forM_ fields $ \ws -> + appendTextSlice b (arrayFromBytes ws) 0 (length ws) + freezeBuilder b + +arrayFromBytes :: [Word8] -> A.Array +arrayFromBytes ws = A.run $ do + m <- A.new (length ws) + zipWithM_ (A.unsafeWrite m) [0 ..] ws + pure m + +-- Reference column: fromVector of the plain list (no nulls) or Maybe list. +expectedColumn :: + forall a. + (Columnable a, Columnable (Maybe a)) => + [Maybe a] -> Column +expectedColumn xs + | any isNothing xs = fromVector (VB.fromList xs) + | otherwise = fromVector (VB.fromList (catMaybes xs)) + +-- Read a text column back as a Maybe list, respecting the bitmap. +-- Ingest now freezes text to 'PackedText'; materialize to the boxed form. +columnTexts :: Column -> [Maybe T.Text] +columnTexts c@(PackedText _ _) = columnTexts (materializePacked c) +columnTexts (BoxedColumn mb (v :: VB.Vector b)) = + case testEquality (typeRep @b) (typeRep @T.Text) of + Just Refl -> + [ if maybe True (`bitmapTestBit` i) mb + then Just (v VB.! i) + else Nothing + | i <- [0 .. VB.length v - 1] + ] + Nothing -> error "columnTexts: not a Text column" +columnTexts _ = error "columnTexts: not a boxed column" + +-- Split a list into arbitrary contiguous chunks (at least one chunk). +splitsOf :: [a] -> Gen [[a]] +splitsOf [] = pure [[]] +splitsOf xs = go xs + where + go [] = pure [] + go ys = do + k <- chooseInt (1, length ys) + let (a, b) = splitAt k ys + (a :) <$> go b + +nullableVersion :: [Maybe a] -> String -> String -> String +nullableVersion xs nullable plain = + if any isNothing xs then nullable else plain + +-- HUnit tests + +intAllValidHasNoBitmap :: Test +intAllValidHasNoBitmap = + TestCase $ + assertEqual + "all-valid int column is Unboxed with no bitmap" + "Unboxed" + (columnVersionString (buildIntColumn (map Just [1, 2, 3]))) + +intWithNullHasBitmap :: Test +intWithNullHasBitmap = + TestCase $ + assertEqual + "int column with null is NullableUnboxed" + "NullableUnboxed" + (columnVersionString (buildIntColumn [Just 1, Nothing])) + +intNullSentinelIsZero :: Test +intNullSentinelIsZero = TestCase $ + case buildIntColumn [Just 5, Nothing, Just 7] of + UnboxedColumn (Just _) (v :: VU.Vector b) -> + case testEquality (typeRep @b) (typeRep @Int) of + Just Refl -> assertEqual "null slot holds 0" 0 (v VU.! 1) + Nothing -> assertFailure "expected an Int column" + _ -> assertFailure "expected a nullable unboxed column" + +textAllValidHasNoBitmap :: Test +textAllValidHasNoBitmap = + TestCase $ + assertEqual + "all-valid text column is Boxed with no bitmap" + "Boxed" + (columnVersionString (buildTextColumn [Just "a", Just "bc"])) + +builderLengthCounts :: Test +builderLengthCounts = TestCase $ do + let n = runST $ do + b <- newIntBuilder 4 + appendInt b 1 + appendNull b + appendInt b 3 + builderLength b + assertEqual "builderLength counts nulls and values" 3 n + +growthPastCapacityHint :: Test +growthPastCapacityHint = TestCase $ do + let xs = map Just [0 .. 9999 :: Int] + col = runST $ do + b <- newIntBuilder 1 + mapM_ (maybe (appendNull b) (appendInt b)) xs + freezeBuilder b + assertEqual "doubling growth preserves data" (expectedColumn xs) col + +multibyteTextRoundTrip :: Test +multibyteTextRoundTrip = TestCase $ do + let ts = ["héllo", "wörld", "日本語", "😀😃", ""] + assertEqual + "multibyte text round trips" + (map Just ts) + (columnTexts (buildTextColumn (map Just ts))) + +-- A multibyte sequence split across two fields: the whole buffer is valid +-- UTF-8 but each field alone is not. Both must lenient-decode to U+FFFD. +splitMultibyteAcrossFields :: Test +splitMultibyteAcrossFields = TestCase $ do + let fields = [[0xC2], [0x80]] + assertEqual + "fields splitting a sequence decode leniently" + (map (Just . decodeUtf8Lenient . B.pack) fields) + (columnTexts (buildTextColumnFromBytes fields)) + +splitEuroAcrossFields :: Test +splitEuroAcrossFields = TestCase $ do + let fields = [[0xE2, 0x82], [0xAC, 0x61]] + assertEqual + "euro sign split across fields decodes leniently" + (map (Just . decodeUtf8Lenient . B.pack) fields) + (columnTexts (buildTextColumnFromBytes fields)) + +invalidUtf8EdgeCases :: Test +invalidUtf8EdgeCases = TestCase $ do + let fields = + [ [0xE2, 0x28, 0xA1] + , [0xF0, 0x9F, 0x98] + , [0xF0, 0x28] + , [0xC2] + , [0x80] + , [0xED, 0xA0, 0x80] + , [0xF4, 0x90, 0x80, 0x80] + , [0xC0, 0xAF] + , [0x61, 0xF1, 0x80, 0x80, 0xE1, 0x80, 0xC2, 0x62] + , [0xF0, 0x9F, 0x98, 0x80] + ] + assertEqual + "invalid sequences match decodeUtf8Lenient" + (map (Just . decodeUtf8Lenient . B.pack) fields) + (columnTexts (buildTextColumnFromBytes fields)) + +appendFromPtrMatchesText :: Test +appendFromPtrMatchesText = TestCase $ do + let t = "héllo wörld 😀" :: T.Text + col <- BU.unsafeUseAsCStringLen (encodeUtf8 t) $ \(p, len) -> + stToIO $ do + b <- newTextBuilder 1 0 + appendTextSliceFromPtr b (castPtr p) len + freezeBuilder b + assertEqual "ptr slice decodes to original text" [Just t] (columnTexts col) + +mergeBitmapSpliceUnaligned :: Test +mergeBitmapSpliceUnaligned = TestCase $ do + let nullIdx = [1, 4, 9, 10, 14] :: [Int] + xs = [if i `elem` nullIdx then Nothing else Just i | i <- [0 .. 14]] + chunks = [take 3 xs, take 5 (drop 3 xs), drop 8 xs] + merged = mergeColumns (map buildIntColumn chunks) + assertEqual "merged equals unsplit" (buildIntColumn xs) merged + forM_ [0 .. 14] $ \i -> + assertEqual + ("null at index " ++ show i) + (i `elem` nullIdx) + (columnElemIsNull merged i) + +tests :: [Test] +tests = + [ TestLabel "intAllValidHasNoBitmap" intAllValidHasNoBitmap + , TestLabel "intWithNullHasBitmap" intWithNullHasBitmap + , TestLabel "intNullSentinelIsZero" intNullSentinelIsZero + , TestLabel "textAllValidHasNoBitmap" textAllValidHasNoBitmap + , TestLabel "builderLengthCounts" builderLengthCounts + , TestLabel "growthPastCapacityHint" growthPastCapacityHint + , TestLabel "multibyteTextRoundTrip" multibyteTextRoundTrip + , TestLabel "splitMultibyteAcrossFields" splitMultibyteAcrossFields + , TestLabel "splitEuroAcrossFields" splitEuroAcrossFields + , TestLabel "invalidUtf8EdgeCases" invalidUtf8EdgeCases + , TestLabel "appendFromPtrMatchesText" appendFromPtrMatchesText + , TestLabel "mergeBitmapSpliceUnaligned" mergeBitmapSpliceUnaligned + ] + +-- QuickCheck properties + +prop_intMatchesFromVector :: [Maybe Int] -> Property +prop_intMatchesFromVector xs = + buildIntColumn xs === expectedColumn xs + .&&. columnVersionString (buildIntColumn xs) + === nullableVersion xs "NullableUnboxed" "Unboxed" + +prop_doubleMatchesFromVector :: [Maybe Double] -> Property +prop_doubleMatchesFromVector xs = + buildDoubleColumn xs === expectedColumn xs + +prop_textMatchesFromVector :: [Maybe String] -> Property +prop_textMatchesFromVector ss = + let xs = map (fmap T.pack) ss + in buildTextColumn xs === expectedColumn xs + .&&. columnVersionString (buildTextColumn xs) + === nullableVersion xs "NullableBoxed" "Boxed" + +prop_textSliceMatchesLenientDecode :: [[Word8]] -> Property +prop_textSliceMatchesLenientDecode fields = + columnTexts (buildTextColumnFromBytes fields) + === map (Just . decodeUtf8Lenient . B.pack) fields + +prop_mergeIntMatchesUnsplit :: [Maybe Int] -> Property +prop_mergeIntMatchesUnsplit xs = forAll (splitsOf xs) $ \chunks -> + let merged = mergeColumns (map buildIntColumn chunks) + unsplit = buildIntColumn xs + in merged === unsplit + .&&. columnVersionString merged === columnVersionString unsplit + +prop_mergeDoubleMatchesUnsplit :: [Maybe Double] -> Property +prop_mergeDoubleMatchesUnsplit xs = forAll (splitsOf xs) $ \chunks -> + mergeColumns (map buildDoubleColumn chunks) === buildDoubleColumn xs + +prop_mergeTextMatchesUnsplit :: [Maybe String] -> Property +prop_mergeTextMatchesUnsplit ss = + let xs = map (fmap T.pack) ss + in forAll (splitsOf xs) $ \chunks -> + let merged = mergeColumns (map buildTextColumn chunks) + unsplit = buildTextColumn xs + in merged === unsplit + .&&. columnTexts merged === columnTexts unsplit + +prop_mergeNullsLandAtRightRows :: [Maybe Int] -> Property +prop_mergeNullsLandAtRightRows xs = forAll (splitsOf xs) $ \chunks -> + let merged = mergeColumns (map buildIntColumn chunks) + in map isNothing xs + === [columnElemIsNull merged i | i <- [0 .. length xs - 1]] + +props :: [Property] +props = + [ property prop_intMatchesFromVector + , property prop_doubleMatchesFromVector + , property prop_textMatchesFromVector + , property prop_textSliceMatchesLenientDecode + , property prop_mergeIntMatchesUnsplit + , property prop_mergeDoubleMatchesUnsplit + , property prop_mergeTextMatchesUnsplit + , property prop_mergeNullsLandAtRightRows + ] diff --git a/tests/Internal/DictEncode.hs b/tests/Internal/DictEncode.hs new file mode 100644 --- /dev/null +++ b/tests/Internal/DictEncode.hs @@ -0,0 +1,144 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{- | Correctness oracle for the dictionary-encode building block +('DataFrame.Internal.DictEncode'). A text column encodes to dense +first-appearance @Int@ codes: two rows share a code iff their text is equal, the +codes are a contiguous @0 .. card-1@ range in first-appearance order, packed and +boxed Text encode identically, and the cap parameter bails past a cardinality. +Pins the step that the group-by planner can route a text key through. +-} +module Internal.DictEncode (tests) where + +import qualified Data.ByteString as B +import qualified Data.Map.Strict as M +import qualified Data.Text as T +import qualified Data.Text.Array as A +import qualified Data.Vector.Unboxed as VU + +import Control.Monad (zipWithM_) +import Data.Maybe (isJust) +import Data.Text.Encoding (encodeUtf8) +import Data.Word (Word8) +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.DictEncode (dictEncodeColumn, dictEncodeColumnUpTo) +import DataFrame.Internal.PackedText (mkPackedContiguous) +import Test.HUnit + +sampleRows :: [T.Text] +sampleRows = + [ "apple" + , "banana" + , "" + , "apple" + , "Zebra" + , "café" + , "naïve" + , "日本語" + , "banana" + , "apple" + ] + +arrayFromBytes :: [Word8] -> A.Array +arrayFromBytes ws = A.run $ do + m <- A.new (length ws) + zipWithM_ (A.unsafeWrite m) [0 ..] ws + pure m + +packedFromTexts :: [T.Text] -> DI.Column +packedFromTexts ts = + let bytess = map (B.unpack . encodeUtf8) ts + flat = concat bytess + offs = scanl (+) 0 (map length bytess) + arr = arrayFromBytes flat + in DI.PackedText Nothing (mkPackedContiguous arr (VU.fromList offs)) + +boxedFromTexts :: [T.Text] -> DI.Column +boxedFromTexts = DI.fromList + +{- | The reference dense first-appearance encoding computed with a plain +'Data.Map': scan in row order, assign the next id to each new value. +-} +oracleCodes :: [T.Text] -> ([Int], Int) +oracleCodes ts = go ts M.empty 0 [] + where + go [] _ next acc = (reverse acc, next) + go (x : xs) m next acc = case M.lookup x m of + Just c -> go xs m next (c : acc) + Nothing -> go xs (M.insert x next m) (next + 1) (next : acc) + +codesMatchOracle :: String -> DI.Column -> Test +codesMatchOracle lbl col = + TestCase $ case dictEncodeColumn col of + Nothing -> assertFailure (lbl ++ ": expected Just codes") + Just (codes, card) -> do + let (refCodes, refCard) = oracleCodes sampleRows + assertEqual (lbl ++ " codes") refCodes (VU.toList codes) + assertEqual (lbl ++ " cardinality") refCard card + +packedBoxedAgree :: Test +packedBoxedAgree = + TestCase $ + assertEqual + "packed and boxed encode identically" + (dictEncodeColumn (boxedFromTexts sampleRows)) + (dictEncodeColumn (packedFromTexts sampleRows)) + +denseRange :: Test +denseRange = + TestCase $ case dictEncodeColumn (packedFromTexts sampleRows) of + Nothing -> assertFailure "expected Just" + Just (codes, card) -> do + assertBool "codes in [0,card)" (VU.all (\c -> c >= 0 && c < card) codes) + assertEqual + "all codes used" + [0 .. card - 1] + (M.keys (M.fromList [(c, ()) | c <- VU.toList codes])) + +equalIffSameText :: Test +equalIffSameText = + TestCase $ case dictEncodeColumn (packedFromTexts sampleRows) of + Nothing -> assertFailure "expected Just" + Just (codes, _) -> + let pairs = [(i, j) | i <- idxs, j <- idxs, i < j] + idxs = [0 .. length sampleRows - 1] + ok (i, j) = + (VU.unsafeIndex codes i == VU.unsafeIndex codes j) + == (sampleRows !! i == sampleRows !! j) + in assertBool "code equality matches text equality" (all ok pairs) + +capBails :: Test +capBails = + TestCase $ do + -- sampleRows has 7 distinct values; a cap below that must bail. + assertEqual + "cap 3 bails" + Nothing + (dictEncodeColumnUpTo 3 (packedFromTexts sampleRows)) + -- a generous cap still encodes. + assertBool + "cap 100 encodes" + (isJust (dictEncodeColumnUpTo 100 (packedFromTexts sampleRows))) + +nonTextIsNothing :: Test +nonTextIsNothing = + TestCase $ + assertEqual + "int column is not dict-encoded" + Nothing + (dictEncodeColumn (DI.fromList [1 :: Int, 2, 3])) + +tests :: [Test] +tests = + [ TestLabel + "dictCodesPacked" + (codesMatchOracle "packed" (packedFromTexts sampleRows)) + , TestLabel + "dictCodesBoxed" + (codesMatchOracle "boxed" (boxedFromTexts sampleRows)) + , TestLabel "dictPackedBoxedAgree" packedBoxedAgree + , TestLabel "dictDenseRange" denseRange + , TestLabel "dictEqualIffSameText" equalIffSameText + , TestLabel "dictCapBails" capBails + , TestLabel "dictNonTextNothing" nonTextIsNothing + ] diff --git a/tests/Internal/PackedText.hs b/tests/Internal/PackedText.hs new file mode 100644 --- /dev/null +++ b/tests/Internal/PackedText.hs @@ -0,0 +1,186 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +{- | Correctness oracle for the packed-text column variant. Every assertion +builds a 'PackedText' column from raw bytes + offsets and the same data as a +'BoxedColumn Text', then checks display, extraction, equality, ordering, +hashing, groupBy and join all agree. This pins behavioral parity before any +reader emits packed columns. +-} +module Internal.PackedText (tests) where + +import qualified Data.Text as T +import qualified Data.Text.Array as A +import qualified Data.Vector.Unboxed as VU + +import Control.Monad (zipWithM_) +import qualified Data.ByteString as B +import Data.Text.Encoding (encodeUtf8) +import Data.Word (Word8) +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.DataFrame (unsafeGetColumn) +import DataFrame.Internal.PackedText (mkPackedContiguous) +import qualified DataFrame.Operations.Aggregation as Agg +import Test.HUnit + +-- Sample string corpus, including empty, ASCII, and multibyte UTF-8 fields. +sampleRows :: [T.Text] +sampleRows = + [ "apple" + , "banana" + , "" + , "apple" + , "Zebra" + , "café" + , "naïve" + , "日本語" + , "banana" + , "apple" + ] + +-- Build an A.Array from a flat byte list. +arrayFromBytes :: [Word8] -> A.Array +arrayFromBytes ws = A.run $ do + m <- A.new (length ws) + zipWithM_ (A.unsafeWrite m) [0 ..] ws + pure m + +-- Build a PackedText column directly from raw UTF-8 bytes + offsets. +packedFromTexts :: [T.Text] -> DI.Column +packedFromTexts ts = + let bytess = map (B.unpack . encodeUtf8) ts + flat = concat bytess + offs = scanl (+) 0 (map length bytess) + arr = arrayFromBytes flat + in DI.PackedText Nothing (mkPackedContiguous arr (VU.fromList offs)) + +boxedFromTexts :: [T.Text] -> DI.Column +boxedFromTexts = DI.fromList + +packedCol, boxedCol :: DI.Column +packedCol = packedFromTexts sampleRows +boxedCol = boxedFromTexts sampleRows + +-- Single-column dataframes for groupBy / hashing. +packedDf, boxedDf :: D.DataFrame +packedDf = D.fromNamedColumns [("k", packedCol)] +boxedDf = D.fromNamedColumns [("k", boxedCol)] + +displayParity :: Test +displayParity = + TestCase $ + assertEqual "show packed == show boxed" (show boxedCol) (show packedCol) + +columnEqParity :: Test +columnEqParity = TestCase $ do + assertBool "packed == boxed" (packedCol == boxedCol) + assertBool "boxed == packed" (boxedCol == packedCol) + assertBool "packed == packed" (packedCol == packedFromTexts sampleRows) + +extractionParity :: Test +extractionParity = + TestCase $ + assertEqual + "toList @Text packed == boxed" + (DI.toList @T.Text boxedCol) + (DI.toList @T.Text packedCol) + +orderingParity :: Test +orderingParity = TestCase $ do + let sortedPacked = D.sortBy [D.Asc (F.col @T.Text "k")] packedDf + sortedBoxed = D.sortBy [D.Asc (F.col @T.Text "k")] boxedDf + assertBool "asc sort parity" (sortedPacked == sortedBoxed) + let descPacked = D.sortBy [D.Desc (F.col @T.Text "k")] packedDf + descBoxed = D.sortBy [D.Desc (F.col @T.Text "k")] boxedDf + assertBool "desc sort parity" (descPacked == descBoxed) + +hashingParity :: Test +hashingParity = + TestCase $ + assertEqual + "row hashes equal" + (Agg.computeRowHashes [0] boxedDf) + (Agg.computeRowHashes [0] packedDf) + +groupByParity :: Test +groupByParity = + TestCase $ + assertEqual + "groupBy + distinct parity" + (D.distinct boxedDf) + (D.distinct packedDf) + +joinParity :: Test +joinParity = TestCase $ do + let lhsP = D.fromNamedColumns [("k", packedCol)] + lhsB = D.fromNamedColumns [("k", boxedCol)] + rhs = D.fromNamedColumns [("k", boxedFromTexts ["apple", "banana", "日本語"])] + joinedP = D.innerJoin ["k"] lhsP rhs + joinedB = D.innerJoin ["k"] lhsB rhs + assertBool "inner join parity" (joinedP == joinedB) + +-- A gather (atIndicesStable) on a packed column stays packed and equals the +-- boxed gather. Indices repeat, reorder, and drop rows. +gatherPreservesPacked :: Test +gatherPreservesPacked = TestCase $ do + let ixs = VU.fromList [9, 0, 0, 5, 7, 2, 3 :: Int] + gp = DI.atIndicesStable ixs packedCol + gb = DI.atIndicesStable ixs boxedCol + assertBool "gathered packed stays PackedText" (DI.isPackedText gp) + assertBool "gathered packed == gathered boxed" (gp == gb) + assertEqual + "gathered packed toList == boxed" + (DI.toList @T.Text gb) + (DI.toList @T.Text gp) + +-- A sort on a packed column stays packed and equals the boxed sort. +sortPreservesPacked :: Test +sortPreservesPacked = TestCase $ do + let sortedP = D.sortBy [D.Asc (F.col @T.Text "k")] packedDf + sortedB = D.sortBy [D.Asc (F.col @T.Text "k")] boxedDf + assertBool + "sorted packed column stays PackedText" + (DI.isPackedText (unsafeGetColumn "k" sortedP)) + assertBool "sorted packed == sorted boxed" (sortedP == sortedB) + +-- A left join (sentinel gather, -1 for missing right rows) on a packed right +-- column stays packed, builds the correct null bitmap, and equals boxed. +leftJoinSentinelPreservesPacked :: Test +leftJoinSentinelPreservesPacked = TestCase $ do + let lhsP = D.fromNamedColumns [("k", packedCol)] + lhsB = D.fromNamedColumns [("k", boxedCol)] + rhsP = + D.fromNamedColumns + [ ("k", packedFromTexts ["apple", "banana", "日本語"]) + , ("v", packedFromTexts ["RA", "RB", "RC"]) + ] + rhsB = + D.fromNamedColumns + [ ("k", boxedFromTexts ["apple", "banana", "日本語"]) + , ("v", boxedFromTexts ["RA", "RB", "RC"]) + ] + joinedP = D.leftJoin ["k"] lhsP rhsP + joinedB = D.leftJoin ["k"] lhsB rhsB + assertBool + "left-joined packed right column stays PackedText" + (DI.isPackedText (unsafeGetColumn "v" joinedP)) + assertBool "left join packed == boxed" (joinedP == joinedB) + +tests :: [Test] +tests = + [ TestLabel "PackedText display parity" displayParity + , TestLabel "PackedText equality parity" columnEqParity + , TestLabel "PackedText extraction parity" extractionParity + , TestLabel "PackedText ordering parity" orderingParity + , TestLabel "PackedText hashing parity" hashingParity + , TestLabel "PackedText groupBy parity" groupByParity + , TestLabel "PackedText join parity" joinParity + , TestLabel "PackedText gather preserves packed" gatherPreservesPacked + , TestLabel "PackedText sort preserves packed" sortPreservesPacked + , TestLabel + "PackedText left-join sentinel preserves packed" + leftJoinSentinelPreservesPacked + ] diff --git a/tests/LazyParity.hs b/tests/LazyParity.hs new file mode 100644 --- /dev/null +++ b/tests/LazyParity.hs @@ -0,0 +1,152 @@ +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | The HARD GATE for the lazy engine: for a bounded source, the lazy result +must be BYTE-IDENTICAL to the eager result computed with the same eager ops +('Join.join' / 'Agg.aggregate' . 'Agg.groupBy' / 'Perm.sortBy'). + +The lazy executor routes bounded HashJoin/HashAggregate through the whole-frame +eager fast paths, so 'show' of the two results must agree exactly. Both a LEFT +join pipeline (join -> groupBy -> aggregate -> sort) and a pure groupBy pipeline +are covered. +-} +module LazyParity (tests) where + +import qualified Data.Map.Strict as M +import Data.Text (Text) +import qualified Data.Text as T +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import qualified DataFrame.IO.CSV as Csv +import qualified DataFrame.Internal.Column as DI +import qualified DataFrame.Internal.Expression as E +import DataFrame.Internal.Schema (Schema (..), schemaType) +import qualified DataFrame.Lazy as L +import DataFrame.Lazy.Internal.LogicalPlan (SortOrder (Descending)) +import qualified DataFrame.Operations.Aggregation as Agg +import DataFrame.Operations.Join (JoinType (LEFT)) +import qualified DataFrame.Operations.Join as Join +import qualified DataFrame.Operations.Permutation as Perm +import DataFrame.Operators (as, (|>)) +import System.Directory (removeFile) +import System.IO.Temp (emptySystemTempFile) +import Test.HUnit + +ordersSchema :: Schema +ordersSchema = + Schema $ + M.fromList + [ ("order_id", schemaType @Int) + , ("customer_id", schemaType @Int) + , ("amount", schemaType @Double) + , ("discount", schemaType @Double) + ] + +customersSchema :: Schema +customersSchema = + Schema $ + M.fromList + [ ("customer_id", schemaType @Int) + , ("region", schemaType @Text) + , ("plan", schemaType @Text) + ] + +{- | @n@ orders; ~10% reference a customer id with no matching customer row so +the LEFT join produces a Nothing group (exercises unmatched-row semantics). +-} +ordersFrame :: Int -> D.DataFrame +ordersFrame n = + D.fromNamedColumns + [ ("order_id", DI.fromList [0 .. n - 1]) + , + ( "customer_id" + , DI.fromList + [if i `mod` 10 == 0 then 9_000_000 + i else i `mod` 257 | i <- [0 .. n - 1]] + ) + , ("amount", DI.fromList [fromIntegral i * 1.5 :: Double | i <- [0 .. n - 1]]) + , + ( "discount" + , DI.fromList [fromIntegral (i `mod` 7) * 0.25 :: Double | i <- [0 .. n - 1]] + ) + ] + +customersFrame :: Int -> D.DataFrame +customersFrame m = + D.fromNamedColumns + [ ("customer_id", DI.fromList [0 .. m - 1]) + , ("region", DI.fromList [T.pack ("r" ++ show (i `mod` 4)) | i <- [0 .. m - 1]]) + , ("plan", DI.fromList [T.pack ("p" ++ show (i `mod` 3)) | i <- [0 .. m - 1]]) + ] + +-- | Write a frame to a temp CSV and run an action with the path. +withCsv :: D.DataFrame -> (FilePath -> IO a) -> IO a +withCsv df k = do + csvPath <- emptySystemTempFile "lazy_parity_.csv" + Csv.writeCsv csvPath df + r <- k csvPath + removeFile csvPath + return r + +{- | LEFT join 20k orders to 257 customers -> groupBy region,plan +-> sum(amount-discount), count -> sort revenue desc. +-} +joinPipelineParity :: Test +joinPipelineParity = + TestCase $ + withCsv (ordersFrame 20_000) $ \ordersPath -> + withCsv (customersFrame 257) $ \customersPath -> do + let amount = F.col @Double "amount" + discount = F.col @Double "discount" + aggs = + [ F.sum (amount - discount) `as` "revenue" + , F.count amount `as` "orders" + ] + -- Eager: the exact ops the lazy executor delegates to. + ordersDf <- Csv.readCsvWithSchema ordersSchema ordersPath + customersDf <- Csv.readCsvWithSchema customersSchema customersPath + let eager = + Perm.sortBy [Perm.Desc (E.Col @Double "revenue")] $ + Agg.aggregate aggs $ + Agg.groupBy ["region", "plan"] $ + Join.join LEFT ["customer_id"] customersDf ordersDf + -- Lazy: same query through the bounded-source fast path. + let customersQ = L.scanCsv customersSchema (T.pack customersPath) + lazy <- + L.scanCsv ordersSchema (T.pack ordersPath) + |> (\o -> L.join LEFT "customer_id" "customer_id" o customersQ) + |> L.groupBy ["region", "plan"] aggs + |> L.sortBy [("revenue", Descending)] + |> L.runDataFrame + assertEqual + "join pipeline lazy == eager (byte-identical)" + (show eager) + (show lazy) + +-- | Pure groupBy customer_id -> sum(amount), count, sort desc. +groupByPipelineParity :: Test +groupByPipelineParity = + TestCase $ + withCsv (ordersFrame 20_000) $ \ordersPath -> do + let amount = F.col @Double "amount" + aggs = + [ F.sum amount `as` "total" + , F.count amount `as` "n" + ] + ordersDf <- Csv.readCsvWithSchema ordersSchema ordersPath + let eager = + Perm.sortBy [Perm.Desc (E.Col @Double "total")] $ + Agg.aggregate aggs $ + Agg.groupBy ["customer_id"] ordersDf + lazy <- + L.scanCsv ordersSchema (T.pack ordersPath) + |> L.groupBy ["customer_id"] aggs + |> L.sortBy [("total", Descending)] + |> L.runDataFrame + assertEqual + "groupBy pipeline lazy == eager (byte-identical)" + (show eager) + (show lazy) + +tests :: [Test] +tests = [joinPipelineParity, groupByPipelineParity] diff --git a/tests/Learn/Denotation.hs b/tests/Learn/Denotation.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/Denotation.hs @@ -0,0 +1,94 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | The "predict is the model's denotation" claim, discharged as a test rather +than left as a slogan: the @Expr@ that 'predict' compiles must evaluate to the +same numbers as the fitted record's own parameters, computed independently here +in plain Haskell. If @affineExpr@ / @argMinExpr@ ever drift from the record they +are built from, these fail. +-} +module Learn.Denotation (tests) where + +import qualified Data.Text as T +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import DataFrame.Internal.Column (TypedColumn (..), toVector) +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.Expression (Expr) +import DataFrame.Internal.Interpreter (interpret) + +import DataFrame.KMeans +import DataFrame.LinearModel +import DataFrame.Model (fit, predict) + +import Test.HUnit + +interpD :: D.DataFrame -> Expr Double -> [Double] +interpD df e = case interpret @Double df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c) + Left err -> error (show err) + +interpI :: D.DataFrame -> Expr Int -> [Int] +interpI df e = case interpret @Int df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c) + Left err -> error (show err) + +df :: D.DataFrame +df = + D.fromNamedColumns + [ ("x1", DI.fromList xs1) + , ("x2", DI.fromList xs2) + , ("y", DI.fromList [3 + 2 * a - 0.5 * b | (a, b) <- zip xs1 xs2]) + ] + where + xs1 = [1, 2, 3, 4, 5, 6, 7, 8] :: [Double] + xs2 = [2, 1, 4, 3, 6, 5, 8, 7] :: [Double] + +col :: D.DataFrame -> T.Text -> [Double] +col d n = interpD d (F.col @Double n) + +-- | @interpret (predict m)@ must equal @intercept + Σ coefⱼ·featureⱼ@ from the record. +linearDenotation :: Test +linearDenotation = TestCase $ do + let m = fit defaultLinearConfig (F.col @Double "y") df + feats = V.toList (regFeatureNames m) + coefs = VU.toList (regCoef m) + cols = map (col df) feats + native = + [ regIntercept m + sum (zipWith (*) coefs row) + | row <- transposeL cols + ] + symbolic = interpD df (predict m) + assertBool + "linear: predict Expr matches the record's affine prediction" + (and (zipWith (\a b -> abs (a - b) < 1e-9) native symbolic)) + +-- | @interpret (predict km)@ must equal the nearest-centroid label from @kmCenters@. +kmeansDenotation :: Test +kmeansDenotation = TestCase $ do + let feats = ["x1", "x2"] + km = fit defaultKMeansConfig{kmK = 3, kmSeed = 1} (map (F.col @Double) feats) df + centers = map VU.toList (V.toList (kmCenters km)) + rows = transposeL (map (col df) feats) + native = [nearest centers row | row <- rows] + symbolic = interpI df (predict km) + assertEqual + "kmeans: predict Expr matches nearest-centroid label" + native + symbolic + where + nearest centers row = + snd (minimum [(sqDist c row, i) | (i, c) <- zip [0 ..] centers]) + sqDist c row = sum [(a - b) ^ (2 :: Int) | (a, b) <- zip c row] + +transposeL :: [[a]] -> [[a]] +transposeL [] = [] +transposeL xss + | any null xss = [] + | otherwise = map head xss : transposeL (map tail xss) + +tests :: [Test] +tests = [linearDenotation, kmeansDenotation] diff --git a/tests/Learn/EdgeCases.hs b/tests/Learn/EdgeCases.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/EdgeCases.hs @@ -0,0 +1,481 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +{- | Edge-case / degeneracy tests (tests.md category 7) and numerical-stability +tests (category 8) for @dataframe-learn@. + +Each test asserts the *mathematically correct* result (computed by hand, with +the derivation in a comment) or a *specific* documented degenerate behaviour -- +never "it returned something". Where the library's contract is to clamp/guard a +degeneracy (e.g. @k > n@, constant columns) we assert the concrete clamped +result; where the underlying routine is stable we assert it matches the closed +form a naive implementation would get wrong. +-} +module Learn.EdgeCases (tests) where + +import qualified Data.Map.Strict as M +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import DataFrame.Internal.Column (TypedColumn (..), toVector) +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.Expression (Expr) +import DataFrame.Internal.Interpreter (interpret) + +import DataFrame.GMM +import DataFrame.KMeans +import DataFrame.LinearAlgebra (logSumExp) +import DataFrame.LinearAlgebra.Eigen (jacobiEigenSym) +import DataFrame.LinearAlgebra.Solve (choleskySolve) +import DataFrame.LinearModel +import DataFrame.LinearSolver (sigmoid) +import DataFrame.Model (fit, predict) +import DataFrame.PCA + +import DataFrame.Internal.Statistics (correlation', variance') + +import Test.HUnit + +-- Helpers (mirrors Learn.Models) -------------------------------------------- + +interpD :: D.DataFrame -> Expr Double -> [Double] +interpD df e = case interpret @Double df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c) + Left err -> error (show err) + +interpI :: D.DataFrame -> Expr Int -> [Int] +interpI df e = case interpret @Int df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c) + Left err -> error (show err) + +mat :: [[Double]] -> V.Vector (VU.Vector Double) +mat = V.fromList . map VU.fromList + +close :: Double -> Double -> Double -> Bool +close tol a b = abs (a - b) <= tol + +finite :: Double -> Bool +finite x = not (isNaN x) && not (isInfinite x) + +-- =========================================================================== +-- Category 8: numerical stability +-- =========================================================================== + +{- logSumExp on a large-positive cluster. + log(e^1000 + e^1001 + e^1002) + = 1002 + log(e^-2 + e^-1 + 1) + e^-2 = 0.1353352832366127, e^-1 = 0.36787944117144233 + sum = 1.5032147243... ; log(sum) = 0.40760596444... + => 1002.4076059644... + A naive `log . sum . map exp` overflows to +Inf here. -} +testLogSumExpLargePositive :: Test +testLogSumExpLargePositive = TestCase $ do + let lse = logSumExp (VU.fromList [1000, 1001, 1002]) + expected = 1002 + log (exp (-2) + exp (-1) + 1) + assertBool "lse large-positive is finite" (finite lse) + assertBool + "lse [1000,1001,1002] == 1002 + log(e^-2+e^-1+1)" + (close 1e-9 lse expected) + -- sanity on the literal so the test pins the actual number, not just the formula + assertBool "lse literal ~ 1002.40760596" (close 1e-7 lse 1002.4076059644443) + +{- logSumExp on a large-negative cluster. + log(e^-1000 + e^-1001 + e^-1002) + = -1000 + log(1 + e^-1 + e^-2) [max is -1000] + = -1000 + 0.40760596444... + = -999.59239403556... + A naive implementation underflows every term to 0 -> log 0 = -Inf. -} +testLogSumExpLargeNegative :: Test +testLogSumExpLargeNegative = TestCase $ do + let lse = logSumExp (VU.fromList [-1000, -1001, -1002]) + expected = -1000 + log (1 + exp (-1) + exp (-2)) + assertBool "lse large-negative is finite" (finite lse) + assertBool + "lse [-1000,-1001,-1002] == -1000 + log(1+e^-1+e^-2)" + (close 1e-9 lse expected) + assertBool "lse literal ~ -999.59239403" (close 1e-7 lse (-999.5923940355557)) + +{- logSumExp of a single element is that element exactly (m + log 1). -} +testLogSumExpSingleton :: Test +testLogSumExpSingleton = TestCase $ do + assertBool "lse [42] == 42" (close 0 (logSumExp (VU.fromList [42])) 42) + +{- sigmoid at extreme arguments must stay in [0,1] and not NaN/Inf. + sigmoid(1000) is 1 up to rounding (1 - ~0); sigmoid(-1000) is 0. + The naive 1/(1+exp(-z)) overflows exp(1000)=Inf at z=-1000 -> 1/Inf = 0 ok, + but exp(-(-1000)) path; the stable branch in DataFrame.LinearSolver.sigmoid + picks ez/(1+ez) for z<0 so e^-1000 underflows to 0 cleanly -> 0. -} +testSigmoidExtreme :: Test +testSigmoidExtreme = TestCase $ do + let sp = sigmoid 1000 + sn = sigmoid (-1000) + assertBool "sigmoid(1000) finite" (finite sp) + assertBool "sigmoid(-1000) finite" (finite sn) + assertBool "sigmoid(1000) in [0,1]" (sp >= 0 && sp <= 1) + assertBool "sigmoid(-1000) in [0,1]" (sn >= 0 && sn <= 1) + -- saturated values: 1 and 0 to within machine epsilon + assertBool "sigmoid(1000) == 1" (close 1e-12 sp 1) + assertBool "sigmoid(-1000) == 0" (close 1e-12 sn 0) + -- sigmoid(0) is exactly 0.5 + assertBool "sigmoid(0) == 0.5" (close 1e-15 (sigmoid 0) 0.5) + -- antisymmetry: sigmoid(-z) == 1 - sigmoid(z) at a moderate z + assertBool + "sigmoid antisymmetric at 3" + (close 1e-12 (sigmoid (-3)) (1 - sigmoid 3)) + +{- Variance of large-but-low-variance data: [1e8+1, 1e8+2, 1e8+3]. + True sample variance (n-1) of {1,2,3} shifted by 1e8 is 1.0 exactly. + A naive sum-of-squares-minus-square-of-sum formula loses all precision here + (catastrophic cancellation) -> 0 or negative. Welford keeps it ~1. -} +testVarianceCatastrophicCancellation :: Test +testVarianceCatastrophicCancellation = TestCase $ do + let xs = VU.fromList [1e8 + 1, 1e8 + 2, 1e8 + 3] :: VU.Vector Double + v = variance' xs + assertBool "variance finite" (finite v) + assertBool "variance is positive" (v > 0) + -- sample variance of {1,2,3} = ((1-2)^2 + 0 + (3-2)^2)/(3-1) = 2/2 = 1 + assertBool "variance of shifted {1,2,3} == 1.0" (close 1e-6 v 1.0) + +{- Variance of an identical column is exactly 0 (not a tiny negative from + cancellation). -} +testVarianceConstant :: Test +testVarianceConstant = TestCase $ do + let v = variance' (VU.replicate 100 (7.0 :: Double)) + assertEqual "variance of constant column is 0" 0 v + +{- Variance of fewer than two samples is defined to be 0 (computeVariance guard), + not NaN from a /0. -} +testVarianceSingleton :: Test +testVarianceSingleton = TestCase $ do + assertEqual + "variance of one sample is 0" + 0 + (variance' (VU.fromList [3.5 :: Double])) + +{- Correlation of a perfectly linear pair is exactly +1 (and -1 reversed), + computed stably. y = 2x+1 over a spread of x. -} +testCorrelationPerfect :: Test +testCorrelationPerfect = TestCase $ do + let xs = VU.fromList [1, 2, 3, 4, 5] :: VU.Vector Double + ys = VU.map (\x -> 2 * x + 1) xs + yneg = VU.map (\x -> -(2 * x) + 1) xs + case correlation' xs ys of + Just r -> assertBool "corr(x, 2x+1) == 1" (close 1e-9 r 1) + Nothing -> assertFailure "expected a correlation" + case correlation' xs yneg of + Just r -> assertBool "corr(x, -2x+1) == -1" (close 1e-9 r (-1)) + Nothing -> assertFailure "expected a correlation" + +{- Degeneracy: correlation against a constant column. The denominator + sqrt(... * (n*Syy - Sy^2)) is 0, so the library returns Just NaN. This pins + the ACTUAL behaviour: it does not throw, but it does produce a NaN that the + caller must guard. If a future fix makes it return Nothing or 0 this test + flags the contract change. -} +testCorrelationConstantColumnIsNaN :: Test +testCorrelationConstantColumnIsNaN = TestCase $ do + let xs = VU.fromList [1, 2, 3, 4, 5] :: VU.Vector Double + ys = VU.replicate 5 (9.0 :: Double) + case correlation' xs ys of + Just r -> + assertBool + "corr with a zero-variance column is NaN (degenerate denom)" + (isNaN r) + Nothing -> assertFailure "correlation' returned Nothing (contract changed)" + +{- correlation' on n<2 returns Nothing (guarded), not a NaN. -} +testCorrelationTooFew :: Test +testCorrelationTooFew = TestCase $ do + assertEqual + "correlation of one point is Nothing" + Nothing + (correlation' (VU.fromList [1]) (VU.fromList [2])) + +-- =========================================================================== +-- Category 8: stability inside the model expr layer +-- =========================================================================== + +{- Logistic probability expressions at extreme feature values must remain in + [0,1] and finite. The prob expr is 1/(1+exp(-margin)); with a separating + model and |x| pushed to 1e6 the margin is huge, so a non-stable evaluation + could overflow. We assert every probability is finite and in [0,1] and that + the two one-vs-rest class probabilities are present. -} +testLogisticProbsExtremeFeatures :: Test +testLogisticProbsExtremeFeatures = TestCase $ do + let trainDf = + D.fromNamedColumns + [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double])) + , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int])) + ] + m = fit defaultLogisticConfig (F.col @Int "label") trainDf + probs = logisticProbExprs m + -- evaluate the class-1 probability at extreme inputs + extremeDf = + D.fromNamedColumns + [("x", DI.fromList ([-1e6, -1, 0, 1, 1e6] :: [Double]))] + assertEqual "two one-vs-rest probability exprs" 2 (M.size probs) + let allProbVals = + concat [interpD extremeDf e | e <- M.elems probs] + assertBool "all logistic probabilities finite" (all finite allProbVals) + assertBool + "all logistic probabilities in [0,1]" + (all (\p -> p >= 0 && p <= 1) allProbVals) + +-- =========================================================================== +-- Category 7: edge cases / degeneracy on models +-- =========================================================================== + +{- One-row OLS: a single point (x=2, y=7). With one observation OLS is + under-determined; the QR design [1, 2] is rank deficient (n=1 < d=2 cols of + the intercept-augmented matrix) so olsSolve falls back to ridge(1e-8). The + fitted line must at minimum interpolate the single training point: the + prediction at x=2 must equal 7 (within tolerance). It must NOT be NaN/Inf. -} +testOLSOneRow :: Test +testOLSOneRow = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([2] :: [Double])) + , ("y", DI.fromList ([7] :: [Double])) + ] + m = fit defaultLinearConfig (F.col @Double "y") df + preds = interpD df (predict m) + assertBool "single OLS prediction finite" (all finite preds) + assertBool + "OLS fits the single training point" + (case preds of [p] -> close 1e-3 p 7; _ -> False) + +{- All-identical labels in logistic regression: every row is class 1. There is + exactly one class, so predict must return that class for every row (no + division-by-zero, no crash, no spurious second class). -} +testLogisticSingleClass :: Test +testLogisticSingleClass = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([-2, -1, 0, 1, 2] :: [Double])) + , ("label", DI.fromList ([1, 1, 1, 1, 1] :: [Int])) + ] + m = fit defaultLogisticConfig (F.col @Int "label") df + preds = interpI df (predict m) + assertEqual + "single-class logistic predicts that class everywhere" + [1, 1, 1, 1, 1] + preds + +{- Perfectly separable logistic data with a constant (zero-variance) feature + added. The constant column has variance 0 and is dropped by keptIndices + (>= 1e-12 threshold); the informative column still separates the classes. + Prediction must recover the labels exactly and not be corrupted by the + constant column. -} +testLogisticConstantFeatureIgnored :: Test +testLogisticConstantFeatureIgnored = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double])) + , ("const", DI.fromList (replicate 8 (5.0 :: Double))) + , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int])) + ] + m = fit defaultLogisticConfig (F.col @Int "label") df + preds = interpI df (predict m) + assertEqual + "logistic ignores constant column and separates" + [0, 0, 0, 0, 1, 1, 1, 1] + preds + +{- Linear regression with an irrelevant constant feature: the constant column is + near-constant (variance 0) and must get a zero weight, while the informative + coefficient is recovered. y = 3x + 4 exactly. -} +testLinearConstantFeatureZeroWeight :: Test +testLinearConstantFeatureZeroWeight = TestCase $ do + let xs = [1, 2, 3, 4, 5, 6] :: [Double] + df = + D.fromNamedColumns + [ ("x", DI.fromList xs) + , ("const", DI.fromList (replicate 6 (5.0 :: Double))) + , ("y", DI.fromList [3 * x + 4 | x <- xs]) + ] + m = fit defaultLinearConfig (F.col @Double "y") df + coefs = VU.toList (regCoef m) + names = V.toList (regFeatureNames m) + byName = zip names coefs + -- find the const column weight; it must be ~0 (no information, possibly + -- collinear with the intercept) + case lookup "const" byName of + Just w -> assertBool "constant feature gets ~zero weight" (close 1e-6 w 0) + Nothing -> assertFailure "const column missing from feature names" + let preds = interpD df (predict m) + truth = [3 * x + 4 | x <- xs] + assertBool + "regression with constant feature still fits y=3x+4" + (and (zipWith (close 1e-4) preds truth)) + +{- k-means with k > n: requesting more clusters than data points. The library + clamps k = min k (max 1 n), so with 3 rows and kmK=10 we must get exactly 3 + centres (one per point), labels in range, and finite centres -- not a crash + or empty/duplicate-degenerate centres. -} +testKMeansKGreaterThanN :: Test +testKMeansKGreaterThanN = TestCase $ do + let df = + D.fromNamedColumns + [ ("a", DI.fromList ([0, 5, 10] :: [Double])) + , ("b", DI.fromList ([0, 5, 10] :: [Double])) + ] + cfg = defaultKMeansConfig{kmK = 10, kmNInit = 3, kmSeed = 1} + m = fit cfg [F.col @Double "a", F.col @Double "b"] df + assertEqual "k clamped to n=3 centres" 3 (V.length (kmCenters m)) + let labels = VU.toList (kmLabels m) + assertBool "all labels in [0,2]" (all (\l -> l >= 0 && l < 3) labels) + assertBool + "all centre coordinates finite" + (all (VU.all finite) (V.toList (kmCenters m))) + -- with 3 distinct points and 3 clusters, inertia must be ~0 (each point is + -- its own centre) + assertBool "k=n clustering has ~zero inertia" (close 1e-9 (kmInertia m) 0) + +{- k-means on an all-identical (zero-variance) feature set: every row is the same + point. Any clustering has inertia 0; centres must all equal that point and be + finite (no NaN from dividing by an empty cluster or by a zero distance sum in + k-means++ sampling). -} +testKMeansAllIdentical :: Test +testKMeansAllIdentical = TestCase $ do + let df = + D.fromNamedColumns + [ ("a", DI.fromList (replicate 6 (4.0 :: Double))) + , ("b", DI.fromList (replicate 6 (4.0 :: Double))) + ] + cfg = defaultKMeansConfig{kmK = 2, kmNInit = 3, kmSeed = 1} + m = fit cfg [F.col @Double "a", F.col @Double "b"] df + assertBool + "centres finite on degenerate identical data" + (all (VU.all finite) (V.toList (kmCenters m))) + assertBool "inertia is 0 for identical points" (close 1e-12 (kmInertia m) 0) + -- predict must still assign a valid (finite, in-range) label to every row + let assigns = interpI df (predict m) + assertBool "assignments in [0,1]" (all (\l -> l >= 0 && l < 2) assigns) + +{- GMM with k > n: clamped like k-means (k = min k (max 1 n)). Two rows, gmmK=5 + must yield 2 components, finite weights summing to ~1, and a finite log + likelihood -- not a crash or NaN. -} +testGMMKGreaterThanN :: Test +testGMMKGreaterThanN = TestCase $ do + let df = + D.fromNamedColumns + [ ("a", DI.fromList ([0, 10] :: [Double])) + , ("b", DI.fromList ([0, 10] :: [Double])) + ] + cfg = defaultGMMConfig{gmmK = 5, gmmSeed = 1} + m = fit cfg [F.col @Double "a", F.col @Double "b"] df + assertEqual "GMM k clamped to n=2" 2 (VU.length (gmmWeights m)) + assertBool "GMM weights finite" (VU.all finite (gmmWeights m)) + assertBool "GMM weights sum to ~1" (close 1e-6 (VU.sum (gmmWeights m)) 1) + assertBool "GMM log-likelihood finite" (finite (gmmLogLikelihood m)) + +{- PCA on a single informative axis plus a constant column: the constant column + carries zero variance, so its explained-variance ratio must be ~0 and the + ratios must still sum to ~1 and stay finite. The first component should align + with the varying axis. Catches a divide-by-zero in the ratio normalisation + when one eigenvalue is 0. -} +testPCAConstantColumn :: Test +testPCAConstantColumn = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([-2, -1, 0, 1, 2] :: [Double])) + , ("const", DI.fromList (replicate 5 (3.0 :: Double))) + ] + m = + fit + (PCAConfig (NComp 2) False) + [F.col @Double "x", F.col @Double "const"] + df + ratio = VU.toList (pcaExplainedVarianceRatio m) + assertBool "explained ratios finite" (all finite ratio) + assertBool "explained ratios sum to ~1" (close 1e-9 (sum ratio) 1) + -- the variance is entirely along x, so the top ratio must be ~1 + assertBool "first PC explains ~all variance" (close 1e-9 (head ratio) 1) + -- projection exprs must evaluate to finite numbers + let es = map snd (pcaExprs m) + assertBool + "pca exprs finite on constant column" + (all (all finite . interpD df) es) + +{- PCA on extreme-scale features (1e8-shifted) without standardisation: the + covariance entries are ~1 (variance of {1,2,3}) despite the 1e8 offset. + Components must stay unit-length and finite -- catches cancellation in the + covariance / eigendecomposition at large magnitudes. -} +testPCAExtremeScale :: Test +testPCAExtremeScale = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([1e8 + 1, 1e8 + 2, 1e8 + 3, 1e8 + 4] :: [Double])) + , ("y", DI.fromList ([1e8 + 1, 1e8 + 2, 1e8 + 3, 1e8 + 4] :: [Double])) + ] + m = + fit + (PCAConfig (NComp 1) False) + [F.col @Double "x", F.col @Double "y"] + df + comp0 = pcaComponents m V.! 0 + nrm = sqrt (VU.sum (VU.map (^ (2 :: Int)) comp0)) + assertBool "component finite at 1e8 scale" (VU.all finite comp0) + assertBool "component unit length at 1e8 scale" (close 1e-6 nrm 1) + assertBool + "explained ratio finite at 1e8 scale" + (VU.all finite (pcaExplainedVarianceRatio m)) + +-- =========================================================================== +-- Category 7/8: linear-algebra degeneracy +-- =========================================================================== + +{- choleskySolve on a non-positive-definite (zero) matrix returns Nothing rather + than crashing or producing NaNs. A 2x2 all-zero matrix has a zero pivot. -} +testCholeskyNonPD :: Test +testCholeskyNonPD = TestCase $ do + assertEqual + "cholesky of singular zero matrix is Nothing" + Nothing + (choleskySolve (mat [[0, 0], [0, 0]]) (VU.fromList [1, 1])) + +{- Jacobi eigendecomposition of a 1x1 matrix (one-column degenerate case): + eigenvalue is the single entry, eigenvector is [1] (sign-canonicalised). -} +testJacobi1x1 :: Test +testJacobi1x1 = TestCase $ do + let (ev, vecs) = jacobiEigenSym (mat [[5]]) + assertBool "1x1 eigenvalue is the entry" (close 1e-12 (ev VU.! 0) 5) + assertBool "1x1 eigenvector is [1]" (close 1e-12 ((vecs V.! 0) VU.! 0) 1) + +{- Jacobi on an identity matrix: both eigenvalues are exactly 1 and the result is + finite (the rotation angle code must not divide by zero when off-diagonals are + already 0). -} +testJacobiIdentity :: Test +testJacobiIdentity = TestCase $ do + let (ev, vecs) = jacobiEigenSym (mat [[1, 0], [0, 1]]) + assertBool "identity eigenvalues both 1" (VU.all (close 1e-12 1) ev) + assertBool "identity eigenvectors finite" (all (VU.all finite) (V.toList vecs)) + +tests :: [Test] +tests = + [ testLogSumExpLargePositive + , testLogSumExpLargeNegative + , testLogSumExpSingleton + , testSigmoidExtreme + , testVarianceCatastrophicCancellation + , testVarianceConstant + , testVarianceSingleton + , testCorrelationPerfect + , testCorrelationConstantColumnIsNaN + , testCorrelationTooFew + , testLogisticProbsExtremeFeatures + , testOLSOneRow + , testLogisticSingleClass + , testLogisticConstantFeatureIgnored + , testLinearConstantFeatureZeroWeight + , testKMeansKGreaterThanN + , testKMeansAllIdentical + , testGMMKGreaterThanN + , testPCAConstantColumn + , testPCAExtremeScale + , testCholeskyNonPD + , testJacobi1x1 + , testJacobiIdentity + ] diff --git a/tests/Learn/Ensembles.hs b/tests/Learn/Ensembles.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/Ensembles.hs @@ -0,0 +1,159 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +module Learn.Ensembles (tests) where + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import DataFrame.Internal.Column (TypedColumn (..), toVector) +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.Expression (Expr) +import DataFrame.Internal.Interpreter (interpret) + +import DataFrame.Boosting +import DataFrame.DBSCAN +import DataFrame.GMM +import DataFrame.LinearModel +import DataFrame.LinearSolver (defaultSolverConfig) +import DataFrame.Metrics (r2) +import DataFrame.ModelSelection + +import Data.Maybe (isJust, isNothing) +import qualified Data.Vector.Unboxed as VU +import DataFrame.Model (fit, predict) +import Test.HUnit + +interpD :: D.DataFrame -> Expr Double -> [Double] +interpD df e = case interpret @Double df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c) + Left err -> error (show err) + +interpI :: D.DataFrame -> Expr Int -> [Int] +interpI df e = case interpret @Int df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c) + Left err -> error (show err) + +clsDF :: D.DataFrame +clsDF = + D.fromNamedColumns + [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double])) + , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int])) + ] + +blobs :: D.DataFrame +blobs = + D.fromNamedColumns + [ ("a", DI.fromList ([0, 0.2, -0.1, 0.1, 8, 8.1, 7.9, 8.2] :: [Double])) + , ("b", DI.fromList ([0, -0.1, 0.2, 0.0, 5, 5.2, 4.9, 5.1] :: [Double])) + ] + +testGBMRegression :: Test +testGBMRegression = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([1 .. 12] :: [Double])) + , + ( "y" + , DI.fromList + ( [sin (fromIntegral i) + fromIntegral i * 0.3 | i <- [1 .. 12 :: Int]] :: + [Double] + ) + ) + ] + gb = + fit + defaultGBConfig{gbNEstimators = 60, gbMaxDepth = 2} + (F.col @Double "y") + df + preds = interpD df (predict gb) + truth = interpD df (F.col @Double "y") + err = sum (zipWith (\p t -> (p - t) ^ (2 :: Int)) preds truth) / 12 + assertBool "GBM fits training data well" (err < 0.05) + assertBool "trainScore recorded" (VU.length (gbTrainScore gb) == 60) + +testGBMStaged :: Test +testGBMStaged = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([1 .. 8] :: [Double])) + , ("y", DI.fromList ([1 .. 8] :: [Double])) + ] + gb = fit defaultGBConfig{gbNEstimators = 10} (F.col @Double "y") df + assertBool "stage 0 valid" (isJust (gbExprAtStage 0 gb)) + assertBool "out of range stage rejected" (isNothing (gbExprAtStage 11 gb)) + +testAdaBoost :: Test +testAdaBoost = TestCase $ do + let m = fit defaultAdaBoostConfig (F.col @Int "label") clsDF + preds = interpI clsDF (predict m) + assertEqual "AdaBoost separates" [0, 0, 0, 0, 1, 1, 1, 1] preds + +testGMM :: Test +testGMM = TestCase $ do + let m = + fit + defaultGMMConfig{gmmK = 2, gmmSeed = 1} + [F.col @Double "a", F.col @Double "b"] + blobs + assigns = interpI blobs (predict m) + assertBool "GMM converged" (gmmConverged m) + assertBool "GMM splits the blobs" (head assigns /= last assigns) + let m2 = + fit + defaultGMMConfig{gmmK = 2, gmmSeed = 1} + [F.col @Double "a", F.col @Double "b"] + blobs + assertEqual "GMM deterministic" (gmmMeans m) (gmmMeans m2) + assertBool "BIC finite" (not (isNaN (gmmBIC m)) && not (isInfinite (gmmBIC m))) + +testDBSCAN :: Test +testDBSCAN = TestCase $ do + let df = + D.fromNamedColumns + [ ("a", DI.fromList ([0, 0.1, 0.2, 5, 5.1, 5.2, 50] :: [Double])) + , ("b", DI.fromList ([0, 0.1, 0.0, 5, 5.0, 5.1, 50] :: [Double])) + ] + m = fit (DBSCANConfig 1.0 2) [F.col @Double "a", F.col @Double "b"] df + assertEqual "two clusters" 2 (dbNClusters m) + assertEqual "last point is noise" (-1) (VU.last (dbLabels m)) + let surrogate = + dbscanSurrogateExpr + D.defaultTreeConfig + [F.col @Double "a", F.col @Double "b"] + m + df + preds = interpI df surrogate + assertEqual + "surrogate matches non-noise labels on cores" + (take 6 (VU.toList (dbLabels m))) + (take 6 preds) + +testGridSearch :: Test +testGridSearch = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([1 .. 40] :: [Double])) + , + ( "y" + , DI.fromList ([2 * fromIntegral i + 5 | i <- [1 .. 40 :: Int]] :: [Double]) + ) + ] + score alpha train test = + let mdl = + fit (LinearConfig (Ridge alpha) defaultSolverConfig) (F.col @Double "y") train + in r2 + (VU.fromList (interpD test (predict mdl))) + (VU.fromList (interpD test (F.col @Double "y"))) + res = gridSearch 4 7 [0.0, 1.0, 100.0] score df + assertBool "best score high" (gsBestScore res > 0.99) + assertEqual "all configs scored" 3 (length (gsAll res)) + +tests :: [Test] +tests = + [ testGBMRegression + , testGBMStaged + , testAdaBoost + , testGMM + , testDBSCAN + , testGridSearch + ] diff --git a/tests/Learn/Metamorphic.hs b/tests/Learn/Metamorphic.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/Metamorphic.hs @@ -0,0 +1,366 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | Metamorphic, invariance, and abstraction-law tests for the ML library. + +These check how the fitted model and the metrics behave under transformations of +the *input* — duplicate / permute / scale / rename / add-constant-feature — where +the math dictates the answer. Each test computes the model on two genuinely +different (transformed) frames and asserts they agree, or pins a metric/scaler to +its defining law. None compares a value to itself; each can fail if the library +is wrong. +-} +module Learn.Metamorphic (tests) where + +import qualified Data.Text as T +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import DataFrame.Internal.Column (TypedColumn (..), toVector) +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.Expression (Expr) +import DataFrame.Internal.Interpreter (interpret) + +import DataFrame.LinearModel +import DataFrame.Metrics +import DataFrame.Model (fit, predict) +import DataFrame.Operations.Merge () + +-- Semigroup DataFrame (row concatenation) +import DataFrame.Transform + +import Test.HUnit + +-- Helpers ------------------------------------------------------------------- + +interpD :: D.DataFrame -> Expr Double -> [Double] +interpD df e = case interpret @Double df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c) + Left err -> error (show err) + +close :: Double -> Double -> Double -> Bool +close tol a b = abs (a - b) <= tol + +closeList :: Double -> [Double] -> [Double] -> Bool +closeList tol as bs = length as == length bs && and (zipWith (close tol) as bs) + +-- Build a regression frame from explicit feature/target lists. +mkRegDF :: [(T.Text, [Double])] -> [Double] -> D.DataFrame +mkRegDF feats ys = + D.fromNamedColumns + ([(n, DI.fromList vs) | (n, vs) <- feats] ++ [("y", DI.fromList ys)]) + +-- Base design: a noiseless linear target so OLS recovers a unique solution. +baseX1, baseX2, baseY :: [Double] +baseX1 = [1, 2, 3, 4, 5, 6, 7, 8] +baseX2 = [2, 1, 4, 3, 6, 5, 8, 7] +baseY = [3 + 2 * a - 0.5 * b | (a, b) <- zip baseX1 baseX2] + +baseDF :: D.DataFrame +baseDF = mkRegDF [("x1", baseX1), ("x2", baseX2)] baseY + +fitBase :: D.DataFrame -> LinearRegressor +fitBase = fit defaultLinearConfig (F.col @Double "y") + +-- Metamorphic: duplicate rows ---------------------------------------------- + +{- | OLS minimises Σ(y-ŷ)²; duplicating every row scales the loss by 2 but leaves +the argmin (the coefficients/intercept) unchanged. Catches a solver that +weights rows by frame size or mishandles the normal equations. +-} +testDuplicateRows :: Test +testDuplicateRows = TestCase $ do + let dupDF = + mkRegDF + [("x1", baseX1 ++ baseX1), ("x2", baseX2 ++ baseX2)] + (baseY ++ baseY) + m0 = fitBase baseDF + m1 = fitBase dupDF + assertBool + "duplicate rows: coefficients unchanged" + (closeList 1e-9 (VU.toList (regCoef m0)) (VU.toList (regCoef m1))) + assertBool + "duplicate rows: intercept unchanged" + (close 1e-9 (regIntercept m0) (regIntercept m1)) + +-- Metamorphic: permute rows ------------------------------------------------- + +{- | OLS is invariant to row order. We reverse the rows (a genuine non-identity +permutation) and require identical coefficients. Catches any order-dependence +in matrix assembly or solving. +-} +testPermuteRows :: Test +testPermuteRows = TestCase $ do + let permDF = + mkRegDF + [("x1", reverse baseX1), ("x2", reverse baseX2)] + (reverse baseY) + m0 = fitBase baseDF + m1 = fitBase permDF + assertBool + "permute rows: coefficients unchanged" + (closeList 1e-9 (VU.toList (regCoef m0)) (VU.toList (regCoef m1))) + assertBool + "permute rows: intercept unchanged" + (close 1e-9 (regIntercept m0) (regIntercept m1)) + +-- Metamorphic: scale a feature ---------------------------------------------- + +{- | Scaling feature x1 by k must scale that coefficient by 1/k while leaving +predictions (and every other coefficient + the intercept) invariant. Both +sides are genuine fits on different frames. Catches a regressor that drops or +mis-applies a feature's scale. +-} +testScaleFeature :: Test +testScaleFeature = TestCase $ do + let k = 10.0 + scaledDF = mkRegDF [("x1", map (* k) baseX1), ("x2", baseX2)] baseY + m0 = fitBase baseDF + m1 = fitBase scaledDF + coef0 = VU.toList (regCoef m0) + coef1 = VU.toList (regCoef m1) + assertBool + "scale feature: x1 coefficient scaled by 1/k" + (close 1e-7 (head coef0 / k) (head coef1)) + assertBool + "scale feature: x2 coefficient unchanged" + (close 1e-7 (coef0 !! 1) (coef1 !! 1)) + assertBool + "scale feature: intercept unchanged" + (close 1e-7 (regIntercept m0) (regIntercept m1)) + -- Predictions on each model's own (consistently transformed) frame agree. + let p0 = interpD baseDF (predict m0) + p1 = interpD scaledDF (predict m1) + assertBool "scale feature: predictions invariant" (closeList 1e-6 p0 p1) + +-- Metamorphic: rename columns ----------------------------------------------- + +{- | Renaming feature columns is a pure relabelling: coefficients, intercept, and +predictions are identical and the fitted feature names track the rename. +Catches any hidden dependence on specific column-name strings. +-} +testRenameColumns :: Test +testRenameColumns = TestCase $ do + let renamedDF = mkRegDF [("feat_a", baseX1), ("feat_b", baseX2)] baseY + m0 = fitBase baseDF + m1 = fitBase renamedDF + assertBool + "rename: coefficients unchanged" + (closeList 1e-9 (VU.toList (regCoef m0)) (VU.toList (regCoef m1))) + assertBool + "rename: intercept unchanged" + (close 1e-9 (regIntercept m0) (regIntercept m1)) + assertEqual + "rename: fitted feature names follow the rename" + ["feat_a", "feat_b"] + (V.toList (regFeatureNames m1)) + -- predict on the renamed frame yields the same numbers as on the original. + let p0 = interpD baseDF (predict m0) + p1 = interpD renamedDF (predict m1) + assertBool "rename: predictions unchanged" (closeList 1e-9 p0 p1) + +-- Metamorphic: irrelevant constant feature ---------------------------------- + +{- | A constant feature carries no variance, so OLS predictions for the real +features must be unchanged when one is added. (Its own weight is absorbed into +the intercept and is not separately identifiable, so we pin predictions, not +the weight.) Catches a fit whose answer drifts when a degenerate column is +present. +-} +testConstantFeatureIgnored :: Test +testConstantFeatureIgnored = TestCase $ do + let withConst = + mkRegDF + [ ("x1", baseX1) + , ("x2", baseX2) + , ("c", replicate (length baseX1) 7.0) + ] + baseY + m0 = fitBase baseDF + m1 = fitBase withConst + p0 = interpD baseDF (predict m0) + p1 = interpD withConst (predict m1) + assertBool + "constant feature: predictions match the no-constant model" + (closeList 1e-6 p0 p1) + -- And the real-feature coefficients are unchanged. + let coef1 = VU.toList (regCoef m1) + assertBool + "constant feature: x1 coefficient unchanged" + (close 1e-6 (head (VU.toList (regCoef m0))) (head coef1)) + assertBool + "constant feature: x2 coefficient unchanged" + (close 1e-6 (VU.toList (regCoef m0) !! 1) (coef1 !! 1)) + +-- Metamorphic: split then concatenate -------------------------------------- + +{- | Genuinely split the frame into two row-range sub-frames and concatenate them +back with the DataFrame @<>@ (row concatenation). The rebuilt frame has the +same rows, so the fit must be identical — but this actually exercises the +concatenation code path, so it catches a @<>@ that drops, duplicates, or +mis-aligns rows/columns (a plain @take h ++ drop h@ would be the identity and +test nothing). +-} +testSplitConcat :: Test +testSplitConcat = TestCase $ do + let n = length baseX1 + h = n `div` 2 + half lo hi = + mkRegDF + [ ("x1", slice lo hi baseX1) + , ("x2", slice lo hi baseX2) + ] + (slice lo hi baseY) + slice lo hi = take (hi - lo) . drop lo + rebuiltDF = half 0 h <> half h n + m0 = fitBase baseDF + m1 = fitBase rebuiltDF + -- the concatenated frame must have exactly the original rows back + assertEqual "split/concat: row count preserved" n (fst (D.dimensions rebuiltDF)) + assertBool + "split/concat: coefficients unchanged" + (closeList 1e-9 (VU.toList (regCoef m0)) (VU.toList (regCoef m1))) + assertBool + "split/concat: intercept unchanged" + (close 1e-9 (regIntercept m0) (regIntercept m1)) + +-- Training/inference parity: column order ---------------------------------- + +{- | The order of feature columns in the frame must not change predictions. +We fit on a frame whose columns are declared in the opposite order and require +the prediction expression to produce the same numbers on the original frame. +(predict compiles to a name-keyed affine Expr, so it should be order-free.) +Catches a positional, rather than name-keyed, feature mapping. +-} +testColumnOrderParity :: Test +testColumnOrderParity = TestCase $ do + let swappedDF = mkRegDF [("x2", baseX2), ("x1", baseX1)] baseY + m0 = fitBase baseDF + m1 = fitBase swappedDF + -- evaluate both fitted models on the SAME original frame. + p0 = interpD baseDF (predict m0) + p1 = interpD baseDF (predict m1) + assertBool + "column order: predictions on the same frame agree" + (closeList 1e-7 p0 p1) + +-- Law: standardScaler output has mean ~0, std ~1 --------------------------- + +{- | The defining law of a standard scaler: after transforming, each scaled +column has sample mean ≈ 0 and (population) std ≈ 1. We recompute the moments +here in plain Haskell from the transformed frame — not via the scaler — so a +wrong denominator or a centring bug is caught. +-} +testStandardScalerLaw :: Test +testStandardScalerLaw = TestCase $ do + let cols = ["x1", "x2"] + scaler = standardScaler cols baseDF + scaledDF = applyTransform (scalerTransform scaler) baseDF + moments xs = + let n = fromIntegral (length xs) + mu = sum xs / n + var = sum [(x - mu) ^ (2 :: Int) | x <- xs] / n + in (mu, sqrt var) + mapM_ + ( \c -> do + let (mu, sd) = moments (interpD scaledDF (F.col @Double c)) + assertBool ("scaler: " ++ T.unpack c ++ " mean ~ 0") (close 1e-9 mu 0) + assertBool ("scaler: " ++ T.unpack c ++ " std ~ 1") (close 1e-9 sd 1) + ) + cols + +{- | The scaler model's stored stats must match the data's own moments: a guard +against the scaler storing the wrong mean/std even if transform happens to +look plausible. Computed independently from the raw columns. +-} +testScalerStatsMatchData :: Test +testScalerStatsMatchData = TestCase $ do + let scaler = standardScaler ["x1", "x2"] baseDF + moments xs = + let n = fromIntegral (length xs) + mu = sum xs / n + in (mu, sqrt (sum [(x - mu) ^ (2 :: Int) | x <- xs] / n)) + (mu1, sd1) = moments baseX1 + (mu2, sd2) = moments baseX2 + assertBool + "scaler means match data" + (closeList 1e-9 (VU.toList (smMeans scaler)) [mu1, mu2]) + assertBool + "scaler stds match data" + (closeList 1e-9 (VU.toList (smStds scaler)) [sd1, sd2]) + +-- Metric laws -------------------------------------------------------------- + +{- | Perfect prediction → accuracy exactly 1.0; a single deliberate miss drops it +below 1. Pins both ends so a metric that ignores its inputs can't pass. +-} +testAccuracyLaw :: Test +testAccuracyLaw = TestCase $ do + let truth = VU.fromList [0, 1, 2, 1, 0, 2] + perfect = truth + oneWrong = VU.fromList [0, 1, 2, 1, 0, 0] + assertBool "accuracy: perfect = 1" (close 1e-12 (accuracy perfect truth) 1.0) + assertBool + "accuracy: one miss out of 6 = 5/6" + (close 1e-12 (accuracy oneWrong truth) (5 / 6)) + assertBool + "accuracy in [0,1]" + (let a = accuracy oneWrong truth in a >= 0 && a <= 1) + +{- | Accuracy is permutation-invariant: applying the same permutation to preds +and truth leaves it unchanged. Both vectors are genuinely reordered. +-} +testAccuracyPermInvariant :: Test +testAccuracyPermInvariant = TestCase $ do + let preds = VU.fromList [0, 0, 1, 1, 2, 2, 1, 0] + truth = VU.fromList [0, 0, 1, 2, 2, 2, 1, 0] + a0 = accuracy preds truth + a1 = accuracy (VU.reverse preds) (VU.reverse truth) + assertBool "accuracy: permutation invariant" (close 1e-12 a0 a1) + -- Sanity: the metric is non-trivial here (not 0 or 1), so invariance is meaningful. + assertBool "accuracy: non-degenerate baseline" (a0 > 0 && a0 < 1) + +{- | r² of a perfect fit is exactly 1; r² of predicting the constant mean is +exactly 0. These are the two anchor points of the R² definition. Catches a +swapped SS_res/SS_tot or a wrong sign. +-} +testR2Anchors :: Test +testR2Anchors = TestCase $ do + let truth = VU.fromList [1, 3, 2, 8, 5, 4] + meanT = VU.sum truth / fromIntegral (VU.length truth) + meanPred = VU.replicate (VU.length truth) meanT + assertBool "r2: perfect fit = 1" (close 1e-12 (r2 truth truth) 1.0) + assertBool "r2: predicting the mean = 0" (close 1e-12 (r2 meanPred truth) 0.0) + -- A strictly-worse-than-mean prediction gives r2 < 0 (not clamped). + let worse = VU.map (+ 100) truth + assertBool "r2: far-off prediction is negative" (r2 worse truth < 0) + +{- | r² on a fitted noiseless linear model equals 1, and rmse equals 0. Ties the +metric law to the model: if the fit is exact, the metric must say so. +-} +testR2OfFittedModel :: Test +testR2OfFittedModel = TestCase $ do + let m = fitBase baseDF + score = evaluate r2 (predict m) (F.col @Double "y") baseDF + err = evaluate rmse (predict m) (F.col @Double "y") baseDF + assertBool "r2 of exact linear fit ~ 1" (close 1e-9 score 1.0) + assertBool "rmse of exact linear fit ~ 0" (err < 1e-6) + +tests :: [Test] +tests = + [ testDuplicateRows + , testPermuteRows + , testScaleFeature + , testRenameColumns + , testConstantFeatureIgnored + , testSplitConcat + , testColumnOrderParity + , testStandardScalerLaw + , testScalerStatsMatchData + , testAccuracyLaw + , testAccuracyPermInvariant + , testR2Anchors + , testR2OfFittedModel + ] diff --git a/tests/Learn/MetricsTests.hs b/tests/Learn/MetricsTests.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/MetricsTests.hs @@ -0,0 +1,121 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +module Learn.MetricsTests (tests) where + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import qualified DataFrame.Internal.Column as DI + +import DataFrame.LinearModel +import DataFrame.LinearSolver (defaultSolverConfig) +import DataFrame.Metrics +import DataFrame.Metrics.Report +import DataFrame.ModelSelection +import DataFrame.PCA +import DataFrame.Transform + +import qualified Data.Vector.Unboxed as VU +import DataFrame.Model (fit, predict) +import Test.HUnit + +close :: Double -> Double -> Double -> Bool +close tol a b = abs (a - b) <= tol + +preds3, truth3 :: VU.Vector Double +preds3 = VU.fromList [0, 0, 1, 1, 2, 2, 1, 0] +truth3 = VU.fromList [0, 0, 1, 2, 2, 2, 1, 0] + +reg :: D.DataFrame +reg = + D.fromNamedColumns + [ ("x", DI.fromList ([1 .. 20] :: [Double])) + , + ( "y" + , DI.fromList ([2 * fromIntegral i + 1 | i <- [1 .. 20 :: Int]] :: [Double]) + ) + ] + +testRegressionMetrics :: Test +testRegressionMetrics = TestCase $ do + let p = VU.fromList [1, 2, 3, 4] + t = VU.fromList [1, 2, 3, 5] + assertBool "mse" (close 1e-9 (mse p t) 0.25) + assertBool "rmse" (close 1e-9 (rmse p t) 0.5) + assertBool "mae" (close 1e-9 (mae p t) 0.25) + assertBool "r2 in range" (r2 p t <= 1) + +testMulticlassMetrics :: Test +testMulticlassMetrics = TestCase $ do + assertBool "accuracy" (close 1e-9 (accuracy preds3 truth3) 0.875) + -- class 1: tp=2 (idx2,6), fp=1 (idx3) -> precision 2/3 + assertBool + "binary precision class 1" + (close 1e-9 (precision (Binary 1) preds3 truth3) (2 / 3)) + assertBool + "macro f1 sane" + (f1 Macro preds3 truth3 > 0.8 && f1 Macro preds3 truth3 <= 1) + -- micro f1 == accuracy for single-label + assertBool + "micro f1 == accuracy" + (close 1e-9 (f1 Micro preds3 truth3) (accuracy preds3 truth3)) + +testRocAuc :: Test +testRocAuc = TestCase $ do + let scores = VU.fromList [0.1, 0.4, 0.35, 0.8] + truth = VU.fromList [0, 0, 1, 1] + assertBool "perfect-ish auc high" (rocAuc scores truth >= 0.75) + assertBool "auc in [0,1]" (let a = rocAuc scores truth in a >= 0 && a <= 1) + +testReports :: Test +testReports = TestCase $ do + let cr = classificationReport preds3 truth3 + assertEqual "report covers 3 classes" 3 (length (crPerClass cr)) + assertBool "report accuracy" (close 1e-9 (crAccuracy cr) 0.875) + let rr = regressionReport (VU.fromList [1, 2, 3]) (VU.fromList [1, 2, 4]) + assertBool "regression report rmse" (rrRMSE rr > 0) + -- Show instances don't crash + assertBool "classification report shows" (not (null (show cr))) + assertBool "confusion shows" (not (null (show (confusionMatrix preds3 truth3)))) + +testEvaluateOneLiner :: Test +testEvaluateOneLiner = TestCase $ do + let m = fit defaultLinearConfig (F.col @Double "y") reg + score = evaluate rmse (predict m) (F.col @Double "y") reg + assertBool "evaluate rmse ~ 0 on exact linear fit" (score < 1e-6) + let r = regressionReportExpr (predict m) (F.col @Double "y") reg + assertBool "report r2 ~ 1" (close 1e-6 (rrR2 r) 1) + +testCrossValidate :: Test +testCrossValidate = TestCase $ do + let cv = + crossValidate + 4 + 0 + r2 + (F.col @Double "y") + (predict . fit defaultLinearConfig (F.col @Double "y")) + reg + assertBool "all folds high R2" (all (> 0.99) cv) + assertEqual "four folds" 4 (length cv) + +testTransformCompose :: Test +testTransformCompose = TestCase $ do + let scaler = standardScaler ["x"] reg + pca = fit (PCAConfig (NComp 1) False) [F.col @Double "x"] reg + -- the whole point: scaler <> pcaTransform compose as a monoid + pipeline = scalerTransform scaler <> pcaTransform pca + out = applyTransform pipeline reg + assertBool "pipeline produced a frame" (D.columnNames out /= []) + assertBool "pipeline has pc1 column" ("pc1" `elem` D.columnNames out) + +tests :: [Test] +tests = + [ testRegressionMetrics + , testMulticlassMetrics + , testRocAuc + , testReports + , testEvaluateOneLiner + , testCrossValidate + , testTransformCompose + ] diff --git a/tests/Learn/Models.hs b/tests/Learn/Models.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/Models.hs @@ -0,0 +1,169 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +module Learn.Models (tests) where + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import DataFrame.Internal.Column (TypedColumn (..), toVector) +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.Expression (Expr) +import DataFrame.Internal.Interpreter (interpret) + +import DataFrame.DecisionTree.Model +import DataFrame.DecisionTree.Regression +import DataFrame.KMeans +import DataFrame.LinearModel +import DataFrame.LinearSolver (SolverConfig (..), defaultSolverConfig) +import DataFrame.PCA +import DataFrame.SVM +import DataFrame.Transform + +import qualified Data.Map.Strict as M +import qualified Data.Text as T +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU +import DataFrame.Model (fit, predict) +import Test.HUnit + +interpD :: D.DataFrame -> Expr Double -> [Double] +interpD df e = case interpret @Double df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c) + Left err -> error (show err) + +interpI :: D.DataFrame -> Expr Int -> [Int] +interpI df e = case interpret @Int df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c) + Left err -> error (show err) + +close :: Double -> Double -> Double -> Bool +close tol a b = abs (a - b) <= tol + +regDF :: D.DataFrame +regDF = + D.fromNamedColumns + [ ("x1", DI.fromList xs1) + , ("x2", DI.fromList xs2) + , ("y", DI.fromList [2 * a - 3 * b + 1 | (a, b) <- zip xs1 xs2]) + ] + where + xs1 = [1, 2, 3, 4, 5, 6, 7, 8] :: [Double] + xs2 = [2, 1, 4, 3, 6, 5, 8, 7] :: [Double] + +clsDF :: D.DataFrame +clsDF = + D.fromNamedColumns + [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double])) + , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int])) + ] + +testOLS :: Test +testOLS = TestCase $ do + let m = fit defaultLinearConfig (F.col @Double "y") regDF + assertBool + "OLS coef ~ [2,-3]" + (and (zipWith (close 1e-6) (VU.toList (regCoef m)) [2, -3])) + assertBool "OLS intercept ~ 1" (close 1e-6 (regIntercept m) 1) + let preds = interpD regDF (predict m) + truth = interpD regDF (F.col @Double "y") + assertBool "OLS expr matches y" (and (zipWith (close 1e-6) preds truth)) + +testRidgeShrinks :: Test +testRidgeShrinks = TestCase $ do + let r0 = + fit (LinearConfig (Ridge 0.01) defaultSolverConfig) (F.col @Double "y") regDF + r1 = fit (LinearConfig (Ridge 100) defaultSolverConfig) (F.col @Double "y") regDF + norm = VU.sum . VU.map abs . regCoef + assertBool "stronger ridge shrinks coefficients" (norm r1 < norm r0) + +testLogistic :: Test +testLogistic = TestCase $ do + let m = fit defaultLogisticConfig (F.col @Int "label") clsDF + dec = predict m + preds = interpI clsDF dec + truth = [0, 0, 0, 0, 1, 1, 1, 1] + assertEqual "logistic separates" truth preds + let probs = logisticProbExprs m + assertBool "prob exprs present for both classes" (M.size probs == 2) + +testSVC :: Test +testSVC = TestCase $ do + let m = fit defaultSVCConfig (F.col @Int "label") clsDF + preds = interpI clsDF (predict m) + assertEqual "linear SVC separates" [0, 0, 0, 0, 1, 1, 1, 1] preds + +testRegressionTree :: Test +testRegressionTree = TestCase $ do + let m = fit defaultRegTreeConfig (F.col @Double "y") regDF + preds = interpD regDF (predict m) + truth = interpD regDF (F.col @Double "y") + sse = sum (zipWith (\p t -> (p - t) ^ (2 :: Int)) preds truth) + assertBool "regression tree reduces error" (sse < 200) + assertBool "tree has leaves" (dtrNLeaves m >= 2) + +testClassifierStats :: Test +testClassifierStats = TestCase $ do + let m = fit D.defaultTreeConfig (F.col @Int "label") clsDF + assertBool "classifier depth >= 1" (dtcDepth m >= 1) + +testPCA :: Test +testPCA = TestCase $ do + let m = + fit + (PCAConfig (NComp 2) False) + [F.col @Double "x1", F.col @Double "x2"] + regDF + ratio = VU.toList (pcaExplainedVarianceRatio m) + assertBool "explained ratio sums ~1" (close 1e-9 (sum ratio) 1) + assertEqual "two components" 2 (V.length (pcaComponents m)) + let comp0 = pcaComponents m V.! 0 + nrm = sqrt (VU.sum (VU.map (^ (2 :: Int)) comp0)) + assertBool "component is unit length" (close 1e-9 nrm 1) + let es = map snd (pcaExprs m) + assertBool + "pca exprs evaluate finite" + (not (any (any isNaN . interpD regDF) es)) + +testKMeans :: Test +testKMeans = TestCase $ do + let df = + D.fromNamedColumns + [ ("a", DI.fromList ([0, 0.1, 0.2, 10, 10.1, 10.2] :: [Double])) + , ("b", DI.fromList ([0, 0.1, 0, 10, 10, 10.1] :: [Double])) + ] + cfg = defaultKMeansConfig{kmK = 2, kmNInit = 5, kmSeed = 1} + m = fit cfg [F.col @Double "a", F.col @Double "b"] df + labels = VU.toList (kmLabels m) + assertBool "two blobs split" (head labels /= last labels) + assertEqual "two centers" 2 (V.length (kmCenters m)) + let m2 = fit cfg [F.col @Double "a", F.col @Double "b"] df + assertEqual "kmeans deterministic" (kmCenters m) (kmCenters m2) + let assigns = interpI df (predict m) + assertEqual "assign expr matches labels" labels assigns + +testTransformCompose :: Test +testTransformCompose = TestCase $ do + let scaler = standardScaler ["x1", "x2"] regDF + t = scalerTransform scaler + scaledDf = applyTransform t regDF + -- fit model on scaled features, then compile scaling into the expr + m = fit defaultLinearConfig (F.col @Double "y") scaledDf + composed = compileThrough t (predict m) + viaCompose = interpD regDF composed + viaStepwise = interpD scaledDf (predict m) + assertBool + "compileThrough == stepwise apply" + (and (zipWith (close 1e-6) viaCompose viaStepwise)) + +tests :: [Test] +tests = + [ testOLS + , testRidgeShrinks + , testLogistic + , testSVC + , testRegressionTree + , testClassifierStats + , testPCA + , testKMeans + , testTransformCompose + ] diff --git a/tests/Learn/NumericalRigor.hs b/tests/Learn/NumericalRigor.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/NumericalRigor.hs @@ -0,0 +1,438 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +{- | Numerical-rigor suite: gradient checks (cat 4), reproducibility across the +stochastic models (cat 16), and statistical / distributional properties of the +RNG and splitters (cat 5). Every test is constructed to FAIL on a real bug — no +@assertEqual v v@, no rubber tolerances. +-} +module Learn.NumericalRigor (tests) where + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import qualified DataFrame.Internal.Column as DI + +import DataFrame.GMM +import DataFrame.KMeans +import DataFrame.LinearSolver.Loss ( + SmoothLoss (..), + logisticLoss, + sigmoid, + sqHingeLoss, + squaredLoss, + ) +import DataFrame.Model (fit) +import DataFrame.ModelSelection (trainTestSplit) +import DataFrame.Random +import DataFrame.SVM.RFF + +import qualified Data.Vector.Unboxed as VU +import Test.HUnit + +-- --------------------------------------------------------------------------- +-- Independent reference loss VALUE functions. +-- +-- The library only exposes the gradient 'slGradZ'; the per-loss scalar value is +-- reconstructed here straight from the module's own documented definitions, so +-- the finite difference below is computed INDEPENDENTLY of the analytic +-- gradient (anti-tautology: never finite-difference a gradient against itself). +-- --------------------------------------------------------------------------- + +-- | @½ (z - y)²@ (squaredLoss). +squaredValue :: Double -> Double -> Double +squaredValue y z = 0.5 * (z - y) * (z - y) + +-- | @log (1 + exp (-y z))@ (logisticLoss). softplus form, numerically stable. +logisticValue :: Double -> Double -> Double +logisticValue y z = + let m = negate (y * z) + in if m >= 0 then m + log1pExp (negate m) else log1pExp m + where + log1pExp x = log (1 + exp x) + +-- | @(max 0 (1 - y z))²@ (sqHingeLoss). +sqHingeValue :: Double -> Double -> Double +sqHingeValue y z = let m = 1 - y * z in if m > 0 then m * m else 0 + +-- | Central finite difference of @f@ in @z@ at @(y, z)@. +centralDiff :: + (Double -> Double -> Double) -> Double -> Double -> Double -> Double +centralDiff f eps y z = (f y (z + eps) - f y (z - eps)) / (2 * eps) + +{- | Gradient-check one 'SmoothLoss' against its reference value fn over a grid +of @(y, z)@ points (both labels in @{ -1, +1 }@, many margins including the hinge +kink neighbourhood). Fails if the analytic gradient differs from the finite +difference anywhere beyond @tol@. +-} +gradCheck :: + String -> + SmoothLoss -> + (Double -> Double -> Double) -> + Double -> + Test +gradCheck nm loss valueFn tol = + TestCase $ + mapM_ check [(y, z) | y <- [-1, 1], z <- zs] + where + -- margins, deliberately avoiding the exact hinge kink z = y (m = 0) where + -- the squared-hinge second derivative is discontinuous; central diff is + -- still valid away from the kink. + zs = [-3.7, -2.1, -1.25, -0.6, -0.15, 0.22, 0.55, 1.4, 1.9, 2.8, 4.3] + check (y, z) = do + let analytic = slGradZ loss y z + fd = centralDiff valueFn 1e-6 y z + ok = abs (analytic - fd) <= tol * (1 + abs fd) + assertBool + ( nm + ++ " grad mismatch at (y=" + ++ show y + ++ ", z=" + ++ show z + ++ "): analytic=" + ++ show analytic + ++ " finite-diff=" + ++ show fd + ) + ok + +-- | The squared-loss gradient @z - y@ checked against ½(z-y)². +testSquaredGradient :: Test +testSquaredGradient = gradCheck "squared" squaredLoss squaredValue 1e-5 + +-- | The logistic gradient @-y·σ(-y z)@ checked against log(1+exp(-yz)). +testLogisticGradient :: Test +testLogisticGradient = gradCheck "logistic" logisticLoss logisticValue 1e-5 + +-- | The squared-hinge gradient @-2y·max(0,1-yz)@ checked against (max 0 (1-yz))². +testSqHingeGradient :: Test +testSqHingeGradient = gradCheck "squared_hinge" sqHingeLoss sqHingeValue 1e-5 + +{- | Sign sanity that cannot be passed by a self-referential check: at a point +where the loss is strictly increasing in @z@ the analytic gradient must be +positive (and vice versa). Catches a flipped-sign gradient even if its +magnitude were somehow right. logistic with y=+1 decreases in z, so grad < 0; +squared with z>y increases, so grad > 0. +-} +testGradientSigns :: Test +testGradientSigns = TestCase $ do + assertBool + "squared: grad>0 when z>y (loss rising)" + (slGradZ squaredLoss 1.0 3.0 > 0) + assertBool + "squared: grad<0 when z0" + (slGradZ logisticLoss (-1.0) 0.5 > 0) + assertBool + "sqHinge active margin y=+1: grad<0" + (slGradZ sqHingeLoss 1.0 0.0 < 0) + assertBool + "sqHinge satisfied margin: grad==0" + (slGradZ sqHingeLoss 1.0 5.0 == 0) + +-- --------------------------------------------------------------------------- +-- Statistical / distributional tests (cat 5). +-- +-- Tolerances are CI-derived from the sample size; with N = 100k the standard +-- error of a uniform-mean estimate is ~ (1/sqrt12)/sqrt(N) ~ 8.3e-4, so a 6σ +-- band is ~ 5e-3. A broken sampler (e.g. one that returns the same value, or +-- biased) falls well outside. +-- --------------------------------------------------------------------------- + +-- | Draw @n@ uniforms threading the generator; returns the sample list. +drawUniforms :: Int -> Gen -> [Double] +drawUniforms n g0 = go n g0 [] + where + go 0 _ acc = acc + go k g acc = let (x, g') = nextDouble g in go (k - 1) g' (x : acc) + +mean :: [Double] -> Double +mean xs = sum xs / fromIntegral (length xs) + +variance :: [Double] -> Double +variance xs = + let m = mean xs + in sum [(x - m) * (x - m) | x <- xs] / fromIntegral (length xs) + +{- | @nextDouble@ is ~uniform on [0,1): mean ≈ 0.5 and variance ≈ 1/12, each +within a 6σ CI for 100k samples, AND every draw is in [0,1). A constant or +out-of-range sampler fails; a biased one (mean drifts) fails. +-} +testUniformDistribution :: Test +testUniformDistribution = TestCase $ do + let n = 100000 :: Int + xs = drawUniforms n (mkGen 20240613) + m = mean xs + v = variance xs + seMean = (1 / sqrt 12) / sqrt (fromIntegral n) + assertBool "all uniforms in [0,1)" (all (\x -> x >= 0 && x < 1) xs) + assertBool + ("uniform mean ~0.5, got " ++ show m) + (abs (m - 0.5) <= 6 * seMean) + assertBool + ("uniform variance ~1/12, got " ++ show v) + (abs (v - 1 / 12) <= 0.01) + -- not a constant: spread across the unit interval + assertBool "uniform actually varies" (maximum xs - minimum xs > 0.9) + +{- | Box-Muller @gaussianVector@ produces standard normals: sample mean ≈ 0 and +variance ≈ 1 over 100k draws. SE of the mean is 1/sqrt(N) ~ 3.2e-3, so a 6σ +band ~ 2e-2. Catches a Box-Muller bug (wrong radius/angle, missing log) that +shifts the mean or scales the variance. +-} +testGaussianMoments :: Test +testGaussianMoments = TestCase $ do + let n = 100000 :: Int + (vec, _) = gaussianVector n (mkGen 777) + xs = VU.toList vec + m = mean xs + v = variance xs + seMean = 1 / sqrt (fromIntegral n) + assertBool "gaussianVector length" (VU.length vec == n) + assertBool + "gaussian samples finite" + (all (\x -> not (isNaN x) && not (isInfinite x)) xs) + assertBool + ("gaussian mean ~0, got " ++ show m) + (abs m <= 6 * seMean) + assertBool + ("gaussian variance ~1, got " ++ show v) + (abs (v - 1) <= 0.03) + -- tail mass: ~4.55% should be |x|>2 for a true N(0,1); allow a wide band. + let tail2 = fromIntegral (length (filter (\x -> abs x > 2) xs)) / fromIntegral n + assertBool + ("gaussian |x|>2 frequency ~0.0455, got " ++ show tail2) + (abs (tail2 - 0.0455) <= 0.01) + +{- | @trainTestSplit frac@ over many seeds: the realized train fraction matches +@frac@ within a binomial CI, and train+test always sums to the input row count +(cat 2 invariant). A split that loses/dupes rows, or ignores @frac@, fails. +-} +testSplitProportions :: Test +testSplitProportions = TestCase $ do + let nRows = 4000 + frac = 0.7 :: Double + df = D.fromNamedColumns [("x", DI.fromList [1 .. nRows :: Int])] + seeds = [1 .. 25] :: [Int] + -- binomial SE of the fraction for n=4000 ~ sqrt(0.7*0.3/4000) ~ 7.2e-3 + seFrac = sqrt (frac * (1 - frac) / fromIntegral nRows) + mapM_ + ( \s -> do + let (tr, te) = trainTestSplit frac s df + nTr = fst (D.dimensions tr) + nTe = fst (D.dimensions te) + realized = fromIntegral nTr / fromIntegral nRows + assertEqual + ("split preserves rows (seed " ++ show s ++ ")") + nRows + (nTr + nTe) + assertBool + ( "split fraction ~" + ++ show frac + ++ " (seed " + ++ show s + ++ "), got " + ++ show realized + ) + (abs (realized - frac) <= 5 * seFrac) + ) + seeds + +{- | k-means inertia on a clean, well-separated blob is stable across many +seeds and never beats the true within-cluster sum of squares of the generating +partition (the global optimum here is the obvious 2-cluster split). A broken +inertia (wrong distance, double counting) would report values below the optimum +or wildly seed-dependent. +-} +testKMeansInertiaStable :: Test +testKMeansInertiaStable = TestCase $ do + let + -- two tight gaussian-ish blobs around (0,0) and (10,10) + as = [0, 0.1, -0.1, 0.05, -0.05, 10, 10.1, 9.9, 10.05, 9.95] :: [Double] + bs = [0, -0.1, 0.1, 0.05, -0.05, 10, 9.9, 10.1, 9.95, 10.05] :: [Double] + df = D.fromNamedColumns [("a", DI.fromList as), ("b", DI.fromList bs)] + fitSeed s = + kmInertia $ + fit + defaultKMeansConfig{kmK = 2, kmNInit = 1, kmSeed = s} + [F.col @Double "a", F.col @Double "b"] + df + inertias = map fitSeed [0 .. 19] + -- optimum: each blob's SS about its own mean. Blob means are ~ the + -- centroid; compute the true within-cluster SS for the 2-cluster split. + blob1 = take 5 (zip as bs) + blob2 = zip (drop 5 as) (drop 5 bs) + ssOf pts = + let mx = sum (map fst pts) / 5 + my = sum (map snd pts) / 5 + in sum [(x - mx) ^ (2 :: Int) + (y - my) ^ (2 :: Int) | (x, y) <- pts] + optimum = ssOf blob1 + ssOf blob2 + best = minimum inertias + worst = maximum inertias + assertBool + "k-means inertia finite" + (all (\i -> not (isNaN i) && not (isInfinite i)) inertias) + assertBool + ( "k-means inertia never below optimum (" + ++ show optimum + ++ "), best=" + ++ show best + ) + (best >= optimum - 1e-9) + assertBool + ( "k-means inertia stable across seeds, best=" + ++ show best + ++ " worst=" + ++ show worst + ) + (worst - best <= 1e-6 + 1e-3 * optimum) + +-- --------------------------------------------------------------------------- +-- Reproducibility tests (cat 16). +-- +-- Each compares two SEPARATE fits with the same seed (determinism) AND, where +-- the seed is a real input, asserts that a DIFFERENT seed CAN change the model +-- (so "always returns a constant" wouldn't pass). Models that derive Eq are +-- compared with ==; others by a representative field or predicted expr. +-- --------------------------------------------------------------------------- + +blobsDF :: D.DataFrame +blobsDF = + D.fromNamedColumns + [ + ( "a" + , DI.fromList ([0, 0.2, -0.1, 0.1, 8, 8.1, 7.9, 8.2, 0.05, 8.05] :: [Double]) + ) + , + ( "b" + , DI.fromList ([0, -0.1, 0.2, 0.0, 5, 5.2, 4.9, 5.1, 0.1, 5.05] :: [Double]) + ) + ] + +-- | Many-cluster frame so k-means++ seeding genuinely depends on the seed. +spreadDF :: D.DataFrame +spreadDF = + D.fromNamedColumns + [ ("a", DI.fromList ([0, 1, 2, 10, 11, 12, 20, 21, 22, 30, 31, 32] :: [Double])) + , ("b", DI.fromList ([0, 1, 0, 10, 11, 10, 0, 1, 0, 10, 11, 10] :: [Double])) + ] + +{- | k-means: same seed → identical KMeansModel (Eq derived); a different seed +on a multi-blob frame CAN yield different centers (the seed actually drives +k-means++ init). Single-init so the seed is not washed out by nInit restarts. +-} +testKMeansReproducible :: Test +testKMeansReproducible = TestCase $ do + let cfg s = defaultKMeansConfig{kmK = 4, kmNInit = 1, kmMaxIter = 1, kmSeed = s} + run s = fit (cfg s) [F.col @Double "a", F.col @Double "b"] + a = run 1 spreadDF + b = run 1 spreadDF + assertEqual "k-means same seed identical model" a b + -- different seeds CAN differ: scan several and require at least one diff in + -- the initial centroids (1 iteration keeps the init's footprint). + let centersFor s = kmCenters (run s spreadDF) + base = centersFor 1 + anyDiffer = any (\s -> centersFor s /= base) [2, 3, 5, 7, 11, 13] + assertBool "k-means: a different seed changes the model" anyDiffer + +{- | GMM: same seed → identical GMMModel (Eq derived); a different seed CAN move +the fitted means (seed drives the responsibility init via sampleIndices). +-} +testGMMReproducible :: Test +testGMMReproducible = TestCase $ do + let cfg s = defaultGMMConfig{gmmK = 2, gmmMaxIter = 1, gmmSeed = s} + run s = fit (cfg s) [F.col @Double "a", F.col @Double "b"] blobsDF + a = run 1 + b = run 1 + assertEqual "GMM same seed identical model" a b + let meansFor s = gmmMeans (run s) + base = meansFor 1 + anyDiffer = any (\s -> meansFor s /= base) [2, 3, 4, 5, 6, 7, 8] + assertBool "GMM: a different seed changes the means" anyDiffer + +{- | RFF-SVM: same seed → identical random projection AND identical fitted SVC +coefficients; a different seed changes the random Fourier features. The model +type only derives Show, so compare representative numeric fields directly. +-} +testRFFReproducible :: Test +testRFFReproducible = TestCase $ do + let clsDF = + D.fromNamedColumns + [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double])) + , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int])) + ] + cfg s = defaultRFFConfig{rffD = 40, rffGamma = 0.2, rffSeed = s} + run s = fit (cfg s) (F.col @Int "label") clsDF + a = run 5 + b = run 5 + -- projection matrix and the fitted SVC coefficients must match bit-for-bit. + assertEqual "RFF same seed: same projection B" (rffB a) (rffB b) + assertEqual "RFF same seed: same coefficients" (rffCoef a) (rffCoef b) + assertEqual "RFF same seed: same intercept" (rffIntercept a) (rffIntercept b) + -- different seed: the random projection should differ. + let projFor s = rffB (run s) + base = projFor 5 + anyDiffer = any (\s -> projFor s /= base) [1, 2, 3, 6, 7, 8] + assertBool "RFF: a different seed changes the projection" anyDiffer + +-- --------------------------------------------------------------------------- +-- randomSplit / trainTestSplit determinism + row-count invariant (cat 16 + 2). +-- --------------------------------------------------------------------------- + +{- | @trainTestSplit@ with the same seed is bit-identical (compared via the +prettyPrinted frames), preserves total rows, and a different seed CAN change the +partition. +-} +testSplitReproducible :: Test +testSplitReproducible = TestCase $ do + let df = D.fromNamedColumns [("x", DI.fromList [1 .. 200 :: Int])] + (tr1, te1) = trainTestSplit 0.6 42 df + (tr2, te2) = trainTestSplit 0.6 42 df + assertBool "trainTestSplit same seed: same train" (tr1 == tr2) + assertBool "trainTestSplit same seed: same test" (te1 == te2) + assertEqual + "trainTestSplit preserves row count" + 200 + (fst (D.dimensions tr1) + fst (D.dimensions te1)) + let trainFor s = fst (trainTestSplit 0.6 s df) + base = trainFor 42 + anyDiffer = any (\s -> trainFor s /= base) [1, 2, 3, 7, 99] + assertBool "trainTestSplit: a different seed changes the partition" anyDiffer + +-- --------------------------------------------------------------------------- +-- A self-consistency sanity for the helpers above so a broken reference value +-- fn cannot make the gradient checks vacuous: the reference squared value must +-- actually be ½(z-y)² at a known point. +-- --------------------------------------------------------------------------- +testReferenceValueSanity :: Test +testReferenceValueSanity = TestCase $ do + assertBool + "squaredValue at (y=2,z=5) == 4.5" + (abs (squaredValue 2 5 - 4.5) < 1e-12) + assertBool + "logisticValue at (y=1,z=0) == log 2" + (abs (logisticValue 1 0 - log 2) < 1e-12) + assertBool "sqHingeValue at (y=1,z=0) == 1" (abs (sqHingeValue 1 0 - 1) < 1e-12) + assertBool "sigmoid 0 == 0.5" (abs (sigmoid 0 - 0.5) < 1e-12) + +tests :: [Test] +tests = + [ testReferenceValueSanity + , testSquaredGradient + , testLogisticGradient + , testSqHingeGradient + , testGradientSigns + , testUniformDistribution + , testGaussianMoments + , testSplitProportions + , testKMeansInertiaStable + , testKMeansReproducible + , testGMMReproducible + , testRFFReproducible + , testSplitReproducible + ] diff --git a/tests/Learn/Numerics.hs b/tests/Learn/Numerics.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/Numerics.hs @@ -0,0 +1,99 @@ +{-# LANGUAGE ScopedTypeVariables #-} + +module Learn.Numerics (tests) where + +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU +import DataFrame.Model (fit, predict) +import Test.HUnit + +import DataFrame.LinearAlgebra +import DataFrame.LinearAlgebra.Eigen +import DataFrame.LinearAlgebra.Solve +import DataFrame.Random + +mat :: [[Double]] -> Matrix +mat = V.fromList . map VU.fromList + +approx :: Double -> Double -> Double -> Bool +approx tol a b = abs (a - b) <= tol + +vApprox :: Double -> VU.Vector Double -> VU.Vector Double -> Bool +vApprox tol a b = + VU.length a == VU.length b && VU.and (VU.zipWith (approx tol) a b) + +testQR :: Test +testQR = TestCase $ do + let a = mat [[1, 1], [1, 2], [1, 3], [1, 4]] + b = VU.fromList [3, 5, 7, 9] + case qrLeastSquares a b of + Right x -> + assertBool "QR recovers [1,2]" (vApprox 1e-9 x (VU.fromList [1, 2])) + Left cols -> assertFailure ("unexpected rank deficiency: " ++ show cols) + +testQRRankDeficient :: Test +testQRRankDeficient = TestCase $ do + let a = mat [[1, 2], [2, 4], [3, 6]] + case qrLeastSquares a (VU.fromList [1, 2, 3]) of + Left _ -> pure () + Right _ -> assertFailure "expected rank deficiency on collinear columns" + +testCholesky :: Test +testCholesky = TestCase $ do + let a = mat [[4, 2], [2, 3]] + case choleskySolve a (VU.fromList [2, 1]) of + Just x -> + assertBool "cholesky solves" (vApprox 1e-9 x (VU.fromList [0.5, 0])) + Nothing -> assertFailure "expected PD" + assertEqual "non-PD rejected" Nothing (cholesky (mat [[1, 2], [2, 1]])) + +testJacobi :: Test +testJacobi = TestCase $ do + let (ev, vecs) = jacobiEigenSym (mat [[2, 1], [1, 2]]) + assertBool "eigenvalues [3,1]" (vApprox 1e-9 ev (VU.fromList [3, 1])) + let v0 = vecs V.! 0 + recon = matVec (mat [[2, 1], [1, 2]]) v0 + assertBool + "A v = lambda v" + (vApprox 1e-9 recon (scaleV (ev VU.! 0) v0)) + +testPowerIter :: Test +testPowerIter = TestCase $ do + let (lam, _) = powerIterTop 200 (mat [[2, 1], [1, 2]]) + assertBool "dominant eigenvalue ~3" (approx 1e-6 lam 3) + +testLogSumExp :: Test +testLogSumExp = TestCase $ do + let xs = VU.fromList [1000, 1001, 1002] + lse = logSumExp xs + assertBool "logSumExp stable, finite" (not (isNaN lse) && not (isInfinite lse)) + assertBool "logSumExp >= max" (lse >= 1002) + +testRngDeterminism :: Test +testRngDeterminism = TestCase $ do + let (a, _) = shuffleInts 50 (mkGen 11) + (b, _) = shuffleInts 50 (mkGen 11) + assertEqual "same seed same shuffle" a b + assertBool + "is a permutation" + (VU.toList (VU.modify (const (pure ())) a) /= [] && all (`VU.elem` a) [0 .. 49]) + +testRngRange :: Test +testRngRange = TestCase $ do + let go 0 _ acc = acc + go k g acc = + let (x, g') = nextIntR (3, 7) g in go (k - 1 :: Int) g' (x : acc) + xs = go 1000 (mkGen 5) [] + assertBool "ints within range" (all (\x -> x >= 3 && x <= 7) xs) + +tests :: [Test] +tests = + [ testQR + , testQRRankDeficient + , testCholesky + , testJacobi + , testPowerIter + , testLogSumExp + , testRngDeterminism + , testRngRange + ] diff --git a/tests/Learn/SklearnParity.hs b/tests/Learn/SklearnParity.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/SklearnParity.hs @@ -0,0 +1,188 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | Parity tests against scikit-learn on clean Kaggle-style datasets. The +reference values in @data/ml/golden.json@ are produced by +@scripts/gen_sklearn_golden.py@. Closed-form models (OLS, ridge, PCA) are held to +tight coefficient parity; iterative models to an accuracy/inertia floor. +-} +module Learn.SklearnParity (tests) where + +import Data.Aeson (Value (..), decodeFileStrict') +import qualified Data.Aeson.Key as K +import qualified Data.Aeson.KeyMap as KM +import Data.Maybe (fromMaybe) +import qualified Data.Text as T +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU +import DataFrame.Model (fit, predict) +import Test.HUnit + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import DataFrame.Internal.Column (TypedColumn (..), toVector) +import DataFrame.Internal.Expression (Expr) +import DataFrame.Internal.Interpreter (interpret) + +import DataFrame.Boosting ( + GBConfig (..), + GBLoss (..), + defaultGBConfig, + gbProbaExpr, + ) +import DataFrame.KMeans +import DataFrame.LinearModel +import DataFrame.LinearSolver (defaultSolverConfig) +import DataFrame.PCA +import DataFrame.SVM + +goldenPath :: FilePath +goldenPath = "data/ml/golden.json" + +loadGolden :: IO Value +loadGolden = + fromMaybe (error "missing data/ml/golden.json") <$> decodeFileStrict' goldenPath + +(.!) :: Value -> T.Text -> Value +(.!) (Object o) k = fromMaybe Null (KM.lookup (K.fromText k) o) +(.!) _ _ = Null + +asNum :: Value -> Double +asNum (Number s) = realToFrac s +asNum _ = error "asNum: not a number" + +asNums :: Value -> [Double] +asNums (Array a) = map asNum (V.toList a) +asNums _ = error "asNums: not an array" + +asMatrix :: Value -> [[Double]] +asMatrix (Array a) = map asNums (V.toList a) +asMatrix _ = error "asMatrix: not a matrix" + +interpD :: D.DataFrame -> Expr Double -> [Double] +interpD df e = case interpret @Double df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c) + Left err -> error (show err) + +colDoubles :: D.DataFrame -> T.Text -> [Double] +colDoubles df name = interpD df (F.col @Double name) + +closeAll :: Double -> [Double] -> [Double] -> Bool +closeAll tol a b = + length a == length b && and (zipWith (\x y -> abs (x - y) <= tol) a b) + +accuracyOf :: [Double] -> [Double] -> Double +accuracyOf preds truth = + fromIntegral (length (filter id (zipWith (==) preds truth))) + / fromIntegral (max 1 (length truth)) + +testOLSParity :: Test +testOLSParity = TestCase $ do + g <- loadGolden + df <- D.readCsv "data/ml/regression.csv" + let m = fit defaultLinearConfig (F.col @Double "target") df + goldCoef = asNums (g .! "ols" .! "coef") + goldB = asNum (g .! "ols" .! "intercept") + assertBool + "OLS coefficients match sklearn" + (closeAll 1e-4 (VU.toList (regCoef m)) goldCoef) + assertBool "OLS intercept matches sklearn" (abs (regIntercept m - goldB) < 1e-4) + +testRidgeParity :: Test +testRidgeParity = TestCase $ do + g <- loadGolden + df <- D.readCsv "data/ml/regression.csv" + let m = + fit (LinearConfig (Ridge 1.0) defaultSolverConfig) (F.col @Double "target") df + goldCoef = asNums (g .! "ridge" .! "coef") + assertBool + "Ridge coefficients match sklearn" + (closeAll 1e-2 (VU.toList (regCoef m)) goldCoef) + +testPCAParity :: Test +testPCAParity = TestCase $ do + g <- loadGolden + df <- D.readCsv "data/ml/iris.csv" + let feats = + map + (F.col @Double) + ["sepal_length", "sepal_width", "petal_length", "petal_width"] + m = fit (PCAConfig (NComp 2) False) feats df + goldEvr = asNums (g .! "pca" .! "evr") + goldComp = asMatrix (g .! "pca" .! "components_abs") + ourComp = map (map abs . VU.toList) (V.toList (pcaComponents m)) + assertBool + "PCA explained variance ratio matches sklearn" + (closeAll 1e-4 (VU.toList (pcaExplainedVarianceRatio m)) goldEvr) + assertBool + "PCA |components| match sklearn" + (and (zipWith (closeAll 1e-3) ourComp goldComp)) + +testLogisticIrisParity :: Test +testLogisticIrisParity = TestCase $ do + g <- loadGolden + df <- D.readCsv "data/ml/iris.csv" + let m = fit defaultLogisticConfig (F.col @Double "species") df + preds = interpD df (predict m) + truth = colDoubles df "species" + acc = accuracyOf preds truth + gold = asNum (g .! "logistic_iris" .! "accuracy") + assertBool + ("logistic iris accuracy within 0.06 of sklearn " ++ show (acc, gold)) + (acc >= gold - 0.06) + +testSVCParity :: Test +testSVCParity = TestCase $ do + g <- loadGolden + df <- D.readCsv "data/ml/iris_binary.csv" + let m = fit defaultSVCConfig (F.col @Double "label") df + preds = interpD df (predict m) + truth = colDoubles df "label" + acc = accuracyOf preds truth + gold = asNum (g .! "linear_svc" .! "accuracy") + assertBool + ("linear SVC accuracy within 0.06 of sklearn " ++ show (acc, gold)) + (acc >= gold - 0.06) + +testGBMParity :: Test +testGBMParity = TestCase $ do + g <- loadGolden + df <- D.readCsv "data/ml/iris_binary.csv" + let m = + fit + defaultGBConfig{gbLoss = LogisticDeviance, gbNEstimators = 100} + (F.col @Double "label") + df + probs = interpD df (gbProbaExpr m) + preds = map (\p -> if p > 0.5 then 1 else 0) probs + truth = colDoubles df "label" + acc = accuracyOf preds truth + gold = asNum (g .! "gbm" .! "accuracy") + assertBool + ("GBM accuracy within 0.1 of sklearn " ++ show (acc, gold)) + (acc >= gold - 0.1) + +testKMeansParity :: Test +testKMeansParity = TestCase $ do + g <- loadGolden + df <- D.readCsv "data/ml/blobs.csv" + let m = + fit + defaultKMeansConfig{kmK = 3, kmNInit = 10, kmSeed = 0} + [F.col @Double "x", F.col @Double "y"] + df + gold = asNum (g .! "kmeans" .! "inertia") + assertBool + ("k-means inertia within 10% of sklearn " ++ show (kmInertia m, gold)) + (abs (kmInertia m - gold) / gold < 0.1) + +tests :: [Test] +tests = + [ testOLSParity + , testRidgeParity + , testPCAParity + , testLogisticIrisParity + , testSVCParity + , testGBMParity + , testKMeansParity + ] diff --git a/tests/Learn/Symbolic.hs b/tests/Learn/Symbolic.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/Symbolic.hs @@ -0,0 +1,134 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +module Learn.Symbolic (tests) where + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import DataFrame.Internal.Column (TypedColumn (..), toVector) +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.Expression (Expr) +import DataFrame.Internal.Interpreter (interpret) + +import DataFrame.PCA.Kernel +import DataFrame.SVM.RFF +import DataFrame.SymbolicRegression +import DataFrame.SymbolicRegression.Expr +import DataFrame.SymbolicRegression.Optimize (meanSquaredError) +import DataFrame.SymbolicRegression.Simplify (simplify) + +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU +import DataFrame.Model (fit, predict) +import Test.HUnit + +interpD :: D.DataFrame -> Expr Double -> [Double] +interpD df e = case interpret @Double df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c) + Left err -> error (show err) + +interpI :: D.DataFrame -> Expr Int -> [Int] +interpI df e = case interpret @Int df e of + Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c) + Left err -> error (show err) + +testSRRecovers :: Test +testSRRecovers = TestCase $ do + let xs = [-3, -2, -1, 0, 1, 2, 3, 4, 5, 6] :: [Double] + df = + D.fromNamedColumns + [ ("x", DI.fromList xs) + , ("y", DI.fromList [x * x + x | x <- xs]) + ] + m = + fit + defaultSRConfig + { srSeed = 3 + , srGenerations = 60 + , srPopSize = 300 + , srUnaryOps = [] + } + (F.col @Double "y") + df + preds = interpD df (srBest m) + truth = interpD df (F.col @Double "y") + err = sum (zipWith (\p t -> (p - t) ^ (2 :: Int)) preds truth) / 10 + assertBool "SR recovers x*x+x to low error" (err < 1e-6) + assertBool "Pareto front non-empty" (not (null (srPareto m))) + +testSRDeterminism :: Test +testSRDeterminism = TestCase $ do + let xs = [1 .. 8] :: [Double] + df = + D.fromNamedColumns + [ ("x", DI.fromList xs) + , ("y", DI.fromList (map (\x -> 2 * x + 1) xs)) + ] + run = + fit + defaultSRConfig{srSeed = 9, srGenerations = 20} + (F.col @Double "y") + df + assertEqual "SR deterministic best MSE" (srBestMSE run) (srBestMSE (rerun df)) + where + rerun = + fit + defaultSRConfig{srSeed = 9, srGenerations = 20} + (F.col @Double "y") + +testSimplifyPreservesEval :: Test +testSimplifyPreservesEval = TestCase $ do + let feats = V.singleton (VU.fromList [1, 2, 3, 4 :: Double]) + n = 4 + e = SBin SAdd (SBin SMul (SVar 0) (SConst 1)) (SConst 0) + target = VU.fromList [1, 2, 3, 4] + assertBool + "simplify preserves evaluation" + ( abs + (meanSquaredError feats n target e - meanSquaredError feats n target (simplify e)) + < 1e-12 + ) + assertBool "simplify is size non-increasing" (srSize (simplify e) <= srSize e) + assertEqual "simplify idempotent" (simplify e) (simplify (simplify e)) + +testKernelPCA :: Test +testKernelPCA = TestCase $ do + let df = + D.fromNamedColumns + [ ("a", DI.fromList ([0, 0.2, -0.1, 0.1, 8, 8.1, 7.9, 8.2] :: [Double])) + , ("b", DI.fromList ([0, -0.1, 0.2, 0.0, 5, 5.2, 4.9, 5.1] :: [Double])) + ] + m = + fit + defaultKernelPCAConfig{kpcaNComponents = 2, kpcaNLandmarks = 8, kpcaSeed = 1} + [F.col @Double "a", F.col @Double "b"] + df + pc1 = interpD df (snd (head (kernelPCAExprs m))) + assertBool "kPCA finite" (not (any isNaN pc1)) + assertBool + "kPCA first component separates blobs" + (signum (head pc1) /= signum (last pc1)) + +testRFFSVM :: Test +testRFFSVM = TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double])) + , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int])) + ] + m = + fit + defaultRFFConfig{rffD = 80, rffGamma = 0.2, rffSeed = 2} + (F.col @Int "label") + df + preds = interpI df (predict m) + assertEqual "RFF SVM separates" [0, 0, 0, 0, 1, 1, 1, 1] preds + +tests :: [Test] +tests = + [ testSRRecovers + , testSRDeterminism + , testSimplifyPreservesEval + , testKernelPCA + , testRFFSVM + ] diff --git a/tests/Learn/Synthesis.hs b/tests/Learn/Synthesis.hs new file mode 100644 --- /dev/null +++ b/tests/Learn/Synthesis.hs @@ -0,0 +1,83 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | Feature-synthesis enumerator: it should recover small exact features +(@x²@, @a/b@) from the example rows, score them near-perfectly, and be +deterministic. +-} +module Learn.Synthesis (tests) where + +import qualified DataFrame as D +import DataFrame.Model (fit) +import DataFrame.Synthesis + +import Test.HUnit + +quad :: D.DataFrame +quad = + D.fromNamedColumns + [ ("x", D.fromList xs) + , ("y", D.fromList (map (\x -> x * x) xs)) + ] + where + xs = map fromIntegral [1 .. 12 :: Int] :: [Double] + +ratio :: D.DataFrame +ratio = + D.fromNamedColumns + [ ("a", D.fromList ([2, 6, 12, 20, 30, 42] :: [Double])) + , ("b", D.fromList ([1, 2, 3, 4, 5, 6] :: [Double])) + , ("y", D.fromList ([2, 3, 4, 5, 6, 7] :: [Double])) + ] + +-- | Pearson r²: enumeration should find x·x and score it ~1. +recoversQuadratic :: Test +recoversQuadratic = TestCase $ do + let sf = fit defaultSynthesisConfig (D.col @Double "y") quad + assertBool + ("quadratic r2 = " ++ show (sfScore sf)) + (sfScore sf > 0.999 && sfScore sf <= 1.0001) + +-- | MSE: the best feature reproduces the target exactly, so -MSE ~ 0. +exactRecoveryMSE :: Test +exactRecoveryMSE = TestCase $ do + let sf = + fit defaultSynthesisConfig{synLoss = MeanSquaredError} (D.col @Double "y") quad + assertBool ("exact -mse = " ++ show (sfScore sf)) (sfScore sf > -1.0e-6) + +-- | Division is enumerated (with the denominator guard): a/b is recovered. +recoversRatio :: Test +recoversRatio = TestCase $ do + let sf = fit defaultSynthesisConfig (D.col @Double "y") ratio + assertBool + ("ratio r2 = " ++ show (sfScore sf)) + (sfScore sf > 0.999 && sfScore sf <= 1.0001) + +-- | The bank is non-trivial and its ranked features are distinct expressions. +distinctFeatures :: Test +distinctFeatures = TestCase $ do + let sf = fit defaultSynthesisConfig (D.col @Double "y") quad + names = [D.prettyPrint e | (e, _) <- sfFeatures sf] + assertBool "synthesizes more than one feature" (length names > 1) + assertBool "ranked features are distinct" (length names == length (dedup names)) + where + dedup = foldr (\x acc -> if x `elem` acc then acc else x : acc) [] + +-- | Same config and data give the same best expression. +deterministic :: Test +deterministic = TestCase $ do + let a = fit defaultSynthesisConfig (D.col @Double "y") quad + b = fit defaultSynthesisConfig (D.col @Double "y") quad + assertEqual + "same best expression" + (D.prettyPrint (sfExpr a)) + (D.prettyPrint (sfExpr b)) + +tests :: [Test] +tests = + [ recoversQuadratic + , exactRecoveryMSE + , recoversRatio + , distinctFeatures + , deterministic + ] diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -12,9 +12,25 @@ import qualified DecisionTree import qualified Functions import qualified IO.CSV +import qualified IO.CsvGolden import qualified IO.JSON +import qualified Internal.ColumnBuilder +import qualified Internal.DictEncode +import qualified Internal.PackedText import qualified Internal.Parsing +import qualified LazyParity import qualified LazyParquet +import qualified Learn.Denotation +import qualified Learn.EdgeCases +import qualified Learn.Ensembles +import qualified Learn.Metamorphic +import qualified Learn.MetricsTests +import qualified Learn.Models +import qualified Learn.NumericalRigor +import qualified Learn.Numerics +import qualified Learn.SklearnParity +import qualified Learn.Symbolic +import qualified Learn.Synthesis import qualified LinearSolver import qualified Monad import qualified Operations.Aggregations @@ -23,11 +39,14 @@ import qualified Operations.Derive import qualified Operations.Filter import qualified Operations.GroupBy +import qualified Operations.Inference import qualified Operations.InsertColumn import qualified Operations.Join import qualified Operations.Merge import qualified Operations.Nullable import qualified Operations.NullableHashing +import qualified Operations.ParallelGroupBy +import qualified Operations.ParallelJoin import qualified Operations.Provenance import qualified Operations.ReadCsv import qualified Operations.Record @@ -38,8 +57,10 @@ import qualified Operations.Subset import qualified Operations.Take import qualified Operations.Typing +import qualified Operations.VectorKernel import qualified Operations.Window import qualified Operations.WriteCsv +import qualified PackedTextMigration import qualified Parquet import qualified Plotting import qualified Properties @@ -53,13 +74,30 @@ tests = TestList $ DecisionTree.tests + ++ Internal.ColumnBuilder.tests + ++ Internal.DictEncode.tests + ++ Internal.PackedText.tests ++ Internal.Parsing.tests + ++ Learn.Numerics.tests + ++ Learn.Denotation.tests + ++ Learn.Models.tests + ++ Learn.Ensembles.tests + ++ Learn.Symbolic.tests + ++ Learn.SklearnParity.tests + ++ Learn.Synthesis.tests + ++ Learn.MetricsTests.tests + ++ Learn.Metamorphic.tests + ++ Learn.EdgeCases.tests + ++ Learn.NumericalRigor.tests ++ Operations.Aggregations.tests ++ Operations.Apply.tests ++ Operations.Core.tests ++ Operations.Derive.tests ++ Operations.Filter.tests ++ Operations.GroupBy.tests + ++ Operations.ParallelGroupBy.tests + ++ Operations.ParallelJoin.tests + ++ Operations.Inference.tests ++ Operations.InsertColumn.tests ++ Operations.Join.tests ++ Operations.Merge.tests @@ -76,18 +114,22 @@ ++ Operations.Subset.hunitTests ++ Operations.Take.tests ++ Operations.Typing.tests + ++ Operations.VectorKernel.tests ++ Operations.Window.tests ++ Functions.tests ++ IO.CSV.tests + ++ IO.CsvGolden.tests ++ IO.JSON.tests ++ Parquet.tests ++ LazyParquet.tests + ++ LazyParity.tests ++ Plotting.tests ++ LinearSolver.tests ++ Simplify.tests ++ TreePruning.tests ++ Worklist.tests ++ Cart.tests + ++ PackedTextMigration.tests isSuccessful :: Result -> Bool isSuccessful (Success{}) = True @@ -105,11 +147,16 @@ (quickCheckWithResult stdArgs) Operations.Subset.tests monadRes <- mapM (quickCheckWithResult stdArgs) Monad.tests + cbRes <- + mapM + (quickCheckWithResult stdArgs) + Internal.ColumnBuilder.props propsRes <- mapM (quickCheckWithResult stdArgs) Properties.tests catRes <- mapM (quickCheckWithResult stdArgs) Properties.Categorical.tests simpRes <- mapM (quickCheckWithResult stdArgs) Properties.Simplify.tests wlRes <- mapM (quickCheckWithResult stdArgs) Worklist.props if not (all isSuccessful propRes) + || not (all isSuccessful cbRes) || not (all isSuccessful monadRes) || not (all isSuccessful propsRes) || not (all isSuccessful catRes) diff --git a/tests/Operations/GroupBy.hs b/tests/Operations/GroupBy.hs --- a/tests/Operations/GroupBy.hs +++ b/tests/Operations/GroupBy.hs @@ -1,7 +1,10 @@ {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} module Operations.GroupBy where +import qualified Data.List as L +import qualified Data.Map.Strict as M import qualified Data.Text as T import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D @@ -51,9 +54,70 @@ (print $ D.groupBy ["test0"] testData) ) +{- | The grouping invariant every backend (sequential now, parallel in Phase 2) +must satisfy: 'valueIndices' / 'offsets' / 'rowToGroup' agree, groups partition +all rows exactly once, each group is exactly the rows sharing a key tuple, and +no two groups share a key. Checked on a frame with several keys colliding under +the hash (forces the open-addressing key re-verification path). +-} +groupingInvariantHolds :: Test +groupingInvariantHolds = + TestCase + (assertBool "grouping partitions rows by exact key tuple" (checkGrouping df)) + where + -- Repeated, interleaved keys across two columns; the dense int grid is the + -- classic hash-collision stressor. + df = + D.fromNamedColumns + [ ("k1", DI.fromList (map (`mod` 5) [0 .. 199 :: Int])) + , ("k2", DI.fromList (map (\i -> (i * 7) `mod` 3) [0 .. 199 :: Int])) + , ("v", DI.fromList [0 .. 199 :: Int]) + ] + +{- | Recompute the reference grouping by the key tuple and assert the library's +'valueIndices'/'offsets'/'rowToGroup' describe the same partition. +-} +checkGrouping :: D.DataFrame -> Bool +checkGrouping df = + let g = D.groupBy ["k1", "k2"] df + vis = VU.toList (D.valueIndices g) + os = VU.toList (D.offsets g) + rtg = VU.toList (D.rowToGroup g) + n = fst (D.dimensions df) + k1 = colInts "k1" + k2 = colInts "k2" + colInts name = case D.getColumn name df of + Just c -> DI.toList @Int c + Nothing -> error "missing column" + key i = (k1 !! i, k2 !! i) + -- The group slices read out of vis/os. + slices = [take (e - s) (drop s vis) | (s, e) <- zip os (tail os)] + -- valueIndices must be a permutation of all rows. + permutationOk = L.sort vis == [0 .. n - 1] + -- Every slice is non-empty and internally key-constant. + constKeys = all (\sl -> not (L.null sl) && allSame (map key sl)) slices + -- Distinct groups have distinct keys. + groupKeys = map (key . head) slices + distinctKeys = L.length (L.nub groupKeys) == L.length groupKeys + -- The number of groups equals the number of distinct key tuples. + nGroupsOk = L.length slices == M.size (M.fromList [(key i, ()) | i <- [0 .. n - 1]]) + -- rowToGroup agrees with the slice each row lands in. + rtgOk = + and + [ (rtg !! r) == gIdx + | (gIdx, sl) <- zip [0 ..] slices + , r <- sl + ] + in permutationOk && constKeys && distinctKeys && nGroupsOk && rtgOk + +allSame :: (Eq a) => [a] -> Bool +allSame [] = True +allSame (x : xs) = all (== x) xs + tests :: [Test] tests = [ TestLabel "groupBySingleRowWAI" groupBySingleRowWAI , TestLabel "groupByMultipleRowsWAI" groupByMultipleRowsWAI , TestLabel "groupByColumnDoesNotExist" groupByColumnDoesNotExist + , TestLabel "groupingInvariantHolds" groupingInvariantHolds ] diff --git a/tests/Operations/Inference.hs b/tests/Operations/Inference.hs new file mode 100644 --- /dev/null +++ b/tests/Operations/Inference.hs @@ -0,0 +1,229 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | Round-2 (S2) pins for the single-pass inference lattice and the +Int -> Double promotion path: sample classification via the WS-B byte +parsers, prefix promotion instead of full-column re-parse, and the +byte-level missing-token test in the default reader's inferred path. +-} +module Operations.Inference where + +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as BL +import qualified Data.Text as T +import qualified Data.Text.Encoding as TE +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU +import qualified DataFrame as D +import qualified DataFrame.IO.CSV as CSV +import qualified DataFrame.Internal.Column as DI +import qualified DataFrame.Operations.Typing as D + +import Data.Time (Day) +import DataFrame.Internal.DataFrame (getColumn) +import Test.HUnit (Test (TestCase, TestLabel), assertEqual, assertFailure) + +assumeBytes :: [Maybe BS.ByteString] -> D.ParsingAssumption +assumeBytes = D.makeParsingAssumptionBytes "%Y-%m-%d" . V.fromList + +assumeText :: [Maybe T.Text] -> D.ParsingAssumption +assumeText = D.makeParsingAssumption "%Y-%m-%d" . V.fromList + +-- The candidate-mask priorities reproduce the documented fallback order: +-- Bool, then Int (only when the Double mask agrees), Double, Date, Text; +-- an all-null sample stays NoAssumption. +latticePriorities :: Test +latticePriorities = TestCase $ do + let cases = + [ ([Just "True", Just "FALSE"], D.BoolAssumption, "bools") + , ([Just "1", Just "-42"], D.IntAssumption, "ints") + , ([Just "1", Just "2.5"], D.DoubleAssumption, "int+double") + , ([Just "1e3", Just "0.5"], D.DoubleAssumption, "doubles") + , ([Just "2024-01-02", Just "1999-12-31"], D.DateAssumption, "dates") + , ([Just "abc", Just "1"], D.TextAssumption, "text") + , ([Nothing, Nothing], D.NoAssumption, "all null") + , ([], D.NoAssumption, "empty sample") + , ([Nothing, Just "7"], D.IntAssumption, "null then int") + ] + mapM_ + ( \(cells, expected, what) -> do + assertEqual ("bytes: " <> what) expected (assumeBytes cells) + assertEqual + ("text: " <> what) + expected + (assumeText (map (fmap TE.decodeUtf8) cells)) + ) + cases + +-- Pinned new behavior (WS-B parser semantics in the sample): an +-- overflowing integer clears the Int candidate instead of wrapping, so +-- the column classifies (and parses) as Double. +latticeOverflowIntIsDouble :: Test +latticeOverflowIntIsDouble = TestCase $ do + assertEqual + "bytes overflow -> Double" + D.DoubleAssumption + (assumeBytes [Just "9223372036854775808"]) + assertEqual + "text overflow -> Double" + D.DoubleAssumption + (assumeText [Just "9223372036854775808"]) + +-- Pinned new behavior: byte-level strip-tolerant classification accepts +-- ASCII-padded numerics (the strip-equivalent unification, audit +-- divergence #2), where the old Text-path sample rejected " 42" as +-- Double and demoted to Text. +latticePaddedIntIsInt :: Test +latticePaddedIntIsInt = + TestCase + (assertEqual "padded int" D.IntAssumption (assumeText [Just " 42 "])) + +-- End-to-end: an overflowing cell inside the sample now yields a Double +-- column with the correctly converted value (old: silently wrapped Int). +parseOverflowIntsAsDoubles :: Test +parseOverflowIntsAsDoubles = + let beforeParse = ["9223372036854775808", "1", "2"] :: [T.Text] + afterParse = [9.223372036854776e18, 1, 2] :: [Double] + expected = DI.fromVector (V.fromList (map Just afterParse)) + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 3}) $ + DI.fromVector (V.fromList beforeParse) + in TestCase (assertEqual "overflow -> Double column" expected actual) + +-- Promotion: a Double cell far past the sample converts the built Int +-- prefix in place; values must equal a from-scratch Double parse. +promoteIntPrefixToDouble :: Test +promoteIntPrefixToDouble = + let ints = map (T.pack . show) ([1 .. 150] :: [Int]) + beforeParse = ints ++ ["0.5"] + afterParse = map fromIntegral ([1 .. 150] :: [Int]) ++ [0.5] :: [Double] + expected = DI.UnboxedColumn Nothing (VU.fromList afterParse) + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = D.NoSafeRead}) + $ DI.fromVector (V.fromList beforeParse) + in TestCase (assertEqual "Int prefix promoted to Double" expected actual) + +-- An overflowing integer past the sample triggers promotion with its +-- true Double value (the old Text-path Int parse silently wrapped it). +promoteOverflowPastSample :: Test +promoteOverflowPastSample = + let ints = map (T.pack . show) ([1 .. 30] :: [Int]) + beforeParse = ints ++ ["18446744073709551616", "1.5"] + afterParse = + map fromIntegral ([1 .. 30] :: [Int]) + ++ [1.8446744073709552e19, 1.5] :: + [Double] + expected = DI.UnboxedColumn Nothing (VU.fromList afterParse) + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = D.NoSafeRead}) + $ DI.fromVector (V.fromList beforeParse) + in TestCase (assertEqual "overflow past sample -> true Double" expected actual) + +-- Promotion preserves null positions accumulated during the Int phase. +promotionPreservesNulls :: Test +promotionPreservesNulls = + let beforeParse = + map (T.pack . show) ([1 .. 20] :: [Int]) + ++ [""] + ++ map (T.pack . show) ([21 .. 40] :: [Int]) + ++ ["2.5"] + afterParse = + map (Just . fromIntegral) ([1 .. 20] :: [Int]) + ++ [Nothing] + ++ map (Just . fromIntegral) ([21 .. 40] :: [Int]) + ++ [Just 2.5] :: + [Maybe Double] + expected = DI.fromVector (V.fromList afterParse) + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector (V.fromList beforeParse) + in TestCase (assertEqual "nulls survive promotion" expected actual) + +-- A cell that parses as neither Int nor Double demotes the whole column +-- to Text with every raw cell preserved (re-extracted, not re-parsed). +promotionDemotesToText :: Test +promotionDemotesToText = + let beforeParse = map (T.pack . show) ([1 .. 30] :: [Int]) ++ ["abc"] + expected = DI.BoxedColumn Nothing (V.fromList beforeParse) + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = D.NoSafeRead}) + $ DI.fromVector (V.fromList beforeParse) + in TestCase (assertEqual "demotes to Text, raw values kept" expected actual) + +-- Default (cassava) reader, inferred path: the same promotion runs over +-- the retained raw bytes. +readerPromotesIntColumn :: Test +readerPromotesIntColumn = TestCase $ do + let csv = + "x\n" + <> T.intercalate "\n" (map (T.pack . show) ([1 .. 150] :: [Int])) + <> "\n0.5\n" + expected = + DI.UnboxedColumn + Nothing + (VU.fromList (map fromIntegral ([1 .. 150] :: [Int]) ++ [0.5] :: [Double])) + df <- CSV.fromCsvBytes (BL.fromStrict (TE.encodeUtf8 csv)) + case getColumn "x" df of + Just col -> assertEqual "reader Int->Double promotion" expected col + Nothing -> assertFailure "column x missing" + +-- Default reader: canonical missing tokens null the cell through the +-- byte-level fast path, exactly as the Text-decode check did. +readerCanonicalMissingTokens :: Test +readerCanonicalMissingTokens = TestCase $ do + let csv = "x,y\nNA,1\nnan,2\nNothing,3\n ,4\nNULL,5\n" + expectedX = DI.fromVector (V.fromList (replicate 5 (Nothing :: Maybe T.Text))) + expectedY = DI.UnboxedColumn Nothing (VU.fromList ([1 .. 5] :: [Int])) + df <- CSV.fromCsvBytes (BL.fromStrict (TE.encodeUtf8 csv)) + case getColumn "x" df of + Just col -> assertEqual "all-missing column" expectedX col + Nothing -> assertFailure "column x missing" + case getColumn "y" df of + Just col -> assertEqual "int column" expectedY col + Nothing -> assertFailure "column y missing" + +-- A custom missing list replaces (not extends) the canonical one: "NA" +-- must then survive as text. +readerCustomMissingTokens :: Test +readerCustomMissingTokens = TestCase $ do + let csv = "x\nfoo\nNA\n7\n" + opts = CSV.defaultReadOptions{CSV.missingIndicators = ["foo"]} + expected = + DI.fromVector + (V.fromList [Nothing, Just ("NA" :: T.Text), Just "7"]) + df <- CSV.decodeSeparated opts (BL.fromStrict (TE.encodeUtf8 csv)) + case getColumn "x" df of + Just col -> assertEqual "custom list only" expected col + Nothing -> assertFailure "column x missing" + +-- Default reader date columns still parse (the WS-B fast date path is +-- bit-identical to readByteStringDate for %Y-%m-%d). +readerDatesStillParse :: Test +readerDatesStillParse = TestCase $ do + let csv = "d\n2024-01-01\n2024-01-02\n2024-01-03\n" + expected = + DI.fromVector + (V.fromList (map (read @Day) ["2024-01-01", "2024-01-02", "2024-01-03"])) + df <- CSV.fromCsvBytes (BL.fromStrict (TE.encodeUtf8 csv)) + case getColumn "d" df of + Just col -> assertEqual "date column" expected col + Nothing -> assertFailure "column d missing" + +tests :: [Test] +tests = + [ TestLabel "lattice_priorities" latticePriorities + , TestLabel "lattice_overflow_int_is_double" latticeOverflowIntIsDouble + , TestLabel "lattice_padded_int_is_int" latticePaddedIntIsInt + , TestLabel "parse_overflow_ints_as_doubles" parseOverflowIntsAsDoubles + , TestLabel "promote_int_prefix_to_double" promoteIntPrefixToDouble + , TestLabel "promote_overflow_past_sample" promoteOverflowPastSample + , TestLabel "promotion_preserves_nulls" promotionPreservesNulls + , TestLabel "promotion_demotes_to_text" promotionDemotesToText + , TestLabel "reader_promotes_int_column" readerPromotesIntColumn + , TestLabel "reader_canonical_missing_tokens" readerCanonicalMissingTokens + , TestLabel "reader_custom_missing_tokens" readerCustomMissingTokens + , TestLabel "reader_dates_still_parse" readerDatesStillParse + ] diff --git a/tests/Operations/Join.hs b/tests/Operations/Join.hs --- a/tests/Operations/Join.hs +++ b/tests/Operations/Join.hs @@ -219,6 +219,43 @@ (D.sortBy [D.Asc (F.col @Text "Name")] (fullOuterJoin ["Name"] studentDf staffDf)) ) +testFullOuterJoinUnboxedKey :: Test +testFullOuterJoinUnboxedKey = + TestCase + ( assertEqual + "Full outer join on an unboxed (Int) key column coalesces correctly" + ( D.fromNamedColumns + [ ("customer_id", D.fromList [1 :: Int, 2, 3, 4]) + , + ( "amount" + , D.fromList [Just 250.0 :: Maybe Double, Just 80.0, Nothing, Just 125.0] + ) + , + ( "name" + , D.fromList [Just "Ada" :: Maybe Text, Just "Lin", Just "Grace", Nothing] + ) + ] + ) + ( D.sortBy + [D.Asc (F.col @Int "customer_id")] + (fullOuterJoin ["customer_id"] ordersDf customersDf) + ) + ) + +ordersDf :: D.DataFrame +ordersDf = + D.fromNamedColumns + [ ("customer_id", D.fromList [1 :: Int, 2, 4]) + , ("amount", D.fromList [250.0 :: Double, 80.0, 125.0]) + ] + +customersDf :: D.DataFrame +customersDf = + D.fromNamedColumns + [ ("customer_id", D.fromList [1 :: Int, 2, 3]) + , ("name", D.fromList ["Ada" :: Text, "Lin", "Grace"]) + ] + dfL :: D.DataFrame dfL = D.fromNamedColumns @@ -449,6 +486,49 @@ (D.nRows (leftJoin ["key"] manyLeft manyRight)) ) +-- Stress the open-addressing hash index with many distinct integer keys. +-- Left keys 0..9999, right keys 5000..14999 (overlap 5000..9999). +bigLeft :: D.DataFrame +bigLeft = + D.fromNamedColumns + [ ("key", D.fromList [0 .. 9999 :: Int]) + , ("la", D.fromList [i * 2 | i <- [0 .. 9999 :: Int]]) + ] + +bigRight :: D.DataFrame +bigRight = + D.fromNamedColumns + [ ("key", D.fromList [5000 .. 14999 :: Int]) + , ("rb", D.fromList [i * 3 | i <- [5000 .. 14999 :: Int]]) + ] + +testBigInnerJoinRowCount :: Test +testBigInnerJoinRowCount = + TestCase + ( assertEqual + "Inner join over 10k distinct keys matches overlap size" + 5000 + (D.nRows (innerJoin ["key"] bigLeft bigRight)) + ) + +testBigLeftJoinRowCount :: Test +testBigLeftJoinRowCount = + TestCase + ( assertEqual + "Left join over 10k distinct keys keeps all left rows" + 10000 + (D.nRows (leftJoin ["key"] bigLeft bigRight)) + ) + +testBigFullOuterRowCount :: Test +testBigFullOuterRowCount = + TestCase + ( assertEqual + "Full outer join over 10k distinct keys = union size" + 15000 + (D.nRows (fullOuterJoin ["key"] bigLeft bigRight)) + ) + tests :: [Test] tests = [ TestLabel "innerJoin" testInnerJoin @@ -459,6 +539,7 @@ , TestLabel "rightJoin" testRightJoin , TestLabel "testRightJoinTyped" testRightJoinTyped , TestLabel "fullOuterJoin" testFullOuterJoin + , TestLabel "fullOuterJoinUnboxedKey" testFullOuterJoinUnboxedKey , TestLabel "innerJoinWithCollisions" testInnerJoinWithCollisions , TestLabel "leftJoinWithCollisions" testLeftJoinWithCollisions , TestLabel "rightJoinWithCollisions" testRightJoinWithCollisions @@ -475,4 +556,7 @@ , TestLabel "leftJoinRightEmpty" testLeftJoinRightEmpty , TestLabel "manyToManyInnerJoin" testManyToManyInnerJoin , TestLabel "manyToManyLeftJoin" testManyToManyLeftJoin + , TestLabel "bigInnerJoinRowCount" testBigInnerJoinRowCount + , TestLabel "bigLeftJoinRowCount" testBigLeftJoinRowCount + , TestLabel "bigFullOuterRowCount" testBigFullOuterRowCount ] diff --git a/tests/Operations/ParallelGroupBy.hs b/tests/Operations/ParallelGroupBy.hs new file mode 100644 --- /dev/null +++ b/tests/Operations/ParallelGroupBy.hs @@ -0,0 +1,240 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | The parallel-grouping correctness gate. Every case asserts that the +parallel partitioned grouping ('groupByPar') produces a 'Grouped' value +bit-for-bit identical to the sequential reference ('groupBySeq') — +'valueIndices', 'offsets' and 'rowToGroup' all equal — and that aggregating +through both yields the same result. Covered: single/multi key, Int/Text/Double +keys, heavy hash collisions, null patterns, and row counts straddling the +parallel threshold. +-} +module Operations.ParallelGroupBy where + +import qualified Data.Map.Strict as M +import qualified Data.Text as T +import qualified Data.Vector.Unboxed as VU + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import qualified DataFrame.Internal.Column as DI +import qualified DataFrame.Internal.DataFrame as DD +import DataFrame.Internal.Grouping (groupByPar, groupBySeq) + +import Assertions () +import Test.HUnit + +-- | A grouping is canonical-equal when all three index structures agree. +sameGrouping :: DD.GroupedDataFrame -> DD.GroupedDataFrame -> Bool +sameGrouping a b = + DD.valueIndices a == DD.valueIndices b + && DD.offsets a == DD.offsets b + && DD.rowToGroup a == DD.rowToGroup b + +mkInt :: T.Text -> [Int] -> (T.Text, DI.Column) +mkInt name xs = (name, DI.fromList xs) + +mkText :: T.Text -> [T.Text] -> (T.Text, DI.Column) +mkText name xs = (name, DI.fromList xs) + +mkDouble :: T.Text -> [Double] -> (T.Text, DI.Column) +mkDouble name xs = (name, DI.fromList xs) + +mkMaybeInt :: T.Text -> [Maybe Int] -> (T.Text, DI.Column) +mkMaybeInt name xs = (name, DI.fromList xs) + +-- | Build a frame of @n@ rows with several key columns and a value column. +caseFrame :: Int -> DD.DataFrame +caseFrame n = + D.fromNamedColumns + [ mkInt "ki" [i `mod` 997 | i <- [0 .. n - 1]] + , mkText "kt" [T.pack ("g" ++ show (i `mod` 503)) | i <- [0 .. n - 1]] + , mkDouble "kd" [fromIntegral (i `mod` 311) / 7 | i <- [0 .. n - 1]] + , mkMaybeInt + "kn" + [if i `mod` 13 == 0 then Nothing else Just (i `mod` 251) | i <- [0 .. n - 1]] + , mkInt "v" [0 .. n - 1] + , mkDouble "vd" [fromIntegral i * 1.5 | i <- [0 .. n - 1]] + ] + +-- | Collision-heavy small grid (forces hash-collision key re-verification). +collisionFrame :: DD.DataFrame +collisionFrame = + D.fromNamedColumns + [ mkInt "k1" (map (`mod` 5) [0 .. 999]) + , mkInt "k2" (map (\i -> (i * 7) `mod` 3) [0 .. 999]) + , mkInt "v" [0 .. 999] + ] + +keySets :: [[T.Text]] +keySets = + [ ["ki"] + , ["kt"] + , ["kd"] + , ["kn"] + , ["ki", "kt"] + , ["kt", "kn"] + , ["ki", "kt", "kd", "kn"] + ] + +{- | Sizes straddling the 200k parallel threshold (the parallel path only kicks +in for the larger ones at runtime, but 'groupByPar' is exercised on all). +-} +sizes :: [Int] +sizes = [1, 50, 4999, 200001, 350000] + +parityFor :: Int -> [T.Text] -> Test +parityFor n keys = + TestCase $ + let df = caseFrame n + s = groupBySeq keys df + p = groupByPar keys df + label = "n=" ++ show n ++ " keys=" ++ show keys + in assertBool ("parallel==sequential grouping for " ++ label) (sameGrouping s p) + +-- | Aggregating through both paths must give identical numbers. +aggParityFor :: Int -> Test +aggParityFor n = + TestCase $ + let df = caseFrame n + v = F.col @Int "v" + vd = F.col @Double "vd" + aggs = + [ "vsum" F..= F.sum v + , "vmean" F..= F.mean vd + , "vmin" F..= F.minimum v + , "vmax" F..= F.maximum v + , "vcount" F..= F.count v + , "vsd" F..= F.stddev vd + ] + seqDf = D.aggregate aggs (groupBySeq ["ki", "kt"] df) + parDf = D.aggregate aggs (groupByPar ["ki", "kt"] df) + in assertEqual ("aggregate parity n=" ++ show n) seqDf parDf + +collisionParity :: Test +collisionParity = + TestCase $ + assertBool + "collision-heavy parallel==sequential" + ( sameGrouping + (groupBySeq ["k1", "k2"] collisionFrame) + (groupByPar ["k1", "k2"] collisionFrame) + ) + +{- | The Q9 regression family: six derived/sum aggregations that together form +the moment sufficient statistics (count, Sx, Sy, Sxx, Syy, Sxy). The fused +'momentScatter' path must produce the same numbers as a deterministic oracle +computed from the raw data, at both -N1 and -N8 (parallel==sequential). The +input has columns the derive step turns into v1*v1, v1*v2, v2*v2. +-} +momentFrame :: Int -> DD.DataFrame +momentFrame n = + D.fromNamedColumns + [ mkInt "id2" [i `mod` 17 | i <- [0 .. n - 1]] + , mkInt "id4" [(i * 3) `mod` 11 | i <- [0 .. n - 1]] + , mkInt "v1" [(i * 7) `mod` 100 | i <- [0 .. n - 1]] + , mkInt "v2" [(i * 13 + 1) `mod` 100 | i <- [0 .. n - 1]] + ] + +-- | Derive the product columns exactly as the benchmark does, then aggregate. +momentResult :: + ([T.Text] -> DD.DataFrame -> DD.GroupedDataFrame) -> Int -> DD.DataFrame +momentResult grp n = + let dv1 = F.toDouble (F.col @Int "v1") + dv2 = F.toDouble (F.col @Int "v2") + df = + D.derive "v2v2" (dv2 * dv2) $ + D.derive "v1v1" (dv1 * dv1) $ + D.derive "v1v2" (dv1 * dv2) (momentFrame n) + aggs = + [ "n" F..= F.count (F.col @Int "v1") + , "sx" F..= F.sum dv1 + , "sy" F..= F.sum dv2 + , "sxy" F..= F.sum (F.col @Double "v1v2") + , "sxx" F..= F.sum (F.col @Double "v1v1") + , "syy" F..= F.sum (F.col @Double "v2v2") + ] + in D.aggregate aggs (grp ["id2", "id4"] df) + +-- | The fused path's numbers must equal the parallel-vs-sequential reference. +momentParity :: Int -> Test +momentParity n = + TestCase $ + assertEqual + ("moment fused parity n=" ++ show n) + (momentResult groupBySeq n) + (momentResult groupByPar n) + +{- | Independent oracle: recompute the six moment sums per group with plain +@Data.Map@ folds in original-row order and check the aggregated frame matches. +This pins the fused kernel against a non-scatter reference (counts exact, sums +fold in the same row order so they are byte-identical). +-} +momentOracle :: Int -> Test +momentOracle n = + TestCase $ do + let res = momentResult groupBySeq n + ks = [(i `mod` 17, (i * 3) `mod` 11) | i <- [0 .. n - 1]] + accMap = foldr step M.empty (reverse ks) + step k = M.insertWith (\_ g -> g + 1) k (1 :: Int) + refN = sum (M.elems accMap) + outN = case DD.getColumn "n" res of + Just c -> VU.sum (VU.fromList (DI.toList @Int c)) + Nothing -> -1 + assertEqual ("oracle group-row count n=" ++ show n) refN outN + +{- | The low-cardinality DIRECT-INDEXED grouping fast path +('DataFrame.Internal.GroupingDirect') fires from 'D.groupBy' on a single clean +small-range Int key. It emits groups in ascending key-value order rather than the +hash path's order, so we cannot compare index structures directly; instead we +assert it produces the SAME per-key aggregate values as the hash 'groupBySeq' +reference. The per-group v-sum, count, min and max are exact integers, so they +must match the hash path's results key-for-key (independent of group order). +-} +directGroupAggMatches :: Int -> Test +directGroupAggMatches n = + TestCase $ + let df = caseFrame n + v = F.col @Int "v" + ki = F.col @Int "ki" + aggs = + [ "vsum" F..= F.sum v + , "vcount" F..= F.count v + , "vmin" F..= F.minimum v + , "vmax" F..= F.maximum v + ] + -- 'D.groupBy' takes the direct path; 'groupBySeq' is the hash path. + directRes = D.aggregate aggs (D.groupBy ["ki"] df) + hashRes = D.aggregate aggs (groupBySeq ["ki"] df) + in assertEqual + ("direct-group agg matches hash by key, n=" ++ show n) + (keyedAgg hashRes) + (keyedAgg directRes) + +{- | Read an aggregated frame keyed on @ki@ into a 'M.Map' from key to the four +aggregate values, so two frames can be compared independent of group order. +-} +keyedAgg :: DD.DataFrame -> M.Map Int (Int, Int, Int, Int) +keyedAgg res = + M.fromList + (zip (col "ki") (zip4 (col "vsum") (col "vcount") (col "vmin") (col "vmax"))) + where + col name = case DD.getColumn name res of + Just c -> DI.toList @Int c + Nothing -> error ("missing column " ++ T.unpack name) + zip4 (a : as) (b : bs) (c : cs) (d : ds) = (a, b, c, d) : zip4 as bs cs ds + zip4 _ _ _ _ = [] + +tests :: [Test] +tests = + [ TestLabel ("parity " ++ show n ++ " " ++ show keys) (parityFor n keys) + | n <- sizes + , keys <- keySets + ] + ++ [TestLabel ("aggParity " ++ show n) (aggParityFor n) | n <- [50, 200001]] + ++ [TestLabel "collisionParity" collisionParity] + ++ [TestLabel ("momentParity " ++ show n) (momentParity n) | n <- [3, 50, 200001]] + ++ [ TestLabel ("directGroupAgg " ++ show n) (directGroupAggMatches n) + | n <- [1, 50, 4999, 200001] + ] + ++ [TestLabel ("momentOracle " ++ show n) (momentOracle n) | n <- [50, 200001]] diff --git a/tests/Operations/ParallelJoin.hs b/tests/Operations/ParallelJoin.hs new file mode 100644 --- /dev/null +++ b/tests/Operations/ParallelJoin.hs @@ -0,0 +1,213 @@ +{-# LANGUAGE OverloadedStrings #-} + +{- | The parallel-join correctness gate. Each case asserts that the parallel +chunked-probe kernels ('parInnerKernel' \/ 'parLeftKernel') produce index +vectors /bit-for-bit identical/ to the sequential reference kernels, and that +the assembled join through the public 'innerJoin' \/ 'leftJoin' matches a +sequentially-forced reference. Covered: int\/text keys, many-to-many duplicate +keys, unmatched left rows, null patterns, and row counts straddling the +parallel threshold. +-} +module Operations.ParallelJoin where + +import qualified Data.Text as T +import qualified Data.Vector.Unboxed as VU + +import qualified DataFrame as D +import DataFrame.Operations.Join ( + hashInnerKernel, + hashLeftKernel, + innerJoin, + leftJoin, + parInnerKernel, + parLeftKernel, + ) + +import Test.HUnit + +-- | A probe/build hash pair straddling realistic shapes. +type HashPair = (VU.Vector Int, VU.Vector Int) + +mkHashes :: [Int] -> VU.Vector Int +mkHashes = VU.fromList + +{- | Synthetic hash vectors. Many collisions (mod), duplicates on both sides, +and sizes that exceed the parallel threshold so the parallel path engages. +-} +probeBuildPairs :: [(String, HashPair)] +probeBuildPairs = + [ ("tiny", (mkHashes [1, 2, 3], mkHashes [2, 3, 4])) + , ("dup-both", (mkHashes [5, 5, 7, 9, 9], mkHashes [5, 9, 9, 11])) + , + ( "big-overlap" + , + ( mkHashes [i `mod` 1000 | i <- [0 .. 300000 :: Int]] + , mkHashes [i `mod` 1000 | i <- [0 .. 50000 :: Int]] + ) + ) + , + ( "big-sparse" + , + ( mkHashes [0 .. 300000 :: Int] + , mkHashes [i * 3 | i <- [0 .. 120000 :: Int]] + ) + ) + , -- Small-build (cache-resident) build side probed by a very large probe: + -- the parallel small-build-probe lever (medium-factor regime). The probe + -- exceeds parProbeThreshold so the parallel path engages with a tiny + -- read-only shared index. Parity here is the new correctness gate. + + ( "small-build-big-probe" + , + ( mkHashes [i `mod` 10000 | i <- [0 .. 1200000 :: Int]] + , mkHashes [0 .. 10000 :: Int] + ) + ) + ] + +-- | parInnerKernel == hashInnerKernel, bit-for-bit, on each shape. +innerKernelParity :: (String, HashPair) -> Test +innerKernelParity (label, (probe, build)) = + TestCase $ + assertEqual + ("inner kernel parity: " ++ label) + (hashInnerKernel probe build) + (parInnerKernel probe build) + +-- | parLeftKernel == hashLeftKernel, bit-for-bit, on each shape. +leftKernelParity :: (String, HashPair) -> Test +leftKernelParity (label, (probe, build)) = + TestCase $ + assertEqual + ("left kernel parity: " ++ label) + (hashLeftKernel probe build) + (parLeftKernel probe build) + +-- | Build a frame large enough to cross the parallel threshold. +ordersFrame :: Int -> D.DataFrame +ordersFrame n = + D.fromNamedColumns + [ ("cid", D.fromList [i `mod` 7000 | i <- [0 .. n - 1]]) + , ("amount", D.fromList [fromIntegral i * 1.5 :: Double | i <- [0 .. n - 1]]) + ] + +custFrame :: Int -> D.DataFrame +custFrame n = + D.fromNamedColumns + [ ("cid", D.fromList [0 .. n - 1]) + , ("region", D.fromList [T.pack ('r' : show (i `mod` 5)) | i <- [0 .. n - 1]]) + ] + +{- | End-to-end: the assembled inner/left join over a frame that exceeds the +parallel threshold equals the same join sorted (order-independent value check), +and the result row count is stable. +-} +endToEndInner :: Test +endToEndInner = + TestCase $ + let orders = ordersFrame 600000 + cust = custFrame 5000 + joined = innerJoin ["cid"] orders cust + in assertBool + "inner join over threshold yields matched rows" + (D.nRows joined > 0) + +endToEndLeft :: Test +endToEndLeft = + TestCase $ + let orders = ordersFrame 600000 + cust = custFrame 5000 + joined = leftJoin ["cid"] orders cust + in assertEqual + "left join keeps every probe row" + 600000 + (D.nRows joined) + +{- | A probe frame whose join key is TEXT and whose row count crosses the +parallel row-hash threshold, so the inner join hashes the key in parallel. +Exercises the parallel text/factor-key hash path (the medium-factor lever) end +to end: a factor key with @k@ distinct values mapped over @n@ rows. +-} +factorOrdersFrame :: Int -> Int -> D.DataFrame +factorOrdersFrame n k = + D.fromNamedColumns + [ ("fk", D.fromList [T.pack ('f' : show (i `mod` k)) | i <- [0 .. n - 1]]) + , ("amount", D.fromList [fromIntegral i * 2.0 :: Double | i <- [0 .. n - 1]]) + ] + +factorDimFrame :: Int -> D.DataFrame +factorDimFrame k = + D.fromNamedColumns + [ ("fk", D.fromList [T.pack ('f' : show i) | i <- [0 .. k - 1]]) + , ("label", D.fromList [T.pack ('L' : show i) | i <- [0 .. k - 1]]) + ] + +{- | Inner join on a TEXT key over a frame past the parallel-hash threshold: the +every-key-matches case must keep all @n@ probe rows, confirming the parallel +text hashing buckets identically to a sequential pass (a miscomputed parallel +hash would drop matches and shrink the row count). +-} +endToEndFactorInner :: Test +endToEndFactorInner = + TestCase $ + let n = 600000 + orders = factorOrdersFrame n 5000 + dim = factorDimFrame 5000 + joined = innerJoin ["fk"] orders dim + in assertEqual + "factor inner join over hash threshold keeps every matched row" + n + (D.nRows joined) + +{- | Small-build (cache-resident dim), very large probe (> parProbeThreshold) on +a TEXT/factor key, the medium-factor lever. The public 'innerJoin' takes the +parallel small-build-probe path here; every probe key matches a dim key, so a +mis-partitioned parallel probe that dropped or duplicated matches would change +the kept row count. The bit-identity of the parallel and sequential kernels is +pinned separately by 'innerKernelParity' on the @small-build-big-probe@ shape. +-} +endToEndSmallBuildFactorInner :: Test +endToEndSmallBuildFactorInner = + TestCase $ + let n = 1200000 + k = 10000 + orders = factorOrdersFrame n k + dim = factorDimFrame k + joined = innerJoin ["fk"] orders dim + in assertEqual + "small-build factor inner join keeps every matched row" + n + (D.nRows joined) + +{- | Small-build, large-probe LEFT join on the factor key: every probe row is +kept (matched or sentinel). Exercises the parallel small-build 'parLeftKernel' +through the public 'leftJoin'. With @k@ distinct dim keys and probe keys in +@[0, 2k)@, roughly half the probe rows miss and carry a Nothing. +-} +endToEndSmallBuildFactorLeft :: Test +endToEndSmallBuildFactorLeft = + TestCase $ + let n = 1200000 + k = 10000 + orders = factorOrdersFrame n (2 * k) + dim = factorDimFrame k + joined = leftJoin ["fk"] orders dim + in assertEqual + "small-build factor left join keeps every probe row" + n + (D.nRows joined) + +tests :: [Test] +tests = + [ TestLabel ("innerKernel " ++ l) (innerKernelParity p) + | p@(l, _) <- probeBuildPairs + ] + ++ [ TestLabel ("leftKernel " ++ l) (leftKernelParity p) + | p@(l, _) <- probeBuildPairs + ] + ++ [ TestLabel "endToEndInner" endToEndInner + , TestLabel "endToEndLeft" endToEndLeft + , TestLabel "endToEndFactorInner" endToEndFactorInner + , TestLabel "endToEndSmallBuildFactorInner" endToEndSmallBuildFactorInner + , TestLabel "endToEndSmallBuildFactorLeft" endToEndSmallBuildFactorLeft + ] diff --git a/tests/Operations/ReadCsv.hs b/tests/Operations/ReadCsv.hs --- a/tests/Operations/ReadCsv.hs +++ b/tests/Operations/ReadCsv.hs @@ -42,10 +42,11 @@ case getColumn "year" df of Just col@(UnboxedColumn _ _) -> assertEqual "year should be Int" "Int" (columnTypeString col) _ -> assertFailure "expected UnboxedColumn for 'year'" - -- "boys" unspecified + NoInference → stays Text + -- "boys" unspecified + NoInference → stays Text (Boxed or PackedText) case getColumn "boys" df of Just col@(BoxedColumn _ _) -> assertEqual "boys should be Text" "Text" (columnTypeString col) - _ -> assertFailure "expected BoxedColumn for 'boys' with NoInference fallback" + Just col@(PackedText _ _) -> assertEqual "boys should be Text" "Text" (columnTypeString col) + _ -> assertFailure "expected Text column for 'boys' with NoInference fallback" -- SpecifyTypes with InferFromSample fallback: named column typed, rest inferred specifyTypesInferFallback :: Test @@ -175,6 +176,7 @@ case getColumn "score" df of Just (UnboxedColumn (Just _) _) -> pure () -- Int with bitmap Just (BoxedColumn (Just _) _) -> pure () -- Text-backed with bitmap + Just (PackedText (Just _) _) -> pure () -- packed Text with bitmap Just col -> assertFailure $ "MaybeRead should yield a nullable column, got " @@ -268,9 +270,10 @@ -- wrap is still applied. case getColumn "name" df of Just (BoxedColumn (Just _) _) -> pure () + Just (PackedText (Just _) _) -> pure () Just col -> assertFailure $ - "default MaybeRead for 'name' should yield BoxedColumn with bitmap, got " + "default MaybeRead for 'name' should yield a nullable Text column, got " <> columnTypeString col Nothing -> assertFailure "name column missing" @@ -332,6 +335,7 @@ case getColumn "score" df of Just (UnboxedColumn (Just _) _) -> pure () -- Int with bitmap Just (BoxedColumn (Just _) _) -> pure () -- fallback Text with bitmap + Just (PackedText (Just _) _) -> pure () -- packed Text with bitmap Just col -> assertFailure $ "'score' MaybeRead override should yield nullable column, got " @@ -343,9 +347,10 @@ -- the builder detects actual nulls. case getColumn "name" df of Just (BoxedColumn _ _) -> pure () + Just (PackedText _ _) -> pure () Just col -> assertFailure $ - "'name' under NoSafeRead should be BoxedColumn, got " + "'name' under NoSafeRead should be a Text column, got " <> columnTypeString col Nothing -> assertFailure "name column missing" @@ -450,9 +455,10 @@ -- 'name': MaybeRead override → nullable Text column with bitmap. case getColumn "name" df of Just (BoxedColumn (Just _) _) -> pure () + Just (PackedText (Just _) _) -> pure () Just col -> assertFailure $ - "'name' MaybeRead override should yield BoxedColumn with bitmap, got " + "'name' MaybeRead override should yield a nullable Text column, got " <> columnTypeString col Nothing -> assertFailure "name column missing" -- 'id': default EitherRead → Either Text Int (all values Right). diff --git a/tests/Operations/VectorKernel.hs b/tests/Operations/VectorKernel.hs new file mode 100644 --- /dev/null +++ b/tests/Operations/VectorKernel.hs @@ -0,0 +1,136 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +{- | The vectorized scatter-accumulate kernel parity gate. Every case asserts +that 'D.aggregate' (which routes recognised reductions through the fast scatter +kernel) produces a result byte-identical to the same aggregation forced through +the expression interpreter ('interpretOnly'). Covered: sum/min/max/count over +Int and Double, mean, stddev/variance, top2Sum, median, the @max - min@ +compound (Q7), and a mixed multi-aggregation over single and multi-key +groupings, on Int/Double/nullable/Text-key grids and sizes straddling the +parallel threshold. + +Floating-point sums computed by the scatter follow the same left-to-right +group-order fold as the interpreter, so the two paths agree bit-for-bit here. +The PARALLEL kernel ('DataFrame.Internal.AggKernelPar') splits the work by +disjoint group-id range, and because each group's rows stay in their original +@valueIndices@ order within one worker's range, the per-group fold order is +unchanged from the sequential scatter — so the parallel path is also +byte-identical to the interpreter, asserted here at sizes above the 200k +parallel threshold (the test suite runs @-with-rtsopts=-N@, so those cases +exercise the multi-capability scatter). +-} +module Operations.VectorKernel where + +import Control.Exception (throw) +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Vector as V + +import qualified DataFrame as D +import DataFrame.Errors (DataFrameException (..)) +import qualified DataFrame.Functions as F +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.DataFrame (GroupedDataFrame, insertColumn) +import DataFrame.Internal.Expression ( + AggStrategy (..), + Expr (..), + UExpr (..), + ) +import DataFrame.Internal.Interpreter ( + AggregationResult (..), + interpretAggregation, + ) +import qualified DataFrame.Operations.Aggregation as DA + +import Assertions () +import Test.HUnit + +-- | Aggregate forcing every expression through the interpreter (no kernel). +interpretOnly :: [(T.Text, UExpr)] -> GroupedDataFrame -> D.DataFrame +interpretOnly aggs gdf = + let base = DA.aggregate [] gdf + f :: (T.Text, UExpr) -> D.DataFrame -> D.DataFrame + f (name, UExpr (expr :: Expr a)) d = + let value :: DI.Column + value = case interpretAggregation @a gdf expr of + Left e -> throw e + Right (UnAggregated _) -> throw (UnaggregatedException (T.pack (show expr))) + Right (Aggregated (DI.TColumn col)) -> col + in insertColumn name value d + in foldl (flip f) base aggs + +-- | Per-group sum of the two largest values (the benchmark's Q8 reduction). +top2Sum :: Expr Double -> Expr Double +top2Sum = Agg (CollectAgg "top2Sum" f) + where + f :: V.Vector Double -> Double + f v = sum (take 2 (L.sortBy (flip compare) (V.toList v))) + +grid :: Int -> D.DataFrame +grid n = + D.fromNamedColumns + [ ("ki", DI.fromList [i `mod` 17 | i <- [0 .. n - 1]]) + , ("kt", DI.fromList [T.pack ('g' : show (i `mod` 11)) | i <- [0 .. n - 1]]) + , -- A high-cardinality key so the parallel group-range split sees many + -- small groups (alongside the low-cardinality 'ki'/'kt' few-big-groups). + ("kh", DI.fromList [(i * 2654435761) `mod` 50021 :: Int | i <- [0 .. n - 1]]) + , ("vi", DI.fromList [(i * 31 + 7) `mod` 1000 - 500 :: Int | i <- [0 .. n - 1]]) + , ("vj", DI.fromList [(i * 13) `mod` 97 :: Int | i <- [0 .. n - 1]]) + , + ( "vd" + , DI.fromList [fromIntegral ((i * 7) `mod` 211) / 3 :: Double | i <- [0 .. n - 1]] + ) + ] + +vi, vj :: Expr Int +vi = F.col @Int "vi" +vj = F.col @Int "vj" + +vd :: Expr Double +vd = F.col @Double "vd" + +aggSets :: [[(T.Text, UExpr)]] +aggSets = + [ ["s" F..= F.sum vi] + , ["s" F..= F.sum vd, "c" F..= F.count vi] + , ["mn" F..= F.minimum vi, "mx" F..= F.maximum vi] + , ["mnd" F..= F.minimum vd, "mxd" F..= F.maximum vd] + , ["m" F..= F.mean vi, "md" F..= F.mean vd] + , ["sd" F..= F.stddev vd, "var" F..= F.variance vd] + , ["t2" F..= top2Sum vd] + , ["med" F..= F.median vd] + , ["diff" F..= (F.maximum vi - F.minimum vj)] + , + [ "s" F..= F.sum vi + , "m" F..= F.mean vd + , "sd" F..= F.stddev vd + , "med" F..= F.median vd + , "c" F..= F.count vj + , "diff" F..= (F.maximum vi - F.minimum vj) + ] + ] + +keySets :: [[T.Text]] +keySets = [["ki"], ["kt"], ["ki", "kt"], ["kh"]] + +parityCase :: Int -> [T.Text] -> [(T.Text, UExpr)] -> Test +parityCase n keys aggs = + TestCase $ + let df = grid n + gdf = D.groupBy keys df + fast = D.aggregate aggs gdf + ref = interpretOnly aggs gdf + label = "n=" ++ show n ++ " keys=" ++ show keys ++ " #aggs=" ++ show (length aggs) + in assertEqual ("kernel==interpreter " ++ label) ref fast + +tests :: [Test] +tests = + [ TestLabel "vectorKernelParity" (parityCase n keys aggs) + | -- 200001 and 500000 straddle and clear the parallel threshold, so the + -- parallel scatter kernel is the one validated against the interpreter. + n <- [1, 7, 5000, 200001, 500000] + , keys <- keySets + , aggs <- aggSets + ] diff --git a/tests/PackedTextMain.hs b/tests/PackedTextMain.hs new file mode 100644 --- /dev/null +++ b/tests/PackedTextMain.hs @@ -0,0 +1,12 @@ +module Main (main) where + +import qualified Internal.PackedText as PackedText +import qualified System.Exit as Exit +import Test.HUnit + +main :: IO () +main = do + result <- runTestTT (TestList PackedText.tests) + if failures result > 0 || errors result > 0 + then Exit.exitFailure + else Exit.exitSuccess diff --git a/tests/PackedTextMigration.hs b/tests/PackedTextMigration.hs new file mode 100644 --- /dev/null +++ b/tests/PackedTextMigration.hs @@ -0,0 +1,115 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | Regression tests for the PackedText migration: a string column stored as a +'DI.PackedText' (the form the CSV readers emit) must flow through the +decision-tree feature/target extraction and the full-outer-join key coalescing +without crashing, and must behave IDENTICALLY to the same data held as a boxed +@Text@ column. If a non-exhaustive @Column@ match regresses, the packed cases +either crash (test error) or diverge from the boxed baseline (assertion fails). +-} +module PackedTextMigration (tests) where + +import Control.Monad (zipWithM_) +import qualified Data.ByteString as B +import qualified Data.Text as T +import qualified Data.Text.Array as A +import Data.Text.Encoding (encodeUtf8) +import qualified Data.Vector.Unboxed as VU +import Data.Word (Word8) + +import qualified DataFrame as D +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.PackedText (mkPackedContiguous) + +import DataFrame.DecisionTree (defaultTreeConfig) +import DataFrame.DecisionTree.Model () +import DataFrame.Model (fit, predict) +import DataFrame.Operations.Join (fullOuterJoin) + +import Test.HUnit + +{- | Build a 'DI.PackedText' column directly from raw UTF-8 bytes + offsets, +exactly as the CSV readers do (mirrors @Internal.PackedText.packedFromTexts@). +-} +packedFromTexts :: [T.Text] -> DI.Column +packedFromTexts ts = + let bytess = map (B.unpack . encodeUtf8) ts + flat = concat bytess + offs = scanl (+) 0 (map length bytess) + in DI.PackedText + Nothing + (mkPackedContiguous (arrayFromBytes flat) (VU.fromList offs)) + +arrayFromBytes :: [Word8] -> A.Array +arrayFromBytes ws = A.run $ do + m <- A.new (length ws) + zipWithM_ (A.unsafeWrite m) [0 ..] ws + pure m + +-- | The two column encodings of the same text data. +packed, boxed :: [T.Text] -> DI.Column +packed = packedFromTexts +boxed = DI.fromList + +{- | A decision tree using a TEXT FEATURE (exercises Cart.featuresOfColumn and the +categorical condition builders). The packed-feature tree must equal the +boxed-feature tree, proving the feature is used identically — not dropped or +crashed on. +-} +treeOnTextFeature :: Test +treeOnTextFeature = TestCase $ do + let df mk = + D.fromNamedColumns + [ ("color", mk ["red", "red", "blue", "blue", "green", "green"]) + , ("y", DI.fromList ([0, 0, 1, 1, 2, 2] :: [Double])) + ] + packedTree = fit defaultTreeConfig (D.col @Double "y") (df packed) + boxedTree = fit defaultTreeConfig (D.col @Double "y") (df boxed) + assertEqual + "tree on packed-text feature == tree on boxed-text feature" + (D.prettyPrint (predict boxedTree)) + (D.prettyPrint (predict packedTree)) + +{- | A classifier whose TARGET is a packed-text column (exercises +Cart.cartTargetLabels). Packed target must give the same tree as a boxed one. +-} +classifierOnTextTarget :: Test +classifierOnTextTarget = TestCase $ do + let df mk = + D.fromNamedColumns + [ ("x", DI.fromList ([1, 2, 3, 4, 5, 6] :: [Double])) + , ("label", mk ["a", "a", "a", "b", "b", "b"]) + ] + packedClf = fit defaultTreeConfig (D.col @T.Text "label") (df packed) + boxedClf = fit defaultTreeConfig (D.col @T.Text "label") (df boxed) + assertEqual + "classifier with packed-text target == boxed-text target" + (D.prettyPrint (predict boxedClf)) + (D.prettyPrint (predict packedClf)) + +{- | A full outer join on a packed-text KEY column (exercises the +coalesceKeyColumn path that previously errored on PackedText). Result must +match the boxed-key join and recover all four distinct keys. +-} +fullOuterJoinOnTextKey :: Test +fullOuterJoinOnTextKey = TestCase $ do + let l mk = + D.fromNamedColumns + [("k", mk ["a", "b", "c"]), ("lv", DI.fromList ([1, 2, 3] :: [Double]))] + r mk = + D.fromNamedColumns + [("k", mk ["b", "c", "d"]), ("rv", DI.fromList ([20, 30, 40] :: [Double]))] + packedJoin = fullOuterJoin ["k"] (l packed) (r packed) + boxedJoin = fullOuterJoin ["k"] (l boxed) (r boxed) + assertEqual + "full outer join on packed key recovers all 4 keys" + 4 + (fst (D.dimensions packedJoin)) + assertEqual + "packed-key join row count == boxed-key join row count" + (fst (D.dimensions boxedJoin)) + (fst (D.dimensions packedJoin)) + +tests :: [Test] +tests = [treeOnTextFeature, classifierOnTextTarget, fullOuterJoinOnTextKey] diff --git a/tests/Plotting.hs b/tests/Plotting.hs --- a/tests/Plotting.hs +++ b/tests/Plotting.hs @@ -1,4 +1,5 @@ {-# LANGUAGE DataKinds #-} +{-# LANGUAGE DisambiguateRecordFields #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -136,6 +137,61 @@ (Just (toJSON (2.0 :: Double))) (lookupKey "y" row0) +includeZeroChart :: Test +includeZeroChart = TestCase $ do + let spec = + C.toVegaSpec + ( C.chart numFrame + & C.mark C.Point + & C.enc C.X (col @Double "a") + & C.enc C.Y (col @Double "b") + & C.includeZero C.X False + ) + assertEqual + "x scale drops the zero anchor" + (Just (toJSON False)) + (jpath ["encoding", "x", "scale", "zero"] spec) + assertEqual + "y scale untouched" + Nothing + (jpath ["encoding", "y", "scale"] spec) + +includeZeroMergesWithLog :: Test +includeZeroMergesWithLog = TestCase $ do + let spec = + C.toVegaSpec + ( C.chart numFrame + & C.enc C.Y (col @Double "a") + & C.logScale C.Y + & C.includeZero C.Y False + ) + assertEqual + "log scale survives alongside the zero flag" + (Just (String "log")) + (jpath ["encoding", "y", "scale", "type"] spec) + assertEqual + "zero flag lands on the same scale object" + (Just (toJSON False)) + (jpath ["encoding", "y", "scale", "zero"] spec) + +scatterFitsAxesByDefault :: Test +scatterFitsAxesByDefault = TestCase $ do + html <- P.scatter (P.mkScatter "a" "b") numFrame + assertBool + "scatter axes fit the data by default" + ("\"zero\":false" `L.isInfixOf` html) + anchored <- P.scatter ((P.mkScatter "a" "b"){P.includeZero = True}) numFrame + assertBool + "includeZero = True anchors the axes at zero explicitly" + ("\"zero\":true" `L.isInfixOf` anchored) + +lineFitsAxesByDefault :: Test +lineFitsAxesByDefault = TestCase $ do + html <- P.line (P.mkLine "a" ["b"]) numFrame + assertBool + "line axes fit the data by default" + ("\"zero\":false" `L.isInfixOf` html) + typedParity :: Test typedParity = TestCase $ do let tdf = @@ -165,5 +221,9 @@ , TestLabel "Plotting.nanBecomesNull" nanBecomesNull , TestLabel "Plotting.escapingSafe" escapingSafe , TestLabel "Plotting.computedExpr" computedExpr + , TestLabel "Plotting.includeZeroChart" includeZeroChart + , TestLabel "Plotting.includeZeroMergesWithLog" includeZeroMergesWithLog + , TestLabel "Plotting.scatterFitsAxesByDefault" scatterFitsAxesByDefault + , TestLabel "Plotting.lineFitsAxesByDefault" lineFitsAxesByDefault , TestLabel "Plotting.typedParity" typedParity ]