dataframe 2.1.0.3 → 2.2.0.0

raw patch · 35 files changed

+5939/−32 lines, 35 filesdep +temporarydep ~dataframe-coredep ~dataframe-csvdep ~dataframe-lazyPVP ok

version bump matches the API change (PVP)

Dependencies added: temporary

Dependency ranges changed: dataframe-core, dataframe-csv, dataframe-lazy, dataframe-learn, dataframe-operations, dataframe-parsing, dataframe-viz, directory

API changes (from Hackage documentation)

Files

• CHANGELOG.md +14/−0
• data/ml/blobs.csv +151/−0
• data/ml/golden.json +68/−0
• data/ml/iris.csv +151/−0
• data/ml/iris_binary.csv +151/−0
• data/ml/regression.csv +443/−0
• dataframe.cabal +74/−27
• ffi/DataFrame/IR.hs +4/−0
• tests/IO/CsvGolden.hs +560/−0
• tests/Internal/ColumnBuilder.hs +298/−0
• tests/Internal/DictEncode.hs +144/−0
• tests/Internal/PackedText.hs +186/−0
• tests/LazyParity.hs +152/−0
• tests/Learn/Denotation.hs +94/−0
• tests/Learn/EdgeCases.hs +481/−0
• tests/Learn/Ensembles.hs +159/−0
• tests/Learn/Metamorphic.hs +366/−0
• tests/Learn/MetricsTests.hs +121/−0
• tests/Learn/Models.hs +169/−0
• tests/Learn/NumericalRigor.hs +438/−0
• tests/Learn/Numerics.hs +99/−0
• tests/Learn/SklearnParity.hs +188/−0
• tests/Learn/Symbolic.hs +134/−0
• tests/Learn/Synthesis.hs +83/−0
• tests/Main.hs +47/−0
• tests/Operations/GroupBy.hs +64/−0
• tests/Operations/Inference.hs +229/−0
• tests/Operations/Join.hs +84/−0
• tests/Operations/ParallelGroupBy.hs +240/−0
• tests/Operations/ParallelJoin.hs +213/−0
• tests/Operations/ReadCsv.hs +11/−5
• tests/Operations/VectorKernel.hs +136/−0
• tests/PackedTextMain.hs +12/−0
• tests/PackedTextMigration.hs +115/−0
• tests/Plotting.hs +60/−0

@@ -1,5 +1,19 @@ # Revision history for dataframe +## 2.2.0.0++A large performance and ML release.++### Highlights+* **Much faster I/O and analytics.** CSV reading, group-by, joins and sorting were rebuilt on compact unboxed / `PackedText` columns, parallel open-addressing hash tables, and vectorized aggregation; the default reader drops `cassava` for a single-pass scanner and `dataframe-fastcsv` adds multicore chunking. The end-to-end join + group-by pipeline runs several times faster with much lower memory and scales with `-threaded` / `+RTS -N`; results are byte-identical (golden-tested).+* **Lazy engine on par with eager** — bounded-source queries route through the same fast paths (streaming preserved for unbounded); lazy `sortBy` now also orders non-`Text` columns correctly.+* **New ML library (`dataframe-learn`).** scikit-learn-style estimators behind a uniform `fit` / `predict`: linear / ridge / lasso / logistic regression, SVM, trees, boosting, k-means, GMM, DBSCAN, PCA and kernel PCA, symbolic regression and feature synthesis, with metrics and cross-validation. Pure and deterministic; every model also compiles to a dataframe `Expr`.+* **Typed joins are checked at compile time** — keys must exist in both schemas with matching types (previously a runtime failure or silent empty result).++### Breaking changes+* The `Column` GADT gains a `PackedText` constructor — exhaustive matches need a new arm (`materializePacked` decodes it back to boxed `Text`). CSV reads are now strict / fully forced; schema columns parse as their declared type; ragged rows pad with null instead of silently misaligning columns; overflowing integers parse as `Double`. `dataframe-learn`'s old beam-search synthesis and per-model `fit*` helpers are replaced by `fit` / `predict`.+* Coordinated major bumps (`dataframe-core`, `dataframe-learn` → `1.1.0.0`; umbrella `dataframe` → `2.2.0.0`) with inter-package lower bounds tightened so a newer package cannot resolve against an incompatible sibling. Drops `cassava` and `unordered-containers`; requires `text >= 2.1`.+ ## 2.1.0.3 ### Packaging * Fix dependency resolution for the `dataframe` meta-package and its satellites

@@ -0,0 +1,151 @@+x,y,cluster+10.338264760556216,0.847533060653685,2.0+-8.030169963007346,5.6974805690319235,0.0+10.72808765826904,0.8199785012860177,2.0+-9.06787703268451,6.456351950062689,0.0+-1.9752744232671209,4.798741021511607,1.0+-7.950069649558368,6.313857688658719,0.0+9.185263691410944,-0.02689531876281781,2.0+-1.2924664642268644,5.094345005459218,1.0+-8.386915515325072,6.748967461389298,0.0+9.47365658115339,-0.3512223275732281,2.0+-9.794694684045528,6.026719329863166,0.0+-8.474546521207202,4.194596399725956,0.0+-1.8006579280809625,4.866203894410084,1.0+-8.798016496863006,3.7677237632249145,0.0+9.608066918226854,1.550543508284739,2.0+-0.46295420029132506,4.91792529440678,1.0+9.085876965700251,0.8012523279907401,2.0+-7.2757229200427895,5.372267655941989,0.0+9.40388503768542,1.2438256022898402,2.0+-8.208254840130058,6.1863971123041805,0.0+-9.8405055750545,4.1558969180185,0.0+-1.0130643168373106,3.870050967862403,1.0+-1.750299605496018,5.8673653480977395,1.0+-7.250170505806332,6.133910399530022,0.0+-0.8427472133887682,4.013725740204026,1.0+-8.783546167149966,7.221633621411183,0.0+-7.801247618806028,5.454386723800413,0.0+10.043495267038827,1.5187267683935621,2.0+-9.428707750327403,4.758362395941994,0.0+-0.7221251049321777,4.573801938428627,1.0+-8.768888987279134,4.073279941977919,0.0+-2.3850503444929023,3.3944007047064506,1.0+9.52005242972435,1.344762816352543,2.0+-1.6485932394255796,4.637081202200104,1.0+-7.634316973046253,5.265196389986278,0.0+-2.873923777802405,6.120339997469102,1.0+-8.01938470265847,4.996106087665121,0.0+8.650080809399894,0.5164250456305327,2.0+-6.944219122655304,5.788217322112048,0.0+9.411501808610677,1.1207212024133286,2.0+8.727685388903897,0.7881397821440224,2.0+9.940157883783744,0.31129800696905785,2.0+-1.5441245562713808,3.7925654469165777,1.0+-8.668033475458893,4.43578271480966,0.0+-8.974177391694264,5.461137235845716,0.0+-8.579735970574333,5.350794772331189,0.0+-0.8507452109763282,5.35913594514609,1.0+9.799881364516688,1.4185320368386882,2.0+7.715643621051086,1.4926421993810217,2.0+9.204875175494418,0.6321707789639058,2.0+10.106053390869812,0.1776808950835841,2.0+-8.549654572872772,6.552844582677501,0.0+10.555503658306312,1.421054556659824,2.0+-0.8409150384679815,4.759055557379698,1.0+10.580079627664762,0.7854064702080787,2.0+-10.125366055017796,5.068648373068846,0.0+-0.36588996340355817,3.8233854113954955,1.0+9.426230450546782,0.5860742620560562,2.0+-1.885479568645208,4.623149715673959,1.0+9.223577182486164,1.5910144728588864,2.0+-7.2711793961873274,5.428189471188245,0.0+9.453560211741014,-0.7499716472172048,2.0+-1.984038662573283,4.364283036544825,1.0+-8.392687024351563,5.800438033246547,0.0+11.030025753690552,0.7766578881818996,2.0+-2.2915284396471147,5.001466504584993,1.0+-1.6145496750257826,4.777612786934544,1.0+10.168318216276937,0.6986657131606543,2.0+-2.2266293822669425,4.0573617784558555,1.0+-8.254266273512885,4.37715621930642,0.0+-9.437210089000642,6.767956346573112,0.0+10.590353086938768,0.2993071633588282,2.0+-2.79339785001823,4.767968183672174,1.0+-1.2140195803496816,4.126048681774031,1.0+-8.141432121082447,4.859540119431719,0.0+9.089586732772219,-0.9279621694174798,2.0+-7.1532744595716755,5.721844273167379,0.0+-8.510143036409373,4.437832885485033,0.0+-1.138567419393835,2.986258859412659,1.0+-8.034838116698193,6.139679036447002,0.0+-1.4384684377722188,5.3796868046940745,1.0+-1.2202242152011087,3.034384563178521,1.0+9.10502555609624,0.429281886212569,2.0+10.585464339946231,0.008965041738488957,2.0+-7.659707807990062,6.078774657537933,0.0+9.823590118434234,0.6308585417952228,2.0+-2.476845488661084,5.076283717062412,1.0+-8.553522718036747,6.958005682199637,0.0+-8.069594713163454,5.389290712648975,0.0+-1.4505216161787537,3.810535489405869,1.0+0.14236866854605823,4.102802122310017,1.0+-8.713989201960986,4.654390170035729,0.0+-1.462202464398755,4.709150238434744,1.0+9.139446464353718,0.9973645548743786,2.0+-2.0942032865826947,3.003362783235963,1.0+9.384194636323251,0.5881152884036329,2.0+-0.09043844586306131,4.594778743582171,1.0+-8.630656062367228,5.125759989711783,0.0+-8.258304908114546,5.178692149281611,0.0+-1.6564983727218412,3.57173364605436,1.0+-8.747527831262346,5.220834523095475,0.0+8.289191474819738,0.9762956762429598,2.0+-0.2667896158894523,5.2537491444891975,1.0+-1.4567017014328651,3.240421719648844,1.0+-2.578532846092601,5.281315504987536,1.0+-7.060905510177389,5.334844843271841,0.0+9.544505126636004,1.2726118372517006,2.0+-1.789088183709013,4.9356797992475885,1.0+-9.012750362421714,5.553681884762233,0.0+-2.0885293775780265,3.819536691473433,1.0+-8.160597657718792,4.443878519201542,0.0+9.953505807271908,0.7455103282555472,2.0+-8.628118580800681,4.175965562435486,0.0+-0.4807339560151918,4.5869562471000975,1.0+9.458262920723499,0.3546075672900924,2.0+-1.3869004247455148,5.027138844442485,1.0+-1.3046003840083444,5.232595722796476,1.0+10.903405766013115,0.2879089162540264,2.0+-8.167336731114846,7.396451887048452,0.0+-1.1612914315034135,3.4000871166542685,1.0+-1.7253529304545083,4.179477773883045,1.0+-0.9912710391638034,5.444758663793727,1.0+-2.3277031463654856,5.054227064960736,1.0+8.147437705805391,1.071564094656846,2.0+9.877041844526358,1.7070321683770884,2.0+-0.3870567318504991,4.922009547520749,1.0+-8.713603202383117,6.635883998983632,0.0+9.549394662765959,1.8843175122058762,2.0+-6.665876623219847,6.293607308205353,0.0+-9.6218049853493,5.999675143347125,0.0+-8.780572710866066,4.886490355985694,0.0+10.03429276137997,0.5300549650285779,2.0+9.157196652555177,1.2812079409319164,2.0+8.1447728798683,0.5915572830522724,2.0+9.146956239818431,1.017246090363041,2.0+-9.165426205789453,5.897872124592354,0.0+9.959392438724798,0.19080965893214255,2.0+8.486697738274069,1.4784940535418505,2.0+-0.3582643293862715,5.840807754114097,1.0+-2.2186345261641804,4.615624117910637,1.0+10.47604297470776,0.4921540622294554,2.0+-2.7604144732798868,4.9784338438022795,1.0+9.937212447233478,-0.1112822931713835,2.0+-8.483627961806011,5.53784507561264,0.0+9.270455589800079,1.4007210675022714,2.0+0.25622916659958644,3.2714846427335473,1.0+10.148892613250826,-0.16841584321546238,2.0+9.702422433216821,2.162194444522933,2.0+-9.104972635421447,5.6000283031270515,0.0+-0.46045498366900267,4.696212695686656,1.0

@@ -0,0 +1,68 @@+{+  "gbm": {+    "accuracy": 1.0+  },+  "kmeans": {+    "inertia": 173.75480764755966+  },+  "linear_svc": {+    "accuracy": 1.0+  },+  "logistic_binary": {+    "accuracy": 1.0+  },+  "logistic_iris": {+    "accuracy": 0.9733333333333334+  },+  "ols": {+    "coef": [+      -10.009866299810287,+      -239.81564367242294,+      519.8459200544604,+      324.38464550232385,+      -792.1756385522323,+      476.7390210052585,+      101.04326793803448,+      177.06323767134674,+      751.2736995571044,+      67.62669218370488+    ],+    "intercept": 152.13348416289597+  },+  "pca": {+    "components_abs": [+      [+        0.3613865917853659,+        0.08452251406457255,+        0.8566706059498347,+        0.3582891971515517+      ],+      [+        0.6565887712868534,+        0.7301614347850159,+        0.17337266279585964,+        0.0754810199174582+      ]+    ],+    "evr": [+      0.9246187232017291,+      0.05306648311706544+    ]+  },+  "ridge": {+    "alpha": 1.0,+    "coef": [+      29.466111893477002,+      -83.15427636187523,+      306.3526801506861,+      201.62773437326962,+      5.909614367497247,+      -29.515495079689597,+      -152.04028006186414,+      117.31173160030173,+      262.9442900143125,+      111.87895643952352+    ],+    "intercept": 152.133484162896+  }+}

@@ -0,0 +1,151 @@+sepal_length,sepal_width,petal_length,petal_width,species+5.1,3.5,1.4,0.2,0.0+4.9,3.0,1.4,0.2,0.0+4.7,3.2,1.3,0.2,0.0+4.6,3.1,1.5,0.2,0.0+5.0,3.6,1.4,0.2,0.0+5.4,3.9,1.7,0.4,0.0+4.6,3.4,1.4,0.3,0.0+5.0,3.4,1.5,0.2,0.0+4.4,2.9,1.4,0.2,0.0+4.9,3.1,1.5,0.1,0.0+5.4,3.7,1.5,0.2,0.0+4.8,3.4,1.6,0.2,0.0+4.8,3.0,1.4,0.1,0.0+4.3,3.0,1.1,0.1,0.0+5.8,4.0,1.2,0.2,0.0+5.7,4.4,1.5,0.4,0.0+5.4,3.9,1.3,0.4,0.0+5.1,3.5,1.4,0.3,0.0+5.7,3.8,1.7,0.3,0.0+5.1,3.8,1.5,0.3,0.0+5.4,3.4,1.7,0.2,0.0+5.1,3.7,1.5,0.4,0.0+4.6,3.6,1.0,0.2,0.0+5.1,3.3,1.7,0.5,0.0+4.8,3.4,1.9,0.2,0.0+5.0,3.0,1.6,0.2,0.0+5.0,3.4,1.6,0.4,0.0+5.2,3.5,1.5,0.2,0.0+5.2,3.4,1.4,0.2,0.0+4.7,3.2,1.6,0.2,0.0+4.8,3.1,1.6,0.2,0.0+5.4,3.4,1.5,0.4,0.0+5.2,4.1,1.5,0.1,0.0+5.5,4.2,1.4,0.2,0.0+4.9,3.1,1.5,0.2,0.0+5.0,3.2,1.2,0.2,0.0+5.5,3.5,1.3,0.2,0.0+4.9,3.6,1.4,0.1,0.0+4.4,3.0,1.3,0.2,0.0+5.1,3.4,1.5,0.2,0.0+5.0,3.5,1.3,0.3,0.0+4.5,2.3,1.3,0.3,0.0+4.4,3.2,1.3,0.2,0.0+5.0,3.5,1.6,0.6,0.0+5.1,3.8,1.9,0.4,0.0+4.8,3.0,1.4,0.3,0.0+5.1,3.8,1.6,0.2,0.0+4.6,3.2,1.4,0.2,0.0+5.3,3.7,1.5,0.2,0.0+5.0,3.3,1.4,0.2,0.0+7.0,3.2,4.7,1.4,1.0+6.4,3.2,4.5,1.5,1.0+6.9,3.1,4.9,1.5,1.0+5.5,2.3,4.0,1.3,1.0+6.5,2.8,4.6,1.5,1.0+5.7,2.8,4.5,1.3,1.0+6.3,3.3,4.7,1.6,1.0+4.9,2.4,3.3,1.0,1.0+6.6,2.9,4.6,1.3,1.0+5.2,2.7,3.9,1.4,1.0+5.0,2.0,3.5,1.0,1.0+5.9,3.0,4.2,1.5,1.0+6.0,2.2,4.0,1.0,1.0+6.1,2.9,4.7,1.4,1.0+5.6,2.9,3.6,1.3,1.0+6.7,3.1,4.4,1.4,1.0+5.6,3.0,4.5,1.5,1.0+5.8,2.7,4.1,1.0,1.0+6.2,2.2,4.5,1.5,1.0+5.6,2.5,3.9,1.1,1.0+5.9,3.2,4.8,1.8,1.0+6.1,2.8,4.0,1.3,1.0+6.3,2.5,4.9,1.5,1.0+6.1,2.8,4.7,1.2,1.0+6.4,2.9,4.3,1.3,1.0+6.6,3.0,4.4,1.4,1.0+6.8,2.8,4.8,1.4,1.0+6.7,3.0,5.0,1.7,1.0+6.0,2.9,4.5,1.5,1.0+5.7,2.6,3.5,1.0,1.0+5.5,2.4,3.8,1.1,1.0+5.5,2.4,3.7,1.0,1.0+5.8,2.7,3.9,1.2,1.0+6.0,2.7,5.1,1.6,1.0+5.4,3.0,4.5,1.5,1.0+6.0,3.4,4.5,1.6,1.0+6.7,3.1,4.7,1.5,1.0+6.3,2.3,4.4,1.3,1.0+5.6,3.0,4.1,1.3,1.0+5.5,2.5,4.0,1.3,1.0+5.5,2.6,4.4,1.2,1.0+6.1,3.0,4.6,1.4,1.0+5.8,2.6,4.0,1.2,1.0+5.0,2.3,3.3,1.0,1.0+5.6,2.7,4.2,1.3,1.0+5.7,3.0,4.2,1.2,1.0+5.7,2.9,4.2,1.3,1.0+6.2,2.9,4.3,1.3,1.0+5.1,2.5,3.0,1.1,1.0+5.7,2.8,4.1,1.3,1.0+6.3,3.3,6.0,2.5,2.0+5.8,2.7,5.1,1.9,2.0+7.1,3.0,5.9,2.1,2.0+6.3,2.9,5.6,1.8,2.0+6.5,3.0,5.8,2.2,2.0+7.6,3.0,6.6,2.1,2.0+4.9,2.5,4.5,1.7,2.0+7.3,2.9,6.3,1.8,2.0+6.7,2.5,5.8,1.8,2.0+7.2,3.6,6.1,2.5,2.0+6.5,3.2,5.1,2.0,2.0+6.4,2.7,5.3,1.9,2.0+6.8,3.0,5.5,2.1,2.0+5.7,2.5,5.0,2.0,2.0+5.8,2.8,5.1,2.4,2.0+6.4,3.2,5.3,2.3,2.0+6.5,3.0,5.5,1.8,2.0+7.7,3.8,6.7,2.2,2.0+7.7,2.6,6.9,2.3,2.0+6.0,2.2,5.0,1.5,2.0+6.9,3.2,5.7,2.3,2.0+5.6,2.8,4.9,2.0,2.0+7.7,2.8,6.7,2.0,2.0+6.3,2.7,4.9,1.8,2.0+6.7,3.3,5.7,2.1,2.0+7.2,3.2,6.0,1.8,2.0+6.2,2.8,4.8,1.8,2.0+6.1,3.0,4.9,1.8,2.0+6.4,2.8,5.6,2.1,2.0+7.2,3.0,5.8,1.6,2.0+7.4,2.8,6.1,1.9,2.0+7.9,3.8,6.4,2.0,2.0+6.4,2.8,5.6,2.2,2.0+6.3,2.8,5.1,1.5,2.0+6.1,2.6,5.6,1.4,2.0+7.7,3.0,6.1,2.3,2.0+6.3,3.4,5.6,2.4,2.0+6.4,3.1,5.5,1.8,2.0+6.0,3.0,4.8,1.8,2.0+6.9,3.1,5.4,2.1,2.0+6.7,3.1,5.6,2.4,2.0+6.9,3.1,5.1,2.3,2.0+5.8,2.7,5.1,1.9,2.0+6.8,3.2,5.9,2.3,2.0+6.7,3.3,5.7,2.5,2.0+6.7,3.0,5.2,2.3,2.0+6.3,2.5,5.0,1.9,2.0+6.5,3.0,5.2,2.0,2.0+6.2,3.4,5.4,2.3,2.0+5.9,3.0,5.1,1.8,2.0

@@ -0,0 +1,151 @@+sepal_length,sepal_width,petal_length,petal_width,label+5.1,3.5,1.4,0.2,0.0+4.9,3.0,1.4,0.2,0.0+4.7,3.2,1.3,0.2,0.0+4.6,3.1,1.5,0.2,0.0+5.0,3.6,1.4,0.2,0.0+5.4,3.9,1.7,0.4,0.0+4.6,3.4,1.4,0.3,0.0+5.0,3.4,1.5,0.2,0.0+4.4,2.9,1.4,0.2,0.0+4.9,3.1,1.5,0.1,0.0+5.4,3.7,1.5,0.2,0.0+4.8,3.4,1.6,0.2,0.0+4.8,3.0,1.4,0.1,0.0+4.3,3.0,1.1,0.1,0.0+5.8,4.0,1.2,0.2,0.0+5.7,4.4,1.5,0.4,0.0+5.4,3.9,1.3,0.4,0.0+5.1,3.5,1.4,0.3,0.0+5.7,3.8,1.7,0.3,0.0+5.1,3.8,1.5,0.3,0.0+5.4,3.4,1.7,0.2,0.0+5.1,3.7,1.5,0.4,0.0+4.6,3.6,1.0,0.2,0.0+5.1,3.3,1.7,0.5,0.0+4.8,3.4,1.9,0.2,0.0+5.0,3.0,1.6,0.2,0.0+5.0,3.4,1.6,0.4,0.0+5.2,3.5,1.5,0.2,0.0+5.2,3.4,1.4,0.2,0.0+4.7,3.2,1.6,0.2,0.0+4.8,3.1,1.6,0.2,0.0+5.4,3.4,1.5,0.4,0.0+5.2,4.1,1.5,0.1,0.0+5.5,4.2,1.4,0.2,0.0+4.9,3.1,1.5,0.2,0.0+5.0,3.2,1.2,0.2,0.0+5.5,3.5,1.3,0.2,0.0+4.9,3.6,1.4,0.1,0.0+4.4,3.0,1.3,0.2,0.0+5.1,3.4,1.5,0.2,0.0+5.0,3.5,1.3,0.3,0.0+4.5,2.3,1.3,0.3,0.0+4.4,3.2,1.3,0.2,0.0+5.0,3.5,1.6,0.6,0.0+5.1,3.8,1.9,0.4,0.0+4.8,3.0,1.4,0.3,0.0+5.1,3.8,1.6,0.2,0.0+4.6,3.2,1.4,0.2,0.0+5.3,3.7,1.5,0.2,0.0+5.0,3.3,1.4,0.2,0.0+7.0,3.2,4.7,1.4,1.0+6.4,3.2,4.5,1.5,1.0+6.9,3.1,4.9,1.5,1.0+5.5,2.3,4.0,1.3,1.0+6.5,2.8,4.6,1.5,1.0+5.7,2.8,4.5,1.3,1.0+6.3,3.3,4.7,1.6,1.0+4.9,2.4,3.3,1.0,1.0+6.6,2.9,4.6,1.3,1.0+5.2,2.7,3.9,1.4,1.0+5.0,2.0,3.5,1.0,1.0+5.9,3.0,4.2,1.5,1.0+6.0,2.2,4.0,1.0,1.0+6.1,2.9,4.7,1.4,1.0+5.6,2.9,3.6,1.3,1.0+6.7,3.1,4.4,1.4,1.0+5.6,3.0,4.5,1.5,1.0+5.8,2.7,4.1,1.0,1.0+6.2,2.2,4.5,1.5,1.0+5.6,2.5,3.9,1.1,1.0+5.9,3.2,4.8,1.8,1.0+6.1,2.8,4.0,1.3,1.0+6.3,2.5,4.9,1.5,1.0+6.1,2.8,4.7,1.2,1.0+6.4,2.9,4.3,1.3,1.0+6.6,3.0,4.4,1.4,1.0+6.8,2.8,4.8,1.4,1.0+6.7,3.0,5.0,1.7,1.0+6.0,2.9,4.5,1.5,1.0+5.7,2.6,3.5,1.0,1.0+5.5,2.4,3.8,1.1,1.0+5.5,2.4,3.7,1.0,1.0+5.8,2.7,3.9,1.2,1.0+6.0,2.7,5.1,1.6,1.0+5.4,3.0,4.5,1.5,1.0+6.0,3.4,4.5,1.6,1.0+6.7,3.1,4.7,1.5,1.0+6.3,2.3,4.4,1.3,1.0+5.6,3.0,4.1,1.3,1.0+5.5,2.5,4.0,1.3,1.0+5.5,2.6,4.4,1.2,1.0+6.1,3.0,4.6,1.4,1.0+5.8,2.6,4.0,1.2,1.0+5.0,2.3,3.3,1.0,1.0+5.6,2.7,4.2,1.3,1.0+5.7,3.0,4.2,1.2,1.0+5.7,2.9,4.2,1.3,1.0+6.2,2.9,4.3,1.3,1.0+5.1,2.5,3.0,1.1,1.0+5.7,2.8,4.1,1.3,1.0+6.3,3.3,6.0,2.5,1.0+5.8,2.7,5.1,1.9,1.0+7.1,3.0,5.9,2.1,1.0+6.3,2.9,5.6,1.8,1.0+6.5,3.0,5.8,2.2,1.0+7.6,3.0,6.6,2.1,1.0+4.9,2.5,4.5,1.7,1.0+7.3,2.9,6.3,1.8,1.0+6.7,2.5,5.8,1.8,1.0+7.2,3.6,6.1,2.5,1.0+6.5,3.2,5.1,2.0,1.0+6.4,2.7,5.3,1.9,1.0+6.8,3.0,5.5,2.1,1.0+5.7,2.5,5.0,2.0,1.0+5.8,2.8,5.1,2.4,1.0+6.4,3.2,5.3,2.3,1.0+6.5,3.0,5.5,1.8,1.0+7.7,3.8,6.7,2.2,1.0+7.7,2.6,6.9,2.3,1.0+6.0,2.2,5.0,1.5,1.0+6.9,3.2,5.7,2.3,1.0+5.6,2.8,4.9,2.0,1.0+7.7,2.8,6.7,2.0,1.0+6.3,2.7,4.9,1.8,1.0+6.7,3.3,5.7,2.1,1.0+7.2,3.2,6.0,1.8,1.0+6.2,2.8,4.8,1.8,1.0+6.1,3.0,4.9,1.8,1.0+6.4,2.8,5.6,2.1,1.0+7.2,3.0,5.8,1.6,1.0+7.4,2.8,6.1,1.9,1.0+7.9,3.8,6.4,2.0,1.0+6.4,2.8,5.6,2.2,1.0+6.3,2.8,5.1,1.5,1.0+6.1,2.6,5.6,1.4,1.0+7.7,3.0,6.1,2.3,1.0+6.3,3.4,5.6,2.4,1.0+6.4,3.1,5.5,1.8,1.0+6.0,3.0,4.8,1.8,1.0+6.9,3.1,5.4,2.1,1.0+6.7,3.1,5.6,2.4,1.0+6.9,3.1,5.1,2.3,1.0+5.8,2.7,5.1,1.9,1.0+6.8,3.2,5.9,2.3,1.0+6.7,3.3,5.7,2.5,1.0+6.7,3.0,5.2,2.3,1.0+6.3,2.5,5.0,1.9,1.0+6.5,3.0,5.2,2.0,1.0+6.2,3.4,5.4,2.3,1.0+5.9,3.0,5.1,1.8,1.0

@@ -0,0 +1,443 @@+f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,target+0.038075906433423026,0.05068011873981862,0.061696206518683294,0.0218723855140367,-0.04422349842444599,-0.03482076283769895,-0.04340084565202491,-0.002592261998183278,0.019907486170462722,-0.01764612515980379,151.0+-0.0018820165277906047,-0.044641636506989144,-0.051474061238800654,-0.02632752814785296,-0.008448724111216851,-0.019163339748222204,0.07441156407875721,-0.03949338287409329,-0.0683315470939731,-0.092204049626824,75.0+0.08529890629667548,0.05068011873981862,0.04445121333659049,-0.00567042229275739,-0.04559945128264711,-0.03419446591411989,-0.03235593223976409,-0.002592261998183278,0.002861309289833047,-0.025930338989472702,141.0+-0.0890629393522567,-0.044641636506989144,-0.011595014505211082,-0.03665608107540074,0.01219056876179996,0.02499059336410222,-0.036037570043851025,0.03430885887772673,0.022687744966501246,-0.009361911330134878,206.0+0.005383060374248237,-0.044641636506989144,-0.03638469220446948,0.0218723855140367,0.003934851612593237,0.015596139510416171,0.008142083605192267,-0.002592261998183278,-0.03198763948805312,-0.04664087356364498,135.0+-0.09269547780327612,-0.044641636506989144,-0.040695940499992665,-0.019441826196154435,-0.06899064987206617,-0.07928784441181291,0.04127682384197474,-0.0763945037500033,-0.041176166918895155,-0.09634615654165846,97.0+-0.045472477940023646,0.05068011873981862,-0.047162812943277475,-0.015998975220305175,-0.04009563984984263,-0.02480001206043385,0.0007788079970183853,-0.03949338287409329,-0.06291687914365544,-0.03835665973397607,138.0+0.06350367559055897,0.05068011873981862,-0.0018947058402839008,0.0666294482000771,0.09061988167926385,0.10891438112369757,0.022868634821540033,0.01770335448356722,-0.0358161925842373,0.0030644094143684884,63.0+0.04170844488444244,0.05068011873981862,0.061696206518683294,-0.04009893205125,-0.013952535544021335,0.0062016856567301245,-0.028674294435677143,-0.002592261998183278,-0.014959693812643405,0.0113486232440374,110.0+-0.07090024709715959,-0.044641636506989144,0.039062152967186486,-0.03321323009955148,-0.012576582685820214,-0.034507614375909414,-0.024992656631590206,-0.002592261998183278,0.06773705306493534,-0.013504018244969336,310.0+-0.09632801625429555,-0.044641636506989144,-0.08380842345522464,0.008100981610639655,-0.10338947132709418,-0.09056118903623617,-0.01394774321932938,-0.0763945037500033,-0.06291687914365544,-0.03421455281914162,101.0+0.027178291080364757,0.05068011873981862,0.0175059114895705,-0.03321323009955148,-0.007072771253015731,0.045971540304001066,-0.06549067247654655,0.07120997975363674,-0.09643494994048712,-0.05906719430814835,69.0+0.016280675727306498,-0.044641636506989144,-0.028840007687303888,-0.009113273268606652,-0.004320865536613489,-0.009768885894536158,0.04495846164606168,-0.03949338287409329,-0.030747917533098208,-0.042498766648810526,179.0+0.005383060374248237,0.05068011873981862,-0.0018947058402839008,0.008100981610639655,-0.004320865536613489,-0.015718706668537318,-0.002902829807068556,-0.002592261998183278,0.038393928263466416,-0.013504018244969336,185.0+0.04534098333546186,-0.044641636506989144,-0.02560657146566148,-0.012556124244455912,0.017694380194604446,-6.128357906057276e-05,0.0817748396869311,-0.03949338287409329,-0.03198763948805312,-0.07563562196748617,118.0+-0.052737554842062495,0.05068011873981862,-0.018061886948495892,0.08040085210347414,0.08924392882106273,0.10766178727653941,-0.03971920784793797,0.10811110062954676,0.036060333995316066,-0.042498766648810526,171.0+-0.005514554978810025,-0.044641636506989144,0.0422955891888289,0.049415193320830796,0.024574144485610048,-0.02386056667506523,0.07441156407875721,-0.03949338287409329,0.05227699103843915,0.027917050903375224,166.0+0.0707687524925978,0.05068011873981862,0.012116851120166501,0.056300895272529315,0.0342058144930179,0.04941617338368593,-0.03971920784793797,0.03430885887772673,0.02736404910541198,-0.0010776975004659671,144.0+-0.03820740103798481,-0.044641636506989144,-0.010517202431330305,-0.03665608107540074,-0.037343734133440394,-0.019476488210011744,-0.028674294435677143,-0.002592261998183278,-0.018113692315690322,-0.01764612515980379,97.0+-0.027309785684926546,-0.044641636506989144,-0.018061886948495892,-0.04009893205125,-0.0029449126784123676,-0.011334628203483833,0.0375951860378878,-0.03949338287409329,-0.00894339609006817,-0.05492508739331389,168.0+-0.04910501639104307,-0.044641636506989144,-0.05686312160820465,-0.04354178302709927,-0.04559945128264711,-0.043275771306016404,0.0007788079970183853,-0.03949338287409329,-0.011896851335695978,0.015490730158871856,68.0+-0.08543040090123728,0.05068011873981862,-0.022373135244019075,0.0012152796589411327,-0.037343734133440394,-0.02636575436938152,0.01550535921336615,-0.03949338287409329,-0.07213275338232743,-0.01764612515980379,49.0+-0.08543040090123728,-0.044641636506989144,-0.004050329988045492,-0.009113273268606652,-0.0029449126784123676,0.0077674279656778,0.022868634821540033,-0.03949338287409329,-0.061175799045152635,-0.013504018244969336,68.0+0.04534098333546186,0.05068011873981862,0.06061839444480248,0.031064797619554236,0.028702003060213414,-0.04734670130928034,-0.05444575906428573,0.07120997975363674,0.13359728192191356,0.13561183068907107,245.0+-0.06363517019512076,-0.044641636506989144,0.03582871674554409,-0.0228846771720037,-0.030463969842434782,-0.018850191286432668,-0.006584467611155497,-0.002592261998183278,-0.025953110560258,-0.05492508739331389,184.0+-0.06726770864614018,0.05068011873981862,-0.012672826579091896,-0.04009893205125,-0.015328488402222454,0.00463594334778245,-0.05812739686837268,0.03430885887772673,0.019196469166885697,-0.03421455281914162,202.0+-0.1072256316073538,-0.044641636506989144,-0.07734155101193986,-0.02632752814785296,-0.08962994274508297,-0.09619786134844781,0.026550272625626974,-0.0763945037500033,-0.042570854118219384,-0.005219804415300423,137.0+-0.02367724723390713,-0.044641636506989144,0.05954058237092167,-0.04009893205125,-0.04284754556624487,-0.04358891976780594,0.01182372140927921,-0.03949338287409329,-0.015998872510179042,0.040343371647878594,85.0+0.0526060602375007,-0.044641636506989144,-0.0212953231701383,-0.07452744180974262,-0.04009563984984263,-0.03763909899380476,-0.006584467611155497,-0.03949338287409329,-0.0006117353045626216,-0.05492508739331389,131.0+0.06713621404157838,0.05068011873981862,-0.006205954135807083,0.06318659722422783,-0.04284754556624487,-0.09588471288665826,0.05232173725423556,-0.0763945037500033,0.059423623484649676,0.05276969239238195,283.0+-0.06000263174410134,-0.044641636506989144,0.04445121333659049,-0.019441826196154435,-0.009824676969417972,-0.007576846662009428,0.022868634821540033,-0.03949338287409329,-0.02712902329694316,-0.009361911330134878,129.0+-0.02367724723390713,-0.044641636506989144,-0.06548561819925106,-0.08141314376144114,-0.038719686991641515,-0.05360967054507104,0.059685012862409445,-0.0763945037500033,-0.0371288393600719,-0.042498766648810526,59.0+0.0344433679824036,0.05068011873981862,0.12528711887765046,0.028758087465735226,-0.05385516843185383,-0.012900370512431508,-0.10230705051741597,0.10811110062954676,0.00027247814860377354,0.027917050903375224,341.0+0.03081082953138418,-0.044641636506989144,-0.050396249164919873,-0.002227571316908129,-0.04422349842444599,-0.0899348921126571,0.1185912177278005,-0.0763945037500033,-0.018113692315690322,0.0030644094143684884,87.0+0.016280675727306498,-0.044641636506989144,-0.06332999405148947,-0.057313186930496314,-0.0579830270064572,-0.048912443618228024,0.008142083605192267,-0.03949338287409329,-0.05947118135708968,-0.06735140813781726,65.0+0.04897352178648128,0.05068011873981862,-0.03099563183506548,-0.04929134415676754,0.04934129593323023,-0.004132213582324539,0.13331776894414826,-0.05351580880693909,0.021311288972396977,0.019632837073706312,102.0+0.012648137276287077,-0.044641636506989144,0.022894971858974496,0.052858044296680055,0.0080627101871966,-0.02855779360190825,0.0375951860378878,-0.03949338287409329,0.05471997253790904,-0.025930338989472702,265.0+-0.009147093429829445,-0.044641636506989144,0.011039039046285686,-0.057313186930496314,-0.0249601584096303,-0.04296262284422686,0.030231910429713918,-0.03949338287409329,0.017036071348324546,-0.005219804415300423,276.0+-0.0018820165277906047,0.05068011873981862,0.07139651518361048,0.09761510698272045,0.08786797596286161,0.07540749571221732,-0.02131101882750326,0.07120997975363674,0.07142887212197009,0.02377494398854077,252.0+-0.0018820165277906047,0.05068011873981862,0.014272475267928093,-0.07452744180974262,0.0025588987543921156,0.0062016856567301245,-0.01394774321932938,-0.002592261998183278,0.019196469166885697,0.0030644094143684884,90.0+0.005383060374248237,0.05068011873981862,-0.008361578283568675,0.0218723855140367,0.05484510736603471,0.07321545647969056,-0.024992656631590206,0.03430885887772673,0.012551194864223063,0.09419076154072652,100.0+-0.09996055470531495,-0.044641636506989144,-0.06764124234701265,-0.10895595156823522,-0.07449446130487065,-0.07271172671423268,0.01550535921336615,-0.03949338287409329,-0.049872451808799324,-0.009361911330134878,55.0+-0.06000263174410134,0.05068011873981862,-0.010517202431330305,-0.014862834398274925,-0.04972730985725048,-0.02354741821327569,-0.05812739686837268,0.01585829843977173,-0.009918765569334137,-0.03421455281914162,61.0+0.01991321417832592,-0.044641636506989144,-0.023450947317899894,-0.07108459083389335,0.020446285911006685,-0.010082034356325698,0.1185912177278005,-0.0763945037500033,-0.042570854118219384,0.07348022696655424,92.0+0.04534098333546186,0.05068011873981862,0.0681630789619681,0.008100981610639655,-0.016704441260423575,0.00463594334778245,-0.07653558588880739,0.07120997975363674,0.03243232415655107,-0.01764612515980379,259.0+0.027178291080364757,0.05068011873981862,-0.035306880130588664,0.03220093844158448,-0.011200629827619093,0.0015044587298871017,-0.010266105415242439,-0.002592261998183278,-0.014959693812643405,-0.05078298047847944,53.0+-0.056370093293081916,-0.044641636506989144,-0.011595014505211082,-0.03321323009955148,-0.046975404140848234,-0.047659849771069886,0.0044604458011053266,-0.03949338287409329,-0.007977142213412223,-0.08806194271198954,190.0+-0.07816532399919843,-0.044641636506989144,-0.07303030271641665,-0.057313186930496314,-0.0841261313122785,-0.07427746902318036,-0.024992656631590206,-0.03949338287409329,-0.018113692315690322,-0.08391983579715509,142.0+0.06713621404157838,0.05068011873981862,-0.041773752573873474,0.011543832586488917,0.0025588987543921156,0.0058885371949405855,0.04127682384197474,-0.03949338287409329,-0.05947118135708968,-0.021788232074638245,75.0+-0.04183993948900423,0.05068011873981862,0.014272475267928093,-0.00567042229275739,-0.012576582685820214,0.0062016856567301245,-0.07285394808472044,0.07120997975363674,0.03545870422305857,-0.013504018244969336,142.0+0.0344433679824036,-0.044641636506989144,-0.007283766209687899,0.014986683562338177,-0.04422349842444599,-0.03732595053201525,-0.002902829807068556,-0.03949338287409329,-0.021395309255276825,0.007206516329202944,155.0+0.059871137139539544,0.05068011873981862,0.016428099415689682,0.028758087465735226,-0.04147159270804375,-0.02918409052548733,-0.028674294435677143,-0.002592261998183278,-0.002398393416115213,-0.021788232074638245,225.0+-0.052737554842062495,-0.044641636506989144,-0.00943939035744949,-0.00567042229275739,0.039709625925822375,0.04471894645684291,0.026550272625626974,-0.002592261998183278,-0.018113692315690322,-0.013504018244969336,59.0+-0.009147093429829445,-0.044641636506989144,-0.015906262800734303,0.07007229917592636,0.01219056876179996,0.022172257207996385,0.01550535921336615,-0.002592261998183278,-0.03324559264822791,0.0486275854775475,104.0+-0.04910501639104307,-0.044641636506989144,0.02505059600673609,0.008100981610639655,0.020446285911006685,0.017788178742942903,0.05232173725423556,-0.03949338287409329,-0.041176166918895155,0.007206516329202944,182.0+-0.04183993948900423,-0.044641636506989144,-0.049318437091039065,-0.03665608107540074,-0.007072771253015731,-0.022607972827907094,0.08545647749101803,-0.03949338287409329,-0.06649019536676071,0.007206516329202944,128.0+-0.04183993948900423,-0.044641636506989144,0.04121777711494808,-0.02632752814785296,-0.0318399227006359,-0.030436684372645465,-0.036037570043851025,0.0029429061332032365,0.033653814906286016,-0.01764612515980379,52.0+-0.027309785684926546,-0.044641636506989144,-0.06332999405148947,-0.05042748497879779,-0.08962994274508297,-0.1043397213549757,0.05232173725423556,-0.0763945037500033,-0.05615310200706333,-0.06735140813781726,37.0+0.04170844488444244,-0.044641636506989144,-0.06440780612537028,0.03564378941743375,0.01219056876179996,-0.05799374901012452,0.18117906039727852,-0.0763945037500033,-0.0006117353045626216,-0.05078298047847944,170.0+0.06350367559055897,0.05068011873981862,-0.02560657146566148,0.011543832586488917,0.06447677737344255,0.048476727998317336,0.030231910429713918,-0.002592261998183278,0.038393928263466416,0.019632837073706312,170.0+-0.07090024709715959,-0.044641636506989144,-0.004050329988045492,-0.04009893205125,-0.06623874415566393,-0.07866154748823384,0.05232173725423556,-0.0763945037500033,-0.05140387304727299,-0.03421455281914162,61.0+-0.04183993948900423,0.05068011873981862,0.004572166603000912,-0.05387033595464705,-0.04422349842444599,-0.02730519975475012,-0.08021722369289432,0.07120997975363674,0.03664373256235367,0.019632837073706312,144.0+-0.027309785684926546,0.05068011873981862,-0.007283766209687899,-0.04009893205125,-0.011200629827619093,-0.013839815897800126,0.059685012862409445,-0.03949338287409329,-0.08237869071592514,-0.025930338989472702,52.0+-0.03457486258696539,-0.044641636506989144,-0.03746250427835029,-0.060756037906345574,0.020446285911006685,0.043466352609684754,-0.01394774321932938,-0.002592261998183278,-0.030747917533098208,-0.07149351505265171,128.0+0.06713621404157838,0.05068011873981862,-0.02560657146566148,-0.04009893205125,-0.06348683843926169,-0.05987263978086174,-0.002902829807068556,-0.03949338287409329,-0.019198449026275908,0.0113486232440374,71.0+-0.045472477940023646,0.05068011873981862,-0.02452875939178067,0.059743746248378575,0.005310804470794357,0.014969842586837095,-0.05444575906428573,0.07120997975363674,0.04234098419358016,0.015490730158871856,163.0+-0.009147093429829445,0.05068011873981862,-0.018061886948495892,-0.03321323009955148,-0.02083229983502694,0.012151506430731283,-0.07285394808472044,0.07120997975363674,0.00027247814860377354,0.019632837073706312,150.0+0.04170844488444244,0.05068011873981862,-0.014828450726853487,-0.017135116042335426,-0.005696818394814609,0.008393724889256856,-0.01394774321932938,-0.0018542395806650938,-0.011896851335695978,0.0030644094143684884,97.0+0.038075906433423026,0.05068011873981862,-0.029917819761184662,-0.04009893205125,-0.033215875558837024,-0.02417371513685477,-0.010266105415242439,-0.002592261998183278,-0.012908683225401873,0.0030644094143684884,160.0+0.016280675727306498,-0.044641636506989144,-0.04608500086939666,-0.00567042229275739,-0.07587041416307178,-0.06143838208980942,-0.01394774321932938,-0.03949338287409329,-0.05140387304727299,0.019632837073706312,178.0+-0.0018820165277906047,-0.044641636506989144,-0.06979686649477428,-0.012556124244455912,-0.00019300696201012598,-0.009142588970957101,0.07072992627467027,-0.03949338287409329,-0.06291687914365544,0.040343371647878594,48.0+-0.0018820165277906047,-0.044641636506989144,0.03367309259778249,0.12515791478951455,0.024574144485610048,0.02624318721126033,-0.010266105415242439,-0.002592261998183278,0.02671684132010462,0.06105390622205087,270.0+0.06350367559055897,0.05068011873981862,-0.004050329988045492,-0.012556124244455912,0.10300345740307394,0.04878987646010685,0.05600337505832251,-0.002592261998183278,0.08449153066204618,-0.01764612515980379,202.0+0.012648137276287077,0.05068011873981862,-0.020217511096257485,-0.002227571316908129,0.03833367306762126,0.05317395492516036,-0.006584467611155497,0.03430885887772673,-0.005142189801713891,-0.009361911330134878,111.0+0.012648137276287077,0.05068011873981862,0.002416542455239321,0.056300895272529315,0.027326050202012293,0.017161881819363848,0.04127682384197474,-0.03949338287409329,0.0037090603325595967,0.07348022696655424,85.0+-0.009147093429829445,0.05068011873981862,-0.03099563183506548,-0.02632752814785296,-0.011200629827619093,-0.0010007289644291908,-0.02131101882750326,-0.002592261998183278,0.006206735447689297,0.027917050903375224,42.0+-0.030942324135945967,0.05068011873981862,0.028284032228378497,0.07007229917592636,-0.12678066991651324,-0.10684490904929198,-0.05444575906428573,-0.047980640675552584,-0.030747917533098208,0.015490730158871856,170.0+-0.09632801625429555,-0.044641636506989144,-0.03638469220446948,-0.07452744180974262,-0.038719686991641515,-0.02761834821653966,0.01550535921336615,-0.03949338287409329,-0.07409260794346935,-0.0010776975004659671,200.0+0.005383060374248237,-0.044641636506989144,-0.05794093368208547,-0.0228846771720037,-0.06761469701386505,-0.0683276482491792,-0.05444575906428573,-0.002592261998183278,0.04289703595278786,-0.08391983579715509,252.0+-0.10359309315633439,-0.044641636506989144,-0.03746250427835029,-0.02632752814785296,0.0025588987543921156,0.019980217975469634,0.01182372140927921,-0.002592261998183278,-0.0683315470939731,-0.025930338989472702,113.0+0.0707687524925978,-0.044641636506989144,0.012116851120166501,0.04252949136913227,0.07135654166444816,0.053487103386949876,0.05232173725423556,-0.002592261998183278,0.025395078941660074,-0.005219804415300423,143.0+0.012648137276287077,0.05068011873981862,-0.022373135244019075,-0.029770379123702218,0.010814615903598841,0.02843522644378708,-0.02131101882750326,0.03430885887772673,-0.006081096870540014,-0.0010776975004659671,51.0+-0.016412170331868287,-0.044641636506989144,-0.035306880130588664,-0.02632752814785296,0.03282986163481677,0.017161881819363848,0.10018302870736581,-0.03949338287409329,-0.07020936123162536,-0.07977772888232063,52.0+-0.03820740103798481,-0.044641636506989144,0.009961226972404908,-0.04698463400294853,-0.05935897986465832,-0.05298337362149199,-0.010266105415242439,-0.03949338287409329,-0.015998872510179042,-0.042498766648810526,210.0+0.001750521923228816,-0.044641636506989144,-0.039618128426111884,-0.10093410879450646,-0.029088016984233665,-0.03012353591085593,0.04495846164606168,-0.05019470792810719,-0.0683315470939731,-0.1294830118603341,65.0+0.04534098333546186,-0.044641636506989144,0.07139651518361048,0.0012152796589411327,-0.009824676969417972,-0.0010007289644291908,0.01550535921336615,-0.03949338287409329,-0.041176166918895155,-0.07149351505265171,141.0+-0.07090024709715959,0.05068011873981862,-0.07518592686417827,-0.04009893205125,-0.051103262715451604,-0.015092409744958261,-0.03971920784793797,-0.002592261998183278,-0.09643494994048712,-0.03421455281914162,55.0+0.04534098333546186,-0.044641636506989144,-0.006205954135807083,0.011543832586488917,0.06310082451524143,0.01622243643399523,0.09650139090327886,-0.03949338287409329,0.04289703595278786,-0.03835665973397607,134.0+-0.052737554842062495,0.05068011873981862,-0.040695940499992665,-0.06764173985804409,-0.0318399227006359,-0.037012802070225705,0.0375951860378878,-0.03949338287409329,-0.03452177701362266,0.06933812005171978,42.0+-0.045472477940023646,-0.044641636506989144,-0.048240625017158284,-0.019441826196154435,-0.00019300696201012598,-0.016031855130326858,0.06704828847058333,-0.03949338287409329,-0.024795429028792802,0.019632837073706312,111.0+0.012648137276287077,-0.044641636506989144,-0.02560657146566148,-0.04009893205125,-0.030463969842434782,-0.045154662076753616,0.07809320188284416,-0.0763945037500033,-0.07213275338232743,0.0113486232440374,98.0+0.04534098333546186,-0.044641636506989144,0.05199589785375607,-0.05387033595464705,0.06310082451524143,0.06476044801137316,-0.010266105415242439,0.03430885887772673,0.037236246732001224,0.019632837073706312,164.0+-0.020044708782887707,-0.044641636506989144,0.004572166603000912,0.09761510698272045,0.005310804470794357,-0.02072908205716988,0.06336665066649638,-0.03949338287409329,0.012551194864223063,0.0113486232440374,48.0+-0.04910501639104307,-0.044641636506989144,-0.06440780612537028,-0.10207024961653671,-0.0029449126784123676,-0.015405558206747801,0.06336665066649638,-0.047242618258034386,-0.03324559264822791,-0.05492508739331389,96.0+-0.07816532399919843,-0.044641636506989144,-0.01698407487461508,-0.012556124244455912,-0.00019300696201012598,-0.013526667436010586,0.07072992627467027,-0.03949338287409329,-0.041176166918895155,-0.092204049626824,90.0+-0.07090024709715959,-0.044641636506989144,-0.05794093368208547,-0.08141314376144114,-0.04559945128264711,-0.02887094206369779,-0.04340084565202491,-0.002592261998183278,0.0011475759991601464,-0.005219804415300423,162.0+0.056238598688520124,0.05068011873981862,0.009961226972404908,0.049415193320830796,-0.004320865536613489,-0.012274073588852453,-0.04340084565202491,0.03430885887772673,0.060790963876144,0.03205915781820968,150.0+-0.027309785684926546,-0.044641636506989144,0.08864150836570328,-0.02519138732582271,0.02182223876920781,0.04252690722431616,-0.03235593223976409,0.03430885887772673,0.002861309289833047,0.07762233388138869,279.0+0.001750521923228816,0.05068011873981862,-0.0051281420619263066,-0.012556124244455912,-0.015328488402222454,-0.013839815897800126,0.008142083605192267,-0.03949338287409329,-0.006081096870540014,-0.06735140813781726,92.0+-0.0018820165277906047,-0.044641636506989144,-0.06440780612537028,0.011543832586488917,0.027326050202012293,0.03751653183568362,-0.01394774321932938,0.03430885887772673,0.011785484244986226,-0.05492508739331389,83.0+0.016280675727306498,-0.044641636506989144,0.0175059114895705,-0.0228846771720037,0.06034891879883919,0.04440579799505339,0.030231910429713918,-0.002592261998183278,0.037236246732001224,-0.0010776975004659671,128.0+0.016280675727306498,0.05068011873981862,-0.04500718879551588,0.06318659722422783,0.010814615903598841,-0.00037443204085011215,0.06336665066649638,-0.03949338287409329,-0.030747917533098208,0.036201264733044136,102.0+-0.09269547780327612,-0.044641636506989144,0.028284032228378497,-0.015998975220305175,0.03695772020942014,0.02499059336410222,0.05600337505832251,-0.03949338287409329,-0.005142189801713891,-0.0010776975004659671,302.0+0.059871137139539544,0.05068011873981862,0.04121777711494808,0.011543832586488917,0.041085578784023497,0.0707102687853743,-0.036037570043851025,0.03430885887772673,-0.010903250651210127,-0.03007244590430716,198.0+-0.027309785684926546,-0.044641636506989144,0.06492964274032566,-0.002227571316908129,-0.0249601584096303,-0.017284448977484993,0.022868634821540033,-0.03949338287409329,-0.061175799045152635,-0.0632093012229828,95.0+0.02354575262934534,0.05068011873981862,-0.0320734439089463,-0.04009893205125,-0.0318399227006359,-0.0216685274425385,-0.01394774321932938,-0.002592261998183278,-0.010903250651210127,0.019632837073706312,53.0+-0.09632801625429555,-0.044641636506989144,-0.07626373893805906,-0.04354178302709927,-0.04559945128264711,-0.03482076283769895,0.008142083605192267,-0.03949338287409329,-0.05947118135708968,-0.08391983579715509,134.0+0.027178291080364757,-0.044641636506989144,0.04984027370599448,-0.0550064767766773,-0.0029449126784123676,0.04064801645357896,-0.05812739686837268,0.05275941931568174,-0.05296264109357657,-0.005219804415300423,144.0+0.01991321417832592,0.05068011873981862,0.045529025410471304,0.029894228287765473,-0.062110885581060565,-0.0558017097775978,-0.07285394808472044,0.026928634702544724,0.0456043699279467,0.040343371647878594,232.0+0.038075906433423026,0.05068011873981862,-0.00943939035744949,0.002351420480971383,0.001182945896190995,0.03751653183568362,-0.05444575906428573,0.05017634085436802,-0.025953110560258,0.1066170822852299,81.0+0.04170844488444244,0.05068011873981862,-0.0320734439089463,-0.0228846771720037,-0.04972730985725048,-0.04014428668812105,0.030231910429713918,-0.03949338287409329,-0.12609712083330468,0.015490730158871856,104.0+0.01991321417832592,-0.044641636506989144,0.004572166603000912,-0.02632752814785296,0.02319819162740893,0.01027261565999407,0.06704828847058333,-0.03949338287409329,-0.02364686309993755,-0.04664087356364498,59.0+-0.08543040090123728,-0.044641636506989144,0.020739347711212906,-0.02632752814785296,0.005310804470794357,0.019667069513680118,-0.002902829807068556,-0.002592261998183278,-0.02364686309993755,0.0030644094143684884,246.0+0.01991321417832592,0.05068011873981862,0.014272475267928093,0.06318659722422783,0.014942474478202204,0.020293366437259194,-0.04708248345611185,0.03430885887772673,0.04666177983070229,0.09004865462589207,297.0+0.02354575262934534,-0.044641636506989144,0.11019774984331929,0.06318659722422783,0.013566521620001083,-0.03294187206696173,-0.024992656631590206,0.020655444153640023,0.09924057568496533,0.02377494398854077,258.0+-0.030942324135945967,0.05068011873981862,0.0013387303813585058,-0.00567042229275739,0.06447677737344255,0.04941617338368593,-0.04708248345611185,0.10811110062954676,0.08379874486368903,0.0030644094143684884,229.0+0.04897352178648128,0.05068011873981862,0.05846277029704089,0.07007229917592636,0.013566521620001083,0.020606514899048713,-0.02131101882750326,0.03430885887772673,0.02200407477075404,0.027917050903375224,275.0+0.059871137139539544,-0.044641636506989144,-0.0212953231701383,0.08728655405517267,0.045213437358626866,0.031566711061682434,-0.04708248345611185,0.07120997975363674,0.07912244072477838,0.13561183068907107,281.0+-0.056370093293081916,0.05068011873981862,-0.010517202431330305,0.02531523648988596,0.02319819162740893,0.04002171952999989,-0.03971920784793797,0.03430885887772673,0.02060938757142981,0.05691179930721642,179.0+0.016280675727306498,-0.044641636506989144,-0.047162812943277475,-0.002227571316908129,-0.019456346976825818,-0.04296262284422686,0.033913548233800855,-0.03949338287409329,0.02736404910541198,0.027917050903375224,200.0+-0.04910501639104307,-0.044641636506989144,0.004572166603000912,0.011543832586488917,-0.037343734133440394,-0.01853704282464315,-0.01762938102341632,-0.002592261998183278,-0.03980882652740082,-0.021788232074638245,200.0+0.06350367559055897,-0.044641636506989144,0.0175059114895705,0.0218723855140367,0.0080627101871966,0.02154596028441731,-0.036037570043851025,0.03430885887772673,0.019907486170462722,0.0113486232440374,173.0+0.04897352178648128,0.05068011873981862,0.08109682384853766,0.0218723855140367,0.04383748450042574,0.06413415108779408,-0.05444575906428573,0.07120997975363674,0.03243232415655107,0.0486275854775475,180.0+0.005383060374248237,0.05068011873981862,0.03475090467166331,-0.0010914304948778783,0.15253776029831428,0.19878798965729408,-0.06180903467245962,0.18523444326019867,0.015568459328120622,0.07348022696655424,84.0+-0.005514554978810025,-0.044641636506989144,0.023972783932855315,0.008100981610639655,-0.034591828417038145,-0.038891692840962916,0.022868634821540033,-0.03949338287409329,-0.015998872510179042,-0.013504018244969336,121.0+-0.005514554978810025,0.05068011873981862,-0.008361578283568675,-0.002227571316908129,-0.033215875558837024,-0.06363042132233616,-0.036037570043851025,-0.002592261998183278,0.0805900527449823,0.007206516329202944,161.0+-0.0890629393522567,-0.044641636506989144,-0.061174369903727877,-0.02632752814785296,-0.05523112129005496,-0.054549115930439665,0.04127682384197474,-0.0763945037500033,-0.09393727482535742,-0.05492508739331389,99.0+0.0344433679824036,0.05068011873981862,-0.0018947058402839008,-0.012556124244455912,0.03833367306762126,0.013717248739678958,0.07809320188284416,-0.03949338287409329,0.004547695772676124,-0.09634615654165846,109.0+-0.052737554842062495,-0.044641636506989144,-0.06225218197760866,-0.02632752814785296,-0.005696818394814609,-0.005071658967693135,0.030231910429713918,-0.03949338287409329,-0.030747917533098208,-0.07149351505265171,115.0+0.009015598825267658,-0.044641636506989144,0.016428099415689682,0.004658130634790395,0.00943866304539772,0.01058576412178361,-0.028674294435677143,0.03430885887772673,0.03896821122789408,0.11904340302973325,268.0+-0.06363517019512076,0.05068011873981862,0.09618619288286882,0.10450080893441897,-0.0029449126784123676,-0.0047585105059035964,-0.006584467611155497,-0.002592261998183278,0.022687744966501246,0.07348022696655424,274.0+-0.09632801625429555,-0.044641636506989144,-0.06979686649477428,-0.06764173985804409,-0.019456346976825818,-0.010708331279904778,0.01550535921336615,-0.03949338287409329,-0.04688253415273158,-0.07977772888232063,158.0+0.016280675727306498,0.05068011873981862,-0.0212953231701383,-0.009113273268606652,0.0342058144930179,0.047850431074738256,0.0007788079970183853,-0.002592261998183278,-0.012908683225401873,0.02377494398854077,107.0+-0.04183993948900423,0.05068011873981862,-0.05362968538656229,-0.04009893205125,-0.0841261313122785,-0.07177228132886408,-0.002902829807068556,-0.03949338287409329,-0.07213275338232743,-0.03007244590430716,83.0+-0.074532785548179,-0.044641636506989144,0.04337340126270967,-0.03321323009955148,0.01219056876179996,0.00025186488272894433,0.06336665066649638,-0.03949338287409329,-0.02712902329694316,-0.04664087356364498,103.0+-0.005514554978810025,-0.044641636506989144,0.0563071461492793,-0.03665608107540074,-0.04835135699904936,-0.04296262284422686,-0.07285394808472044,0.03799897096531772,0.050782032210405316,0.05691179930721642,272.0+-0.09269547780327612,-0.044641636506989144,-0.08165279930746305,-0.057313186930496314,-0.060734932722859444,-0.06801449978738965,0.04864009945014862,-0.0763945037500033,-0.06649019536676071,-0.021788232074638245,85.0+0.005383060374248237,-0.044641636506989144,0.04984027370599448,0.09761510698272045,-0.015328488402222454,-0.016345003592116398,-0.006584467611155497,-0.002592261998183278,0.017036071348324546,-0.013504018244969336,280.0+0.0344433679824036,0.05068011873981862,0.11127556191720007,0.07695800112762488,-0.0318399227006359,-0.033881317452330355,-0.02131101882750326,-0.002592261998183278,0.02802037249332928,0.07348022696655424,336.0+0.02354575262934534,-0.044641636506989144,0.061696206518683294,0.052858044296680055,-0.034591828417038145,-0.048912443618228024,-0.028674294435677143,-0.002592261998183278,0.05471997253790904,-0.005219804415300423,281.0+0.04170844488444244,0.05068011873981862,0.014272475267928093,0.04252949136913227,-0.030463969842434782,-0.0013138774262187302,-0.04340084565202491,-0.002592261998183278,-0.03324559264822791,0.015490730158871856,118.0+-0.027309785684926546,-0.044641636506989144,0.04768464955823289,-0.04698463400294853,0.0342058144930179,0.057244884928424306,-0.08021722369289432,0.13025177315509276,0.04506654937395887,0.13146972377423663,317.0+0.04170844488444244,0.05068011873981862,0.012116851120166501,0.03908664039328301,0.05484510736603471,0.04440579799505339,0.0044604458011053266,-0.002592261998183278,0.0456043699279467,-0.0010776975004659671,235.0+-0.030942324135945967,-0.044641636506989144,0.005649978676881689,-0.009113273268606652,0.019070333052805567,0.006827982580309182,0.07441156407875721,-0.03949338287409329,-0.041176166918895155,-0.042498766648810526,60.0+0.03081082953138418,0.05068011873981862,0.04660683748435208,-0.015998975220305175,0.020446285911006685,0.05066876723084409,-0.05812739686837268,0.07120997975363674,0.006206735447689297,0.007206516329202944,174.0+-0.04183993948900423,-0.044641636506989144,0.12852055509929283,0.06318659722422783,-0.033215875558837024,-0.03262872360517222,0.01182372140927921,-0.03949338287409329,-0.015998872510179042,-0.05078298047847944,259.0+-0.030942324135945967,0.05068011873981862,0.05954058237092167,0.0012152796589411327,0.01219056876179996,0.031566711061682434,-0.04340084565202491,0.03430885887772673,0.014820979914103747,0.007206516329202944,178.0+-0.056370093293081916,-0.044641636506989144,0.09295275666122646,-0.019441826196154435,0.014942474478202204,0.023424851055154544,-0.028674294435677143,0.02545258986750832,0.026060517932187322,0.040343371647878594,128.0+-0.06000263174410134,0.05068011873981862,0.015350287341808908,-0.019441826196154435,0.03695772020942014,0.04816357953652778,0.019186997017453092,-0.002592261998183278,-0.030747917533098208,-0.0010776975004659671,96.0+-0.04910501639104307,0.05068011873981862,-0.0051281420619263066,-0.04698463400294853,-0.02083229983502694,-0.02041593359538034,-0.0691723102806335,0.07120997975363674,0.06123762840403217,-0.03835665973397607,126.0+0.02354575262934534,-0.044641636506989144,0.07031870310972965,0.02531523648988596,-0.034591828417038145,-0.014466112821379181,-0.03235593223976409,-0.002592261998183278,-0.019198449026275908,-0.009361911330134878,288.0+0.001750521923228816,-0.044641636506989144,-0.004050329988045492,-0.00567042229275739,-0.008448724111216851,-0.02386056667506523,0.05232173725423556,-0.03949338287409329,-0.00894339609006817,-0.013504018244969336,88.0+-0.03457486258696539,0.05068011873981862,-0.0008168937664030856,0.07007229917592636,0.039709625925822375,0.06695248724389988,-0.06549067247654655,0.10811110062954676,0.02671684132010462,0.07348022696655424,292.0+0.04170844488444244,0.05068011873981862,-0.04392937672163507,0.06318659722422783,-0.004320865536613489,0.01622243643399523,-0.01394774321932938,-0.002592261998183278,-0.03452177701362266,0.0113486232440374,71.0+0.06713621404157838,0.05068011873981862,0.020739347711212906,-0.00567042229275739,0.020446285911006685,0.02624318721126033,-0.002902829807068556,-0.002592261998183278,0.008640601344549246,0.0030644094143684884,197.0+-0.027309785684926546,0.05068011873981862,0.06061839444480248,0.049415193320830796,0.08511607024645937,0.08636769187485104,-0.002902829807068556,0.03430885887772673,0.037810529696428806,0.0486275854775475,186.0+-0.016412170331868287,-0.044641636506989144,-0.010517202431330305,0.0012152796589411327,-0.037343734133440394,-0.035760208223067566,0.01182372140927921,-0.03949338287409329,-0.021395309255276825,-0.03421455281914162,25.0+-0.0018820165277906047,0.05068011873981862,-0.033151255982827074,-0.018305685374124185,0.03145390877661565,0.042840055686105716,-0.01394774321932938,0.019917421736121838,0.010226716198682649,0.027917050903375224,84.0+-0.012779631880848867,-0.044641636506989144,-0.06548561819925106,-0.0699484500118631,0.001182945896190995,0.016848733357574308,-0.002902829807068556,-0.007020396503292483,-0.030747917533098208,-0.05078298047847944,96.0+-0.005514554978810025,-0.044641636506989144,0.04337340126270967,0.08728655405517267,0.013566521620001083,0.0071411310420987206,-0.01394774321932938,-0.002592261998183278,0.04234098419358016,-0.01764612515980379,195.0+-0.009147093429829445,-0.044641636506989144,-0.06225218197760866,-0.07452744180974262,-0.02358420555142918,-0.013213518974221048,0.0044604458011053266,-0.03949338287409329,-0.0358161925842373,-0.04664087356364498,53.0+-0.045472477940023646,0.05068011873981862,0.06385183066644486,0.07007229917592636,0.1332744202834986,0.1314610703725441,-0.03971920784793797,0.10811110062954676,0.07574055215648227,0.0859065477110576,217.0+-0.052737554842062495,-0.044641636506989144,0.030439656376140087,-0.07452744180974262,-0.02358420555142918,-0.011334628203483833,-0.002902829807068556,-0.002592261998183278,-0.030747917533098208,-0.0010776975004659671,172.0+0.016280675727306498,0.05068011873981862,0.0724743272574913,0.07695800112762488,-0.008448724111216851,0.0055753887331510465,-0.006584467611155497,-0.002592261998183278,-0.02364686309993755,0.06105390622205087,131.0+0.04534098333546186,-0.044641636506989144,-0.01913969902237667,0.0218723855140367,0.027326050202012293,-0.013526667436010586,0.10018302870736581,-0.03949338287409329,0.017765319557121535,-0.013504018244969336,214.0+-0.04183993948900423,-0.044641636506989144,-0.06656343027313188,-0.04698463400294853,-0.037343734133440394,-0.043275771306016404,0.04864009945014862,-0.03949338287409329,-0.05615310200706333,-0.013504018244969336,59.0+-0.056370093293081916,0.05068011873981862,-0.06009655782984706,-0.03665608107540074,-0.08825398988688185,-0.07083283594349546,-0.01394774321932938,-0.03949338287409329,-0.07813993550229265,-0.10463037037132736,70.0+0.0707687524925978,-0.044641636506989144,0.06924089103584885,0.03795049957125276,0.02182223876920781,0.0015044587298871017,-0.036037570043851025,0.03910600459159503,0.07763659749935449,0.1066170822852299,220.0+0.001750521923228816,0.05068011873981862,0.05954058237092167,-0.002227571316908129,0.06172487165704031,0.0631947057024255,-0.05812739686837268,0.10811110062954676,0.0689858906225002,0.12732761685940217,268.0+-0.0018820165277906047,-0.044641636506989144,-0.0266843835395423,0.049415193320830796,0.05897296594063807,-0.016031855130326858,-0.04708248345611185,0.07120997975363674,0.13359728192191356,0.019632837073706312,152.0+0.02354575262934534,0.05068011873981862,-0.020217511096257485,-0.03665608107540074,-0.013952535544021335,-0.015092409744958261,0.059685012862409445,-0.03949338287409329,-0.09643494994048712,-0.01764612515980379,47.0+-0.020044708782887707,-0.044641636506989144,-0.04608500086939666,-0.09862739864068745,-0.07587041416307178,-0.05987263978086174,-0.01762938102341632,-0.03949338287409329,-0.05140387304727299,-0.04664087356364498,74.0+0.04170844488444244,0.05068011873981862,0.07139651518361048,0.008100981610639655,0.03833367306762126,0.01590928797220569,-0.01762938102341632,0.03430885887772673,0.07340695788833193,0.0859065477110576,295.0+-0.06363517019512076,0.05068011873981862,-0.07949717515970146,-0.00567042229275739,-0.07174255558846841,-0.06644875747844198,-0.010266105415242439,-0.03949338287409329,-0.018113692315690322,-0.05492508739331389,101.0+0.016280675727306498,0.05068011873981862,0.009961226972404908,-0.04354178302709927,-0.09650970703608859,-0.09463211903950011,-0.03971920784793797,-0.03949338287409329,0.017036071348324546,0.007206516329202944,151.0+0.06713621404157838,-0.044641636506989144,-0.03854031635223107,-0.02632752814785296,-0.0318399227006359,-0.02636575436938152,0.008142083605192267,-0.03949338287409329,-0.02712902329694316,0.0030644094143684884,127.0+0.04534098333546186,0.05068011873981862,0.01966153563733209,0.03908664039328301,0.020446285911006685,0.025930038749470818,0.008142083605192267,-0.002592261998183278,-0.003300838074501491,0.019632837073706312,237.0+0.04897352178648128,-0.044641636506989144,0.027206220154497678,-0.02519138732582271,0.02319819162740893,0.018414475666521983,-0.06180903467245962,0.08006624876385515,0.07222192954903682,0.03205915781820968,225.0+0.04170844488444244,-0.044641636506989144,-0.008361578283568675,-0.02632752814785296,0.024574144485610048,0.01622243643399523,0.07072992627467027,-0.03949338287409329,-0.04835926177554553,-0.03007244590430716,81.0+-0.02367724723390713,-0.044641636506989144,-0.015906262800734303,-0.012556124244455912,0.020446285911006685,0.04127431337715804,-0.04340084565202491,0.03430885887772673,0.014073500500086794,-0.009361911330134878,151.0+-0.03820740103798481,0.05068011873981862,0.004572166603000912,0.03564378941743375,-0.011200629827619093,0.0058885371949405855,-0.04708248345611185,0.03430885887772673,0.016306823139527554,-0.0010776975004659671,107.0+0.04897352178648128,-0.044641636506989144,-0.04285156464775429,-0.05387033595464705,0.045213437358626866,0.05004247030726501,0.033913548233800855,-0.002592261998183278,-0.025953110560258,-0.0632093012229828,64.0+0.04534098333546186,0.05068011873981862,0.005649978676881689,0.056300895272529315,0.06447677737344255,0.08918602803095686,-0.03971920784793797,0.07120997975363674,0.015568459328120622,-0.009361911330134878,138.0+0.04534098333546186,0.05068011873981862,-0.035306880130588664,0.06318659722422783,-0.004320865536613489,-0.0016270258880082473,-0.010266105415242439,-0.002592261998183278,0.015568459328120622,0.05691179930721642,185.0+0.016280675727306498,-0.044641636506989144,0.023972783932855315,-0.0228846771720037,-0.0249601584096303,-0.02605260590759198,-0.03235593223976409,-0.002592261998183278,0.037236246732001224,0.03205915781820968,265.0+-0.074532785548179,0.05068011873981862,-0.018061886948495892,0.008100981610639655,-0.019456346976825818,-0.02480001206043385,-0.06549067247654655,0.03430885887772673,0.06731773534487707,-0.01764612515980379,101.0+-0.08179786245021785,0.05068011873981862,0.0422955891888289,-0.019441826196154435,0.039709625925822375,0.05755803339021383,-0.0691723102806335,0.10811110062954676,0.04719048478208009,-0.03835665973397607,137.0+-0.06726770864614018,-0.044641636506989144,-0.05470749746044306,-0.02632752814785296,-0.07587041416307178,-0.08210618056791873,0.04864009945014862,-0.0763945037500033,-0.08682710478958679,-0.10463037037132736,143.0+0.005383060374248237,-0.044641636506989144,-0.002972517914164677,0.049415193320830796,0.0741084473808504,0.0707102687853743,0.04495846164606168,-0.002592261998183278,-0.0014959487577289358,-0.009361911330134878,141.0+-0.0018820165277906047,-0.044641636506989144,-0.06656343027313188,0.0012152796589411327,-0.0029449126784123676,0.003070201038834776,0.01182372140927921,-0.002592261998183278,-0.020292321339471356,-0.025930338989472702,79.0+0.009015598825267658,-0.044641636506989144,-0.012672826579091896,0.028758087465735226,-0.018080394118624697,-0.005071658967693135,-0.04708248345611185,0.03430885887772673,0.02337141516224845,-0.005219804415300423,292.0+-0.005514554978810025,0.05068011873981862,-0.041773752573873474,-0.04354178302709927,-0.07999827273767514,-0.07615635979391756,-0.03235593223976409,-0.03949338287409329,0.010226716198682649,-0.009361911330134878,178.0+0.056238598688520124,0.05068011873981862,-0.03099563183506548,0.008100981610639655,0.019070333052805567,0.02123281182262779,0.033913548233800855,-0.03949338287409329,-0.02952642678336326,-0.05906719430814835,91.0+0.009015598825267658,0.05068011873981862,-0.0051281420619263066,-0.06419888888219483,0.06998058880624704,0.08386250418053477,-0.03971920784793797,0.07120997975363674,0.03954249419232167,0.019632837073706312,116.0+-0.06726770864614018,-0.044641636506989144,-0.05901874575596628,0.03220093844158448,-0.051103262715451604,-0.049538740541807104,-0.010266105415242439,-0.03949338287409329,0.0020044426444966374,0.02377494398854077,86.0+0.027178291080364757,0.05068011873981862,0.02505059600673609,0.014986683562338177,0.02595009734381117,0.048476727998317336,-0.03971920784793797,0.03430885887772673,0.007838428314872565,0.02377494398854077,122.0+-0.02367724723390713,-0.044641636506989144,-0.04608500086939666,-0.03321323009955148,0.03282986163481677,0.03626393798852546,0.0375951860378878,-0.002592261998183278,-0.03324559264822791,0.0113486232440374,72.0+0.04897352178648128,0.05068011873981862,0.0034943545291200974,0.07007229917592636,-0.008448724111216851,0.013404100277889418,-0.05444575906428573,0.03430885887772673,0.013316905483459898,0.036201264733044136,129.0+-0.052737554842062495,-0.044641636506989144,0.05415152200151766,-0.02632752814785296,-0.05523112129005496,-0.033881317452330355,-0.01394774321932938,-0.03949338287409329,-0.07409260794346935,-0.05906719430814835,142.0+0.04170844488444244,-0.044641636506989144,-0.04500718879551588,0.03450764859540349,0.04383748450042574,-0.015718706668537318,0.0375951860378878,-0.014400620678474476,0.0898970830097539,0.007206516329202944,90.0+0.056238598688520124,-0.044641636506989144,-0.05794093368208547,-0.0079771324465764,0.05209320164963247,0.049103024921896415,0.05600337505832251,-0.021411833644897377,-0.028323167238848198,0.044485478562713045,158.0+-0.03457486258696539,0.05068011873981862,-0.05578530953432388,-0.015998975220305175,-0.009824676969417972,-0.007889995123798945,0.0375951860378878,-0.03949338287409329,-0.05296264109357657,0.027917050903375224,39.0+0.08166636784565606,0.05068011873981862,0.0013387303813585058,0.03564378941743375,0.126394655992493,0.09106491880169407,0.019186997017453092,0.03430885887772673,0.08449153066204618,-0.03007244590430716,196.0+-0.0018820165277906047,0.05068011873981862,0.030439656376140087,0.052858044296680055,0.039709625925822375,0.056618588004845226,-0.03971920784793797,0.07120997975363674,0.025395078941660074,0.027917050903375224,222.0+0.11072667545381144,0.05068011873981862,0.006727790750762504,0.028758087465735226,-0.027712064126032544,-0.007263698200219888,-0.04708248345611185,0.03430885887772673,0.0020044426444966374,0.07762233388138869,277.0+-0.030942324135945967,-0.044641636506989144,0.04660683748435208,0.014986683562338177,-0.016704441260423575,-0.047033552847490806,0.0007788079970183853,-0.002592261998183278,0.06345271983825305,-0.025930338989472702,99.0+0.001750521923228816,0.05068011873981862,0.026128408080616904,-0.009113273268606652,0.024574144485610048,0.03845597722105221,-0.02131101882750326,0.03430885887772673,0.009433658771615987,0.0030644094143684884,196.0+0.009015598825267658,-0.044641636506989144,0.045529025410471304,0.028758087465735226,0.01219056876179996,-0.013839815897800126,0.026550272625626974,-0.03949338287409329,0.04613307487932449,0.036201264733044136,202.0+0.03081082953138418,-0.044641636506989144,0.04013996504106731,0.07695800112762488,0.017694380194604446,0.037829680297473134,-0.028674294435677143,0.03430885887772673,-0.0014959487577289358,0.11904340302973325,155.0+0.038075906433423026,0.05068011873981862,-0.018061886948495892,0.0666294482000771,-0.051103262715451604,-0.016658152053905938,-0.07653558588880739,0.03430885887772673,-0.011896851335695978,-0.013504018244969336,77.0+0.009015598825267658,-0.044641636506989144,0.014272475267928093,0.014986683562338177,0.05484510736603471,0.04722413415115918,0.07072992627467027,-0.03949338287409329,-0.03324559264822791,-0.05906719430814835,191.0+0.09256398319871433,-0.044641636506989144,0.0369065288194249,0.0218723855140367,-0.0249601584096303,-0.016658152053905938,0.0007788079970183853,-0.03949338287409329,-0.022516528376302174,-0.021788232074638245,70.0+0.06713621404157838,-0.044641636506989144,0.0034943545291200974,0.03564378941743375,0.04934129593323023,0.03125356259989291,0.07072992627467027,-0.03949338287409329,-0.0006117353045626216,0.019632837073706312,73.0+0.001750521923228816,-0.044641636506989144,-0.07087467856865506,-0.0228846771720037,-0.001568959820211247,-0.0010007289644291908,0.026550272625626974,-0.03949338287409329,-0.022516528376302174,0.007206516329202944,49.0+0.03081082953138418,-0.044641636506989144,-0.033151255982827074,-0.0228846771720037,-0.046975404140848234,-0.08116673518255012,0.10386466651145274,-0.0763945037500033,-0.03980882652740082,-0.05492508739331389,65.0+0.027178291080364757,0.05068011873981862,0.09403056873510728,0.09761510698272045,-0.034591828417038145,-0.032002426681593144,-0.04340084565202491,-0.002592261998183278,0.03664373256235367,0.1066170822852299,263.0+0.012648137276287077,0.05068011873981862,0.03582871674554409,0.049415193320830796,0.053469154507833586,0.07415490186505921,-0.0691723102806335,0.14501222150545676,0.0456043699279467,0.0486275854775475,248.0+0.07440129094361722,-0.044641636506989144,0.031517468450020895,0.10105795795856971,0.04658939021682799,0.03689023491210454,0.01550535921336615,-0.002592261998183278,0.033653814906286016,0.044485478562713045,296.0+-0.04183993948900423,-0.044641636506989144,-0.06548561819925106,-0.04009893205125,-0.005696818394814609,0.014343545663258015,-0.04340084565202491,0.03430885887772673,0.007027139682585861,-0.013504018244969336,214.0+-0.0890629393522567,-0.044641636506989144,-0.041773752573873474,-0.019441826196154435,-0.06623874415566393,-0.07427746902318036,0.008142083605192267,-0.03949338287409329,0.0011475759991601464,-0.03007244590430716,185.0+0.02354575262934534,0.05068011873981862,-0.039618128426111884,-0.00567042229275739,-0.04835135699904936,-0.033255020528751275,0.01182372140927921,-0.03949338287409329,-0.10163995903077562,-0.06735140813781726,78.0+-0.045472477940023646,-0.044641636506989144,-0.03854031635223107,-0.02632752814785296,-0.015328488402222454,0.0008781618063080231,-0.03235593223976409,-0.002592261998183278,0.0011475759991601464,-0.03835665973397607,93.0+-0.02367724723390713,0.05068011873981862,-0.02560657146566148,0.04252949136913227,-0.05385516843185383,-0.047659849771069886,-0.02131101882750326,-0.03949338287409329,0.0011475759991601464,0.019632837073706312,252.0+-0.09996055470531495,-0.044641636506989144,-0.023450947317899894,-0.06419888888219483,-0.0579830270064572,-0.06018578824265128,0.01182372140927921,-0.03949338287409329,-0.018113692315690322,-0.05078298047847944,150.0+-0.027309785684926546,-0.044641636506989144,-0.06656343027313188,-0.11239880254408448,-0.04972730985725048,-0.041396880535279186,0.0007788079970183853,-0.03949338287409329,-0.0358161925842373,-0.009361911330134878,77.0+0.03081082953138418,0.05068011873981862,0.032595280523901676,0.049415193320830796,-0.04009563984984263,-0.04358891976780594,-0.0691723102806335,0.03430885887772673,0.06301517091297482,0.0030644094143684884,208.0+-0.10359309315633439,0.05068011873981862,-0.04608500086939666,-0.02632752814785296,-0.0249601584096303,-0.02480001206043385,0.030231910429713918,-0.03949338287409329,-0.03980882652740082,-0.05492508739331389,77.0+0.06713621404157838,0.05068011873981862,-0.029917819761184662,0.05743703609455957,-0.00019300696201012598,-0.015718706668537318,0.07441156407875721,-0.05056371913686628,-0.03845971734112638,0.007206516329202944,108.0+-0.052737554842062495,-0.044641636506989144,-0.012672826579091896,-0.060756037906345574,-0.00019300696201012598,0.008080576427467316,0.01182372140927921,-0.002592261998183278,-0.02712902329694316,-0.05078298047847944,160.0+-0.027309785684926546,0.05068011873981862,-0.015906262800734303,-0.029770379123702218,0.003934851612593237,-0.0006875805026396515,0.04127682384197474,-0.03949338287409329,-0.02364686309993755,0.0113486232440374,53.0+-0.03820740103798481,0.05068011873981862,0.07139651518361048,-0.057313186930496314,0.15391371315651542,0.1558866503921278,0.0007788079970183853,0.07194800217115493,0.050280674066857343,0.06933812005171978,220.0+0.009015598825267658,-0.044641636506989144,-0.03099563183506548,0.0218723855140367,0.0080627101871966,0.008706873351046395,0.0044604458011053266,-0.002592261998183278,0.009433658771615987,0.0113486232440374,154.0+0.012648137276287077,0.05068011873981862,0.00026091830747769084,-0.011419983422425662,0.039709625925822375,0.057244884928424306,-0.03971920784793797,0.056080520194513636,0.024055085357995654,0.03205915781820968,259.0+0.06713621404157838,-0.044641636506989144,0.0369065288194249,-0.05042748497879779,-0.02358420555142918,-0.034507614375909414,0.04864009945014862,-0.03949338287409329,-0.025953110560258,-0.03835665973397607,90.0+0.04534098333546186,-0.044641636506989144,0.039062152967186486,0.04597234234498153,0.006686757328995478,-0.02417371513685477,0.008142083605192267,-0.012555564634678981,0.06432781768880942,0.05691179930721642,246.0+0.06713621404157838,0.05068011873981862,-0.014828450726853487,0.05860760542634833,-0.05935897986465832,-0.034507614375909414,-0.06180903467245962,0.012906208769698923,-0.005142189801713891,0.0486275854775475,124.0+0.027178291080364757,-0.044641636506989144,0.006727790750762504,0.03564378941743375,0.07961225881365488,0.0707102687853743,0.01550535921336615,0.03430885887772673,0.04067282891595704,0.0113486232440374,67.0+0.056238598688520124,-0.044641636506989144,-0.06871905442089347,-0.06877788068007434,-0.00019300696201012598,-0.0010007289644291908,0.04495846164606168,-0.03764832683029779,-0.04835926177554553,-0.0010776975004659671,72.0+0.0344433679824036,0.05068011873981862,-0.00943939035744949,0.059743746248378575,-0.035967781275239266,-0.007576846662009428,-0.07653558588880739,0.07120997975363674,0.011010658023139448,-0.021788232074638245,257.0+0.02354575262934534,-0.044641636506989144,0.01966153563733209,-0.012556124244455912,0.08374011738825825,0.03876912568284173,0.06336665066649638,-0.002592261998183278,0.06605066658209227,0.0486275854775475,262.0+0.04897352178648128,0.05068011873981862,0.07462995140525285,0.0666294482000771,-0.009824676969417972,-0.002253322811587326,-0.04340084565202491,0.03430885887772673,0.033653814906286016,0.019632837073706312,275.0+0.03081082953138418,0.05068011873981862,-0.008361578283568675,0.004658130634790395,0.014942474478202204,0.02749578105841849,0.008142083605192267,-0.008127430129569777,-0.02952642678336326,0.05691179930721642,177.0+-0.10359309315633439,0.05068011873981862,-0.023450947317899894,-0.0228846771720037,-0.08687803702868073,-0.06770135132560012,-0.01762938102341632,-0.03949338287409329,-0.07813993550229265,-0.07149351505265171,71.0+0.016280675727306498,0.05068011873981862,-0.04608500086939666,0.011543832586488917,-0.033215875558837024,-0.016031855130326858,-0.010266105415242439,-0.002592261998183278,-0.04398377252276359,-0.042498766648810526,47.0+-0.06000263174410134,0.05068011873981862,0.05415152200151766,-0.019441826196154435,-0.04972730985725048,-0.048912443618228024,0.022868634821540033,-0.03949338287409329,-0.04398377252276359,-0.005219804415300423,187.0+-0.027309785684926546,-0.044641636506989144,-0.035306880130588664,-0.029770379123702218,-0.05660707414825608,-0.0586200459337036,0.030231910429713918,-0.03949338287409329,-0.049872451808799324,-0.1294830118603341,125.0+0.04170844488444244,-0.044641636506989144,-0.0320734439089463,-0.06189217872837582,0.07961225881365488,0.05098191569263361,0.05600337505832251,-0.009972486173365287,0.04506654937395887,-0.05906719430814835,78.0+-0.08179786245021785,-0.044641636506989144,-0.08165279930746305,-0.04009893205125,0.0025588987543921156,-0.01853704282464315,0.07072992627467027,-0.03949338287409329,-0.010903250651210127,-0.092204049626824,51.0+-0.04183993948900423,-0.044641636506989144,0.04768464955823289,0.059743746248378575,0.1277706088506941,0.1280164372928592,-0.024992656631590206,0.10811110062954676,0.0638902687635312,0.040343371647878594,258.0+-0.012779631880848867,-0.044641636506989144,0.06061839444480248,0.052858044296680055,0.04796534307502911,0.02937467182915568,-0.01762938102341632,0.03430885887772673,0.07020738137223513,0.007206516329202944,215.0+0.06713621404157838,-0.044641636506989144,0.0563071461492793,0.07351515015177562,-0.013952535544021335,-0.03920484130275244,-0.03235593223976409,-0.002592261998183278,0.07574055215648227,0.036201264733044136,303.0+-0.052737554842062495,0.05068011873981862,0.09834181703063047,0.08728655405517267,0.06034891879883919,0.04878987646010685,-0.05812739686837268,0.10811110062954676,0.08449153066204618,0.040343371647878594,243.0+0.005383060374248237,-0.044641636506989144,0.05954058237092167,-0.05617704610846606,0.024574144485610048,0.052860806463370796,-0.04340084565202491,0.05091436327188625,-0.00422151393810765,-0.03007244590430716,91.0+0.08166636784565606,-0.044641636506989144,0.03367309259778249,0.008100981610639655,0.05209320164963247,0.056618588004845226,-0.01762938102341632,0.03430885887772673,0.03486619005341102,0.06933812005171978,150.0+0.03081082953138418,0.05068011873981862,0.0563071461492793,0.07695800112762488,0.04934129593323023,-0.012274073588852453,-0.036037570043851025,0.07120997975363674,0.12005149644350945,0.09004865462589207,310.0+0.001750521923228816,-0.044641636506989144,-0.06548561819925106,-0.00567042229275739,-0.007072771253015731,-0.019476488210011744,0.04127682384197474,-0.03949338287409329,-0.003300838074501491,0.007206516329202944,153.0+-0.04910501639104307,-0.044641636506989144,0.16085491731571683,-0.04698463400294853,-0.029088016984233665,-0.019789636671801284,-0.04708248345611185,0.03430885887772673,0.02802037249332928,0.0113486232440374,346.0+-0.027309785684926546,0.05068011873981862,-0.05578530953432388,0.02531523648988596,-0.007072771253015731,-0.02354741821327569,0.05232173725423556,-0.03949338287409329,-0.005142189801713891,-0.05078298047847944,63.0+0.07803382939463664,0.05068011873981862,-0.02452875939178067,-0.04240564220506902,0.006686757328995478,0.052860806463370796,-0.0691723102806335,0.08080427118137334,-0.0371288393600719,0.05691179930721642,89.0+0.012648137276287077,-0.044641636506989144,-0.03638469220446948,0.04252949136913227,-0.013952535544021335,0.01293437758520512,-0.026833475533633678,0.005156973385757823,-0.04398377252276359,0.007206516329202944,50.0+0.04170844488444244,-0.044641636506989144,-0.008361578283568675,-0.057313186930496314,0.0080627101871966,-0.031376129758014064,0.151725957964583,-0.0763945037500033,-0.08023652410258396,-0.01764612515980379,39.0+0.04897352178648128,-0.044641636506989144,-0.041773752573873474,0.10450080893441897,0.03558176735121902,-0.02573945744580244,0.17749742259319157,-0.0763945037500033,-0.012908683225401873,0.015490730158871856,103.0+-0.016412170331868287,0.05068011873981862,0.1274427430254121,0.09761510698272045,0.016318427336403322,0.017475030281153364,-0.02131101882750326,0.03430885887772673,0.03486619005341102,0.0030644094143684884,308.0+-0.074532785548179,0.05068011873981862,-0.07734155101193986,-0.04698463400294853,-0.046975404140848234,-0.03262872360517222,0.0044604458011053266,-0.03949338287409329,-0.07213275338232743,-0.01764612515980379,116.0+0.0344433679824036,0.05068011873981862,0.028284032228378497,-0.03321323009955148,-0.04559945128264711,-0.009768885894536158,-0.050764121260198795,-0.002592261998183278,-0.05947118135708968,-0.021788232074638245,145.0+-0.03457486258696539,0.05068011873981862,-0.02560657146566148,-0.017135116042335426,0.001182945896190995,-0.0028796197351664047,0.008142083605192267,-0.015507654304751785,0.014820979914103747,0.040343371647878594,74.0+-0.052737554842062495,0.05068011873981862,-0.06225218197760866,0.011543832586488917,-0.008448724111216851,-0.03669965360843617,0.12227285553188745,-0.0763945037500033,-0.08682710478958679,0.0030644094143684884,45.0+0.059871137139539544,-0.044641636506989144,-0.0008168937664030856,-0.0848559947372904,0.07548440023905152,0.07947842571548126,0.0044604458011053266,0.03430885887772673,0.02337141516224845,0.027917050903375224,115.0+0.06350367559055897,0.05068011873981862,0.08864150836570328,0.07007229917592636,0.020446285911006685,0.03751653183568362,-0.050764121260198795,0.07120997975363674,0.02929655685872395,0.07348022696655424,264.0+0.009015598825267658,-0.044641636506989144,-0.0320734439089463,-0.02632752814785296,0.04246153164222462,-0.010395182818115238,0.15908923357275687,-0.0763945037500033,-0.011896851335695978,-0.03835665973397607,87.0+0.005383060374248237,0.05068011873981862,0.030439656376140087,0.08384370307932341,-0.037343734133440394,-0.04734670130928034,0.01550535921336615,-0.03949338287409329,0.008640601344549246,0.015490730158871856,202.0+0.038075906433423026,0.05068011873981862,0.008883414898524095,0.04252949136913227,-0.04284754556624487,-0.02104223051895942,-0.03971920784793797,-0.002592261998183278,-0.018113692315690322,0.007206516329202944,127.0+0.012648137276287077,-0.044641636506989144,0.006727790750762504,-0.05617704610846606,-0.07587041416307178,-0.06644875747844198,-0.02131101882750326,-0.03764832683029779,-0.018113692315690322,-0.092204049626824,182.0+0.07440129094361722,0.05068011873981862,-0.020217511096257485,0.04597234234498153,0.0741084473808504,0.032819304908840594,-0.036037570043851025,0.07120997975363674,0.10635074572073594,0.036201264733044136,241.0+0.016280675727306498,-0.044641636506989144,-0.02452875939178067,0.03564378941743375,-0.007072771253015731,-0.003192768196955922,-0.01394774321932938,-0.002592261998183278,0.015568459328120622,0.015490730158871856,66.0+-0.005514554978810025,0.05068011873981862,-0.011595014505211082,0.011543832586488917,-0.02220825269322806,-0.015405558206747801,-0.02131101882750326,-0.002592261998183278,0.011010658023139448,0.06933812005171978,94.0+0.012648137276287077,-0.044641636506989144,0.026128408080616904,0.06318659722422783,0.12501870313429186,0.09169121572527314,0.06336665066649638,-0.002592261998183278,0.057573156154827256,-0.021788232074638245,283.0+-0.03457486258696539,-0.044641636506989144,-0.05901874575596628,0.0012152796589411327,-0.05385516843185383,-0.07803525056465478,0.06704828847058333,-0.0763945037500033,-0.021395309255276825,0.015490730158871856,64.0+0.06713621404157838,0.05068011873981862,-0.03638469220446948,-0.0848559947372904,-0.007072771253015731,0.019667069513680118,-0.05444575906428573,0.03430885887772673,0.0011475759991601464,0.03205915781820968,102.0+0.038075906433423026,0.05068011873981862,-0.02452875939178067,0.004658130634790395,-0.026336111267831423,-0.02636575436938152,0.01550535921336615,-0.03949338287409329,-0.015998872510179042,-0.025930338989472702,200.0+0.009015598825267658,0.05068011873981862,0.018583723563451313,0.03908664039328301,0.017694380194604446,0.01058576412178361,0.019186997017453092,-0.002592261998183278,0.016306823139527554,-0.01764612515980379,265.0+-0.09269547780327612,0.05068011873981862,-0.09027529589850945,-0.057313186930496314,-0.0249601584096303,-0.030436684372645465,-0.006584467611155497,-0.002592261998183278,0.024055085357995654,0.0030644094143684884,94.0+0.0707687524925978,-0.044641636506989144,-0.0051281420619263066,-0.00567042229275739,0.08786797596286161,0.10296456034969638,0.01182372140927921,0.03430885887772673,-0.00894339609006817,0.027917050903375224,230.0+-0.016412170331868287,-0.044641636506989144,-0.05255187331268147,-0.03321323009955148,-0.04422349842444599,-0.036386505146646625,0.019186997017453092,-0.03949338287409329,-0.0683315470939731,-0.03007244590430716,181.0+0.04170844488444244,0.05068011873981862,-0.022373135244019075,0.028758087465735226,-0.06623874415566393,-0.045154662076753616,-0.06180903467245962,-0.002592261998183278,0.002861309289833047,-0.05492508739331389,156.0+0.012648137276287077,-0.044641636506989144,-0.020217511096257485,-0.015998975220305175,0.01219056876179996,0.02123281182262779,-0.07653558588880739,0.10811110062954676,0.05987940361514779,-0.021788232074638245,233.0+-0.03820740103798481,-0.044641636506989144,-0.05470749746044306,-0.07797029278559188,-0.033215875558837024,-0.08649025903297221,0.14068104455232217,-0.0763945037500033,-0.019198449026275908,-0.005219804415300423,60.0+0.04534098333546186,-0.044641636506989144,-0.006205954135807083,-0.015998975220305175,0.12501870313429186,0.1251981011367534,0.019186997017453092,0.03430885887772673,0.03243232415655107,-0.005219804415300423,219.0+0.0707687524925978,0.05068011873981862,-0.01698407487461508,0.0218723855140367,0.04383748450042574,0.05630543954305571,0.0375951860378878,-0.002592261998183278,-0.07020936123162536,-0.01764612515980379,80.0+-0.074532785548179,0.05068011873981862,0.055229334075398484,-0.04009893205125,0.053469154507833586,0.05317395492516036,-0.04340084565202491,0.07120997975363674,0.06123762840403217,-0.03421455281914162,68.0+0.059871137139539544,0.05068011873981862,0.07678557555301448,0.02531523648988596,0.001182945896190995,0.016848733357574308,-0.05444575906428573,0.03430885887772673,0.02993464904142137,0.044485478562713045,332.0+0.07440129094361722,-0.044641636506989144,0.018583723563451313,0.06318659722422783,0.06172487165704031,0.042840055686105716,0.008142083605192267,-0.002592261998183278,0.05803805188793539,-0.05906719430814835,248.0+0.009015598825267658,-0.044641636506989144,-0.022373135244019075,-0.03207708927752123,-0.04972730985725048,-0.06864079671096873,0.07809320188284416,-0.0708593356186168,-0.06291687914365544,-0.03835665973397607,84.0+-0.07090024709715959,-0.044641636506989144,0.09295275666122646,0.012679973408519169,0.020446285911006685,0.04252690722431616,0.0007788079970183853,0.0003598276718895252,-0.05453964034510003,-0.0010776975004659671,200.0+0.02354575262934534,0.05068011873981862,-0.03099563183506548,-0.00567042229275739,-0.016704441260423575,0.017788178742942903,-0.03235593223976409,-0.002592261998183278,-0.07409260794346935,-0.03421455281914162,55.0+-0.052737554842062495,0.05068011873981862,0.039062152967186486,-0.04009893205125,-0.005696818394814609,-0.012900370512431508,0.01182372140927921,-0.03949338287409329,0.016306823139527554,0.0030644094143684884,85.0+0.06713621404157838,-0.044641636506989144,-0.061174369903727877,-0.04009893205125,-0.026336111267831423,-0.02448686359864431,0.033913548233800855,-0.03949338287409329,-0.05615310200706333,-0.05906719430814835,89.0+0.001750521923228816,-0.044641636506989144,-0.008361578283568675,-0.06419888888219483,-0.038719686991641515,-0.02448686359864431,0.0044604458011053266,-0.03949338287409329,-0.06468530604998815,-0.05492508739331389,31.0+0.02354575262934534,0.05068011873981862,-0.03746250427835029,-0.04698463400294853,-0.0910058956032841,-0.07553006287033849,-0.03235593223976409,-0.03949338287409329,-0.030747917533098208,-0.013504018244969336,129.0+0.038075906433423026,0.05068011873981862,-0.013750638652972673,-0.015998975220305175,-0.035967781275239266,-0.021981675904328014,-0.01394774321932938,-0.002592261998183278,-0.025953110560258,-0.0010776975004659671,83.0+0.016280675727306498,-0.044641636506989144,0.0735521393313721,-0.04123507287328025,-0.004320865536613489,-0.013526667436010586,-0.01394774321932938,-0.0011162171631468765,0.04289703595278786,0.044485478562713045,275.0+-0.0018820165277906047,0.05068011873981862,-0.02452875939178067,0.052858044296680055,0.027326050202012293,0.03000096875273476,0.030231910429713918,-0.002592261998183278,-0.021395309255276825,0.036201264733044136,65.0+0.012648137276287077,-0.044641636506989144,0.03367309259778249,0.03333707926361473,0.030077955918414535,0.02718263259662897,-0.002902829807068556,0.00884708547334881,0.031192602201596156,0.027917050903375224,198.0+0.07440129094361722,-0.044641636506989144,0.03475090467166331,0.0941722560068712,0.05759701308243695,0.020293366437259194,0.022868634821540033,-0.002592261998183278,0.07379892880056028,-0.021788232074638245,236.0+0.04170844488444244,0.05068011873981862,-0.03854031635223107,0.052858044296680055,0.07686035309725264,0.11642994420664642,-0.03971920784793797,0.07120997975363674,-0.022516528376302174,-0.013504018244969336,253.0+-0.009147093429829445,0.05068011873981862,-0.039618128426111884,-0.04009893205125,-0.008448724111216851,0.01622243643399523,-0.06549067247654655,0.07120997975363674,0.017765319557121535,-0.06735140813781726,124.0+0.009015598825267658,0.05068011873981862,-0.0018947058402839008,0.0218723855140367,-0.038719686991641515,-0.02480001206043385,-0.006584467611155497,-0.03949338287409329,-0.03980882652740082,-0.013504018244969336,44.0+0.06713621404157838,0.05068011873981862,-0.03099563183506548,0.004658130634790395,0.024574144485610048,0.03563764106494638,-0.028674294435677143,0.03430885887772673,0.02337141516224845,0.08176444079622315,172.0+0.001750521923228816,-0.044641636506989144,-0.04608500086939666,-0.03321323009955148,-0.07311850844666953,-0.08147988364433965,0.04495846164606168,-0.06938329078358041,-0.061175799045152635,-0.07977772888232063,114.0+-0.009147093429829445,0.05068011873981862,0.0013387303813585058,-0.002227571316908129,0.07961225881365488,0.07008397186179521,0.033913548233800855,-0.002592261998183278,0.02671684132010462,0.08176444079622315,142.0+-0.005514554978810025,-0.044641636506989144,0.06492964274032566,0.03564378941743375,-0.001568959820211247,0.014969842586837095,-0.01394774321932938,0.0007288388806486174,-0.018113692315690322,0.03205915781820968,109.0+0.09619652164973376,-0.044641636506989144,0.04013996504106731,-0.057313186930496314,0.045213437358626866,0.06068951800810917,-0.02131101882750326,0.03615391492152222,0.012551194864223063,0.02377494398854077,180.0+-0.074532785548179,-0.044641636506989144,-0.023450947317899894,-0.00567042229275739,-0.02083229983502694,-0.014152964359589643,0.01550535921336615,-0.03949338287409329,-0.03845971734112638,-0.03007244590430716,144.0+0.059871137139539544,0.05068011873981862,0.053073709927636895,0.052858044296680055,0.03282986163481677,0.019667069513680118,-0.010266105415242439,0.03430885887772673,0.055203099476237055,-0.0010776975004659671,163.0+-0.02367724723390713,-0.044641636506989144,0.04013996504106731,-0.012556124244455912,-0.009824676969417972,-0.0010007289644291908,-0.002902829807068556,-0.002592261998183278,-0.011896851335695978,-0.03835665973397607,147.0+0.009015598825267658,-0.044641636506989144,-0.020217511096257485,-0.05387033595464705,0.03145390877661565,0.020606514899048713,0.05600337505832251,-0.03949338287409329,-0.010903250651210127,-0.0010776975004659671,97.0+0.016280675727306498,0.05068011873981862,0.014272475267928093,0.0012152796589411327,0.001182945896190995,-0.02135537898074896,-0.03235593223976409,0.03430885887772673,0.0749657259346355,0.040343371647878594,220.0+0.01991321417832592,-0.044641636506989144,-0.03422906805670789,0.05516475445049906,0.0672286830898448,0.07415490186505921,-0.006584467611155497,0.032832814042690325,0.024729639951132837,0.06933812005171978,190.0+0.0889314447476949,-0.044641636506989144,0.006727790750762504,0.02531523648988596,0.030077955918414535,0.008706873351046395,0.06336665066649638,-0.03949338287409329,0.009433658771615987,0.03205915781820968,109.0+0.01991321417832592,-0.044641636506989144,0.004572166603000912,0.04597234234498153,-0.018080394118624697,-0.054549115930439665,0.06336665066649638,-0.03949338287409329,0.028658464676026615,0.06105390622205087,191.0+-0.02367724723390713,-0.044641636506989144,0.030439656376140087,-0.00567042229275739,0.08236416453005713,0.09200436418706266,-0.01762938102341632,0.07120997975363674,0.0330430695314185,0.0030644094143684884,122.0+0.09619652164973376,-0.044641636506989144,0.05199589785375607,0.0792647112814439,0.05484510736603471,0.036577086450315016,-0.07653558588880739,0.14132210941786577,0.0986480615153178,0.06105390622205087,230.0+0.02354575262934534,0.05068011873981862,0.061696206518683294,0.06205045640219759,0.024574144485610048,-0.03607335668485709,-0.09126213710515514,0.15534453535071155,0.13339673866449434,0.08176444079622315,242.0+0.0707687524925978,0.05068011873981862,-0.007283766209687899,0.049415193320830796,0.06034891879883919,-0.004445362044114079,-0.05444575906428573,0.10811110062954676,0.12902124941171242,0.05691179930721642,248.0+0.03081082953138418,-0.044641636506989144,0.005649978676881689,0.011543832586488917,0.07823630595545376,0.0779126834065336,-0.04340084565202491,0.10811110062954676,0.06605066658209227,0.019632837073706312,249.0+-0.0018820165277906047,-0.044641636506989144,0.05415152200151766,-0.06650559903601384,0.07273249452264928,0.056618588004845226,-0.04340084565202491,0.08486339447772344,0.08449153066204618,0.0486275854775475,192.0+0.04534098333546186,0.05068011873981862,-0.008361578283568675,-0.03321323009955148,-0.007072771253015731,0.0011913102680975625,-0.03971920784793797,0.03430885887772673,0.02993464904142137,0.027917050903375224,131.0+0.07440129094361722,-0.044641636506989144,0.11450899813884247,0.028758087465735226,0.024574144485610048,0.02499059336410222,0.019186997017453092,-0.002592261998183278,-0.0006117353045626216,-0.005219804415300423,237.0+-0.03820740103798481,-0.044641636506989144,0.0670852668880873,-0.060756037906345574,-0.029088016984233665,-0.023234269751486174,-0.010266105415242439,-0.002592261998183278,-0.0014959487577289358,0.019632837073706312,78.0+-0.012779631880848867,0.05068011873981862,-0.05578530953432388,-0.002227571316908129,-0.027712064126032544,-0.02918409052548733,0.019186997017453092,-0.03949338287409329,-0.017056282412934727,0.044485478562713045,135.0+0.009015598825267658,0.05068011873981862,0.030439656376140087,0.04252949136913227,-0.0029449126784123676,0.03689023491210454,-0.06549067247654655,0.07120997975363674,-0.02364686309993755,0.015490730158871856,244.0+0.08166636784565606,0.05068011873981862,-0.02560657146566148,-0.03665608107540074,-0.07036660273026729,-0.046407255923911754,-0.03971920784793797,-0.002592261998183278,-0.041176166918895155,-0.005219804415300423,199.0+0.03081082953138418,-0.044641636506989144,0.10480868947391528,0.07695800112762488,-0.011200629827619093,-0.011334628203483833,-0.05812739686837268,0.03430885887772673,0.0571082604217192,0.036201264733044136,270.0+0.027178291080364757,0.05068011873981862,-0.006205954135807083,0.028758087465735226,-0.016704441260423575,-0.0016270258880082473,-0.05812739686837268,0.03430885887772673,0.02929655685872395,0.03205915781820968,164.0+-0.06000263174410134,0.05068011873981862,-0.047162812943277475,-0.0228846771720037,-0.07174255558846841,-0.05768060054833501,-0.006584467611155497,-0.03949338287409329,-0.06291687914365544,-0.05492508739331389,72.0+0.005383060374248237,-0.044641636506989144,-0.048240625017158284,-0.012556124244455912,0.001182945896190995,-0.0066374012766408095,0.06336665066649638,-0.03949338287409329,-0.05140387304727299,-0.05906719430814835,96.0+-0.020044708782887707,-0.044641636506989144,0.08540807214406083,-0.03665608107540074,0.09199583453746497,0.08949917649274639,-0.06180903467245962,0.14501222150545676,0.08094556124677081,0.05276969239238195,306.0+0.01991321417832592,0.05068011873981862,-0.012672826579091896,0.07007229917592636,-0.011200629827619093,0.0071411310420987206,-0.03971920784793797,0.03430885887772673,0.005386331212792652,0.0030644094143684884,91.0+-0.06363517019512076,-0.044641636506989144,-0.033151255982827074,-0.03321323009955148,0.001182945896190995,0.024051147978733624,-0.024992656631590206,-0.002592261998183278,-0.022516528376302174,-0.05906719430814835,214.0+0.027178291080364757,-0.044641636506989144,-0.007283766209687899,-0.05042748497879779,0.07548440023905152,0.056618588004845226,0.033913548233800855,-0.002592261998183278,0.04344397210938562,0.015490730158871856,95.0+-0.016412170331868287,-0.044641636506989144,-0.013750638652972673,0.13204361674121307,-0.009824676969417972,-0.0038190651205350003,0.019186997017453092,-0.03949338287409329,-0.0358161925842373,-0.03007244590430716,216.0+0.03081082953138418,0.05068011873981862,0.05954058237092167,0.056300895272529315,-0.02220825269322806,0.0011913102680975625,-0.03235593223976409,-0.002592261998183278,-0.024795429028792802,-0.01764612515980379,263.0+0.056238598688520124,0.05068011873981862,0.021817159785093684,0.056300895272529315,-0.007072771253015731,0.018101327204732443,-0.03235593223976409,-0.002592261998183278,-0.02364686309993755,0.02377494398854077,178.0+-0.020044708782887707,-0.044641636506989144,0.018583723563451313,0.09072940503102193,0.003934851612593237,0.008706873351046395,0.0375951860378878,-0.03949338287409329,-0.05780302607946657,0.007206516329202944,113.0+-0.1072256316073538,-0.044641636506989144,-0.011595014505211082,-0.04009893205125,0.04934129593323023,0.0644472995495836,-0.01394774321932938,0.03430885887772673,0.007027139682585861,-0.03007244590430716,200.0+0.08166636784565606,0.05068011873981862,-0.002972517914164677,-0.03321323009955148,0.04246153164222462,0.057871181852003385,-0.010266105415242439,0.03430885887772673,-0.0006117353045626216,-0.0010776975004659671,139.0+0.005383060374248237,0.05068011873981862,0.0175059114895705,0.03220093844158448,0.1277706088506941,0.12739014036928015,-0.02131101882750326,0.07120997975363674,0.06257762198769659,0.015490730158871856,139.0+0.038075906433423026,0.05068011873981862,-0.029917819761184662,-0.07452744180974262,-0.012576582685820214,-0.012587222050641968,0.0044604458011053266,-0.002592261998183278,0.0037090603325595967,-0.03007244590430716,88.0+0.03081082953138418,-0.044641636506989144,-0.020217511096257485,-0.00567042229275739,-0.004320865536613489,-0.02949723898727687,0.07809320188284416,-0.03949338287409329,-0.010903250651210127,-0.0010776975004659671,148.0+0.001750521923228816,0.05068011873981862,-0.05794093368208547,-0.04354178302709927,-0.09650970703608859,-0.047033552847490806,-0.09862541271332903,0.03430885887772673,-0.061175799045152635,-0.07149351505265171,88.0+-0.027309785684926546,0.05068011873981862,0.06061839444480248,0.10794365991026823,0.01219056876179996,-0.017597597439274533,-0.002902829807068556,-0.002592261998183278,0.07020738137223513,0.13561183068907107,243.0+-0.08543040090123728,0.05068011873981862,-0.040695940499992665,-0.03321323009955148,-0.08137422559587626,-0.06958024209633733,-0.006584467611155497,-0.03949338287409329,-0.05780302607946657,-0.042498766648810526,71.0+0.012648137276287077,0.05068011873981862,-0.07195249064253588,-0.04698463400294853,-0.051103262715451604,-0.09713730673381639,0.1185912177278005,-0.0763945037500033,-0.020292321339471356,-0.03835665973397607,77.0+-0.052737554842062495,-0.044641636506989144,-0.05578530953432388,-0.03665608107540074,0.08924392882106273,-0.003192768196955922,0.008142083605192267,0.03430885887772673,0.1323757911721786,0.0030644094143684884,109.0+-0.02367724723390713,0.05068011873981862,0.045529025410471304,0.0218723855140367,0.10988322169407955,0.08887287956916731,0.0007788079970183853,0.03430885887772673,0.07419089971278872,0.06105390622205087,272.0+-0.074532785548179,0.05068011873981862,-0.00943939035744949,0.014986683562338177,-0.037343734133440394,-0.0216685274425385,-0.01394774321932938,-0.002592261998183278,-0.03324559264822791,0.0113486232440374,60.0+-0.005514554978810025,0.05068011873981862,-0.033151255982827074,-0.015998975220305175,0.0080627101871966,0.01622243643399523,0.01550535921336615,-0.002592261998183278,-0.028323167238848198,-0.07563562196748617,54.0+-0.06000263174410134,0.05068011873981862,0.04984027370599448,0.01842953453818744,-0.016704441260423575,-0.03012353591085593,-0.01762938102341632,-0.002592261998183278,0.04977020032069951,-0.05906719430814835,221.0+-0.020044708782887707,-0.044641636506989144,-0.08488623552910546,-0.02632752814785296,-0.035967781275239266,-0.03419446591411989,0.04127682384197474,-0.05167075276314359,-0.08237869071592514,-0.04664087356364498,90.0+0.038075906433423026,0.05068011873981862,0.005649978676881689,0.03220093844158448,0.006686757328995478,0.017475030281153364,-0.024992656631590206,0.03430885887772673,0.014820979914103747,0.06105390622205087,311.0+0.016280675727306498,-0.044641636506989144,0.020739347711212906,0.0218723855140367,-0.013952535544021335,-0.013213518974221048,-0.006584467611155497,-0.002592261998183278,0.013316905483459898,0.040343371647878594,281.0+0.04170844488444244,-0.044641636506989144,-0.007283766209687899,0.028758087465735226,-0.04284754556624487,-0.048286146694648965,0.05232173725423556,-0.0763945037500033,-0.07213275338232743,0.02377494398854077,182.0+0.01991321417832592,0.05068011873981862,0.10480868947391528,0.07007229917592636,-0.035967781275239266,-0.02667890283117104,-0.024992656631590206,-0.002592261998183278,0.0037090603325595967,0.040343371647878594,321.0+-0.04910501639104307,0.05068011873981862,-0.02452875939178067,7.913883691088233e-05,-0.046975404140848234,-0.02824464514011871,-0.06549067247654655,0.028404679537581124,0.019196469166885697,0.0113486232440374,58.0+0.001750521923228816,0.05068011873981862,-0.006205954135807083,-0.019441826196154435,-0.009824676969417972,0.004949091809571968,-0.03971920784793797,0.03430885887772673,0.014820979914103747,0.09833286845556097,262.0+0.0344433679824036,-0.044641636506989144,-0.03854031635223107,-0.012556124244455912,0.00943866304539772,0.0052622402713615075,-0.006584467611155497,-0.002592261998183278,0.031192602201596156,0.09833286845556097,206.0+-0.045472477940023646,0.05068011873981862,0.13714305169033927,-0.015998975220305175,0.041085578784023497,0.03187985952347199,-0.04340084565202491,0.07120997975363674,0.07101867000452176,0.0486275854775475,233.0+-0.009147093429829445,0.05068011873981862,0.17055522598064407,0.014986683562338177,0.030077955918414535,0.03375875029420919,-0.02131101882750326,0.03430885887772673,0.033653814906286016,0.03205915781820968,242.0+-0.016412170331868287,0.05068011873981862,0.002416542455239321,0.014986683562338177,0.02182223876920781,-0.010082034356325698,-0.024992656631590206,0.03430885887772673,0.08553070935958189,0.08176444079622315,123.0+-0.009147093429829445,-0.044641636506989144,0.03798434089330568,-0.04009893205125,-0.0249601584096303,-0.0038190651205350003,-0.04340084565202491,0.01585829843977173,-0.005142189801713891,0.027917050903375224,167.0+0.01991321417832592,-0.044641636506989144,-0.05794093368208547,-0.057313186930496314,-0.001568959820211247,-0.012587222050641968,0.07441156407875721,-0.03949338287409329,-0.061175799045152635,-0.07563562196748617,63.0+0.0526060602375007,0.05068011873981862,-0.00943939035744949,0.049415193320830796,0.05071724879143135,-0.019163339748222204,-0.01394774321932938,0.03430885887772673,0.11934047943993234,-0.01764612515980379,197.0+-0.027309785684926546,0.05068011873981862,-0.023450947317899894,-0.015998975220305175,0.013566521620001083,0.012777803354310339,0.026550272625626974,-0.002592261998183278,-0.010903250651210127,-0.021788232074638245,71.0+-0.074532785548179,-0.044641636506989144,-0.010517202431330305,-0.00567042229275739,-0.06623874415566393,-0.05705430362475593,-0.002902829807068556,-0.03949338287409329,-0.042570854118219384,-0.0010776975004659671,168.0+-0.1072256316073538,-0.044641636506989144,-0.03422906805670789,-0.06764173985804409,-0.06348683843926169,-0.07051968748170592,0.008142083605192267,-0.03949338287409329,-0.0006117353045626216,-0.07977772888232063,140.0+0.04534098333546186,0.05068011873981862,-0.002972517914164677,0.10794365991026823,0.03558176735121902,0.02248540566978595,0.026550272625626974,-0.002592261998183278,0.02802037249332928,0.019632837073706312,217.0+-0.0018820165277906047,-0.044641636506989144,0.0681630789619681,-0.00567042229275739,0.11951489170148738,0.13020847652538595,-0.024992656631590206,0.08670845052151895,0.04613307487932449,-0.0010776975004659671,121.0+0.01991321417832592,0.05068011873981862,0.009961226972404908,0.01842953453818744,0.014942474478202204,0.04471894645684291,-0.06180903467245962,0.07120997975363674,0.009433658771615987,-0.0632093012229828,235.0+0.016280675727306498,0.05068011873981862,0.002416542455239321,-0.00567042229275739,-0.005696818394814609,0.010898912583573148,-0.050764121260198795,0.03430885887772673,0.022687744966501246,-0.03835665973397607,245.0+-0.0018820165277906047,-0.044641636506989144,-0.03854031635223107,0.0218723855140367,-0.10889328275989867,-0.11561306597939897,0.022868634821540033,-0.0763945037500033,-0.04688253415273158,0.02377494398854077,40.0+0.016280675727306498,-0.044641636506989144,0.026128408080616904,0.05860760542634833,-0.060734932722859444,-0.044215216691385,-0.01394774321932938,-0.03395821474270679,-0.05140387304727299,-0.025930338989472702,52.0+-0.07090024709715959,0.05068011873981862,-0.08919748382462865,-0.07452744180974262,-0.04284754556624487,-0.02573945744580244,-0.03235593223976409,-0.002592261998183278,-0.012908683225401873,-0.05492508739331389,104.0+0.04897352178648128,-0.044641636506989144,0.06061839444480248,-0.0228846771720037,-0.02358420555142918,-0.07271172671423268,-0.04340084565202491,-0.002592261998183278,0.10413565428651514,0.036201264733044136,132.0+0.005383060374248237,0.05068011873981862,-0.028840007687303888,-0.009113273268606652,-0.0318399227006359,-0.02887094206369779,0.008142083605192267,-0.03949338287409329,-0.018113692315690322,0.007206516329202944,88.0+0.0344433679824036,0.05068011873981862,-0.029917819761184662,0.004658130634790395,0.0933717873956661,0.08699398879843012,0.033913548233800855,-0.002592261998183278,0.024055085357995654,-0.03835665973397607,69.0+0.02354575262934534,0.05068011873981862,-0.01913969902237667,0.049415193320830796,-0.06348683843926169,-0.06112523362801988,0.0044604458011053266,-0.03949338287409329,-0.025953110560258,-0.013504018244969336,219.0+0.01991321417832592,-0.044641636506989144,-0.040695940499992665,-0.015998975220305175,-0.008448724111216851,-0.017597597439274533,0.05232173725423556,-0.03949338287409329,-0.030747917533098208,0.0030644094143684884,72.0+-0.045472477940023646,-0.044641636506989144,0.015350287341808908,-0.07452744180974262,-0.04972730985725048,-0.017284448977484993,-0.028674294435677143,-0.002592261998183278,-0.10436552421115437,-0.07563562196748617,201.0+0.0526060602375007,0.05068011873981862,-0.02452875939178067,0.056300895272529315,-0.007072771253015731,-0.005071658967693135,-0.02131101882750326,-0.002592261998183278,0.02671684132010462,-0.03835665973397607,110.0+-0.005514554978810025,0.05068011873981862,0.0013387303813585058,-0.0848559947372904,-0.011200629827619093,-0.016658152053905938,0.04864009945014862,-0.03949338287409329,-0.041176166918895155,-0.08806194271198954,51.0+0.009015598825267658,0.05068011873981862,0.06924089103584885,0.059743746248378575,0.017694380194604446,-0.023234269751486174,-0.04708248345611185,0.03430885887772673,0.10329701884639861,0.07348022696655424,277.0+-0.02367724723390713,-0.044641636506989144,-0.06979686649477428,-0.06419888888219483,-0.05935897986465832,-0.05047818592717569,0.019186997017453092,-0.03949338287409329,-0.08913335224990723,-0.05078298047847944,63.0+-0.04183993948900423,0.05068011873981862,-0.029917819761184662,-0.002227571316908129,0.02182223876920781,0.036577086450315016,0.01182372140927921,-0.002592261998183278,-0.041176166918895155,0.06519601313688532,118.0+-0.074532785548179,-0.044641636506989144,-0.04608500086939666,-0.04354178302709927,-0.029088016984233665,-0.023234269751486174,0.01550535921336615,-0.03949338287409329,-0.03980882652740082,-0.021788232074638245,69.0+0.0344433679824036,-0.044641636506989144,0.018583723563451313,0.056300895272529315,0.01219056876179996,-0.054549115930439665,-0.0691723102806335,0.07120997975363674,0.13007865931446802,0.007206516329202944,273.0+-0.06000263174410134,-0.044641636506989144,0.0013387303813585058,-0.029770379123702218,-0.007072771253015731,-0.0216685274425385,0.01182372140927921,-0.002592261998183278,0.031812463179073616,-0.05492508739331389,258.0+-0.08543040090123728,0.05068011873981862,-0.03099563183506548,-0.0228846771720037,-0.06348683843926169,-0.05423596746865012,0.019186997017453092,-0.03949338287409329,-0.09643494994048712,-0.03421455281914162,43.0+0.0526060602375007,-0.044641636506989144,-0.004050329988045492,-0.03090651994573247,-0.046975404140848234,-0.05830689747191407,-0.01394774321932938,-0.02583996815000658,0.036060333995316066,0.02377494398854077,198.0+0.012648137276287077,-0.044641636506989144,0.015350287341808908,-0.03321323009955148,0.041085578784023497,0.032193007985261514,-0.002902829807068556,-0.002592261998183278,0.04506654937395887,-0.06735140813781726,242.0+0.059871137139539544,0.05068011873981862,0.022894971858974496,0.049415193320830796,0.016318427336403322,0.011838357968941744,-0.01394774321932938,-0.002592261998183278,0.03954249419232167,0.019632837073706312,232.0+-0.02367724723390713,-0.044641636506989144,0.045529025410471304,0.09072940503102193,-0.018080394118624697,-0.03544705976127803,0.07072992627467027,-0.03949338287409329,-0.03452177701362266,-0.009361911330134878,175.0+0.016280675727306498,-0.044641636506989144,-0.04500718879551588,-0.057313186930496314,-0.034591828417038145,-0.053922819006860585,0.07441156407875721,-0.0763945037500033,-0.042570854118219384,0.040343371647878594,93.0+0.11072667545381144,0.05068011873981862,-0.033151255982827074,-0.0228846771720037,-0.004320865536613489,0.020293366437259194,-0.06180903467245962,0.07120997975363674,0.015568459328120622,0.044485478562713045,168.0+-0.020044708782887707,-0.044641636506989144,0.09726400495674965,-0.00567042229275739,-0.005696818394814609,-0.02386056667506523,-0.02131101882750326,-0.002592261998183278,0.06168429293192034,0.040343371647878594,275.0+-0.016412170331868287,-0.044641636506989144,0.05415152200151766,0.07007229917592636,-0.033215875558837024,-0.0279314966783292,0.008142083605192267,-0.03949338287409329,-0.02712902329694316,-0.009361911330134878,293.0+0.04897352178648128,0.05068011873981862,0.12313149472988882,0.08384370307932341,-0.10476542418529532,-0.10089508827529081,-0.0691723102806335,-0.002592261998183278,0.03664373256235367,-0.03007244590430716,281.0+-0.056370093293081916,-0.044641636506989144,-0.08057498723358228,-0.0848559947372904,-0.037343734133440394,-0.037012802070225705,0.033913548233800855,-0.03949338287409329,-0.05615310200706333,-0.13776722569000302,72.0+0.027178291080364757,-0.044641636506989144,0.09295275666122646,-0.0527341951326168,0.0080627101871966,0.039708571068210366,-0.028674294435677143,0.021024455362399115,-0.04835926177554553,0.019632837073706312,140.0+0.06350367559055897,-0.044641636506989144,-0.050396249164919873,0.10794365991026823,0.03145390877661565,0.019353921051890578,-0.01762938102341632,0.02360753382371283,0.05803805188793539,0.040343371647878594,189.0+-0.052737554842062495,0.05068011873981862,-0.011595014505211082,0.056300895272529315,0.05622106022423583,0.07290230801790105,-0.03971920784793797,0.07120997975363674,0.030563625621508765,-0.005219804415300423,181.0+-0.009147093429829445,0.05068011873981862,-0.027762195613423073,0.008100981610639655,0.04796534307502911,0.037203383373894054,-0.028674294435677143,0.03430885887772673,0.06605066658209227,-0.042498766648810526,209.0+0.005383060374248237,-0.044641636506989144,0.05846277029704089,-0.04354178302709927,-0.07311850844666953,-0.07239857825244314,0.019186997017453092,-0.0763945037500033,-0.05140387304727299,-0.025930338989472702,136.0+0.07440129094361722,-0.044641636506989144,0.08540807214406083,0.06318659722422783,0.014942474478202204,0.013090951816099879,0.01550535921336615,-0.002592261998183278,0.006206735447689297,0.0859065477110576,261.0+-0.052737554842062495,-0.044641636506989144,-0.0008168937664030856,-0.02632752814785296,0.010814615903598841,0.0071411310420987206,0.04864009945014862,-0.03949338287409329,-0.0358161925842373,0.019632837073706312,113.0+0.08166636784565606,0.05068011873981862,0.006727790750762504,-0.00453428147072714,0.10988322169407955,0.11705624113022545,-0.03235593223976409,0.09187460744414634,0.05471997253790904,0.007206516329202944,131.0+-0.005514554978810025,-0.044641636506989144,0.008883414898524095,-0.05042748497879779,0.02595009734381117,0.04722413415115918,-0.04340084565202491,0.07120997975363674,0.014820979914103747,0.0030644094143684884,174.0+-0.027309785684926546,-0.044641636506989144,0.08001901177465684,0.0987512478047507,-0.0029449126784123676,0.018101327204732443,-0.01762938102341632,0.003311917341962329,-0.02952642678336326,0.036201264733044136,257.0+-0.052737554842062495,-0.044641636506989144,0.07139651518361048,-0.07452744180974262,-0.015328488402222454,-0.0013138774262187302,0.0044604458011053266,-0.021411833644897377,-0.04688253415273158,0.0030644094143684884,55.0+0.009015598825267658,-0.044641636506989144,-0.02452875939178067,-0.02632752814785296,0.09887559882847057,0.0941964034195894,0.07072992627467027,-0.002592261998183278,-0.021395309255276825,0.007206516329202944,84.0+-0.020044708782887707,-0.044641636506989144,-0.05470749746044306,-0.05387033595464705,-0.06623874415566393,-0.05736745208654548,0.01182372140927921,-0.03949338287409329,-0.07409260794346935,-0.005219804415300423,42.0+0.02354575262934534,-0.044641636506989144,-0.03638469220446948,7.913883691088233e-05,0.001182945896190995,0.034698195679577784,-0.04340084565202491,0.03430885887772673,-0.03324559264822791,0.06105390622205087,146.0+0.038075906433423026,0.05068011873981862,0.016428099415689682,0.0218723855140367,0.039709625925822375,0.04503209491863242,-0.04340084565202491,0.07120997975363674,0.04977020032069951,0.015490730158871856,212.0+-0.07816532399919843,0.05068011873981862,0.07786338762689529,0.052858044296680055,0.07823630595545376,0.0644472995495836,0.026550272625626974,-0.002592261998183278,0.04067282891595704,-0.009361911330134878,233.0+0.009015598825267658,0.05068011873981862,-0.039618128426111884,0.028758087465735226,0.03833367306762126,0.07352860494148013,-0.07285394808472044,0.10811110062954676,0.015568459328120622,-0.04664087356364498,91.0+0.001750521923228816,0.05068011873981862,0.011039039046285686,-0.019441826196154435,-0.016704441260423575,-0.0038190651205350003,-0.04708248345611185,0.03430885887772673,0.024055085357995654,0.02377494398854077,111.0+-0.07816532399919843,-0.044641636506989144,-0.040695940499992665,-0.08141314376144114,-0.10063756561069194,-0.11279472982329314,0.022868634821540033,-0.0763945037500033,-0.020292321339471356,-0.05078298047847944,152.0+0.03081082953138418,0.05068011873981862,-0.03422906805670789,0.04366563219116252,0.05759701308243695,0.0688313780146371,-0.03235593223976409,0.05755656502955003,0.03545870422305857,0.0859065477110576,120.0+-0.03457486258696539,0.05068011873981862,0.005649978676881689,-0.00567042229275739,-0.07311850844666953,-0.06269097593696756,-0.006584467611155497,-0.03949338287409329,-0.04542403773513769,0.03205915781820968,67.0+0.04897352178648128,0.05068011873981862,0.08864150836570328,0.08728655405517267,0.03558176735121902,0.02154596028441731,-0.024992656631590206,0.03430885887772673,0.06605066658209227,0.13146972377423663,310.0+-0.04183993948900423,-0.044641636506989144,-0.033151255982827074,-0.0228846771720037,0.04658939021682799,0.041587461838947556,0.05600337505832251,-0.024732934523729287,-0.025953110560258,-0.03835665973397607,94.0+-0.009147093429829445,-0.044641636506989144,-0.05686312160820465,-0.05042748497879779,0.02182223876920781,0.04534524338042199,-0.028674294435677143,0.03430885887772673,-0.009918765569334137,-0.01764612515980379,183.0+0.0707687524925978,0.05068011873981862,-0.03099563183506548,0.0218723855140367,-0.037343734133440394,-0.047033552847490806,0.033913548233800855,-0.03949338287409329,-0.014959693812643405,-0.0010776975004659671,66.0+0.009015598825267658,-0.044641636506989144,0.055229334075398484,-0.00567042229275739,0.05759701308243695,0.04471894645684291,-0.002902829807068556,0.023238522614953735,0.05568622641456506,0.1066170822852299,173.0+-0.027309785684926546,-0.044641636506989144,-0.06009655782984706,-0.029770379123702218,0.04658939021682799,0.019980217975469634,0.12227285553188745,-0.03949338287409329,-0.05140387304727299,-0.009361911330134878,72.0+0.016280675727306498,-0.044641636506989144,0.0013387303813585058,0.008100981610639655,0.005310804470794357,0.010898912583573148,0.030231910429713918,-0.03949338287409329,-0.04542403773513769,0.03205915781820968,49.0+-0.012779631880848867,-0.044641636506989144,-0.023450947317899894,-0.04009893205125,-0.016704441260423575,0.00463594334778245,-0.01762938102341632,-0.002592261998183278,-0.03845971734112638,-0.03835665973397607,64.0+-0.056370093293081916,-0.044641636506989144,-0.07410811479029747,-0.05042748497879779,-0.0249601584096303,-0.047033552847490806,0.09281975309919192,-0.0763945037500033,-0.061175799045152635,-0.04664087356364498,48.0+0.04170844488444244,0.05068011873981862,0.01966153563733209,0.059743746248378575,-0.005696818394814609,-0.0025664712733768653,-0.028674294435677143,-0.002592261998183278,0.031192602201596156,0.007206516329202944,178.0+-0.005514554978810025,0.05068011873981862,-0.015906262800734303,-0.06764173985804409,0.04934129593323023,0.07916527725369175,-0.028674294435677143,0.03430885887772673,-0.018113692315690322,0.044485478562713045,104.0+0.04170844488444244,0.05068011873981862,-0.015906262800734303,0.01729339371615719,-0.037343734133440394,-0.013839815897800126,-0.024992656631590206,-0.011079519799642579,-0.04688253415273158,0.015490730158871856,132.0+-0.045472477940023646,-0.044641636506989144,0.039062152967186486,0.0012152796589411327,0.016318427336403322,0.015282991048626631,-0.028674294435677143,0.02655962349378563,0.04452872881997113,-0.025930338989472702,220.0+-0.045472477940023646,-0.044641636506989144,-0.07303030271641665,-0.08141314376144114,0.08374011738825825,0.027808929520208008,0.17381578478910462,-0.03949338287409329,-0.00422151393810765,0.0030644094143684884,57.0

@@ -1,7 +1,6 @@ cabal-version:      3.0 name:               dataframe-version:            2.1.0.3-+version:            2.2.0.0 synopsis: A fast, safe, and intuitive DataFrame library.  description: A fast, safe, and intuitive DataFrame library for exploratory data analysis.@@ -21,6 +20,8 @@                     tests/data/typing/texts_with_empties.txt                     tests/data/typing/texts_with_empties_and_nullish.txt                     data/titanic/*.csv+                    data/ml/*.csv+                    data/ml/golden.json                     data/sharded/*.parquet                     tests/data/*.csv                     tests/data/*.tsv@@ -120,16 +121,16 @@                         DataFrame.Typed.Record,                         DataFrame.Typed.Generic     build-depends:    base >= 4 && <5,-                      dataframe-core ^>= 1.0.2,+                      dataframe-core ^>= 1.1,                       dataframe-json ^>= 1.0,-                      dataframe-operations ^>= 1.1.0.2,-                      dataframe-parsing ^>= 1.0,-                      dataframe-viz ^>= 1.0,-                      dataframe-learn ^>= 1.0+                      dataframe-operations >= 1.1.1 && < 1.2,+                      dataframe-parsing ^>= 1.0.2,+                      dataframe-viz ^>= 1.0.3,+                      dataframe-learn ^>= 1.1      if !flag(no-csv)         reexported-modules: DataFrame.IO.CSV-        build-depends:   dataframe-csv ^>= 1.0+        build-depends:   dataframe-csv ^>= 1.0.2         cpp-options:     -DWITH_CSV      if !flag(no-parquet)@@ -161,7 +162,7 @@                             DataFrame.Lazy.Internal.Optimizer,                             DataFrame.Lazy.Internal.Executor,                             DataFrame.Typed.Lazy-        build-depends:   dataframe-lazy ^>= 1.0+        build-depends:   dataframe-lazy ^>= 1.0.2         cpp-options:     -DWITH_LAZY      if !flag(no-th)@@ -194,13 +195,13 @@         buildable: False     build-depends:         base        >= 4   && < 5,-        dataframe-core ^>= 1.0.2,-        dataframe-csv ^>= 1.0,+        dataframe-core ^>= 1.1,+        dataframe-csv ^>= 1.0.2,         dataframe-json ^>= 1.0,-        dataframe-lazy ^>= 1.0,-        dataframe-operations ^>= 1.1.0.2,+        dataframe-lazy ^>= 1.0.2,+        dataframe-operations >= 1.1.1 && < 1.2,         dataframe-parquet ^>= 1.0.1.1,-        dataframe-parsing ^>= 1.0,+        dataframe-parsing ^>= 1.0.2,         text        >= 2.0 && < 3,         aeson       >= 0.11 && < 3,         bytestring  >= 0.11 && < 0.13,@@ -214,7 +215,7 @@     main-is: Benchmark.hs     build-depends:    base >= 4 && < 5,                       dataframe >= 1 && < 3,-                      dataframe-operations ^>= 1.1,+                      dataframe-operations >= 1.1.1 && < 1.2,                       random >= 1 && < 2,                       time >= 1.12 && < 2,                       vector ^>= 0.13,@@ -227,9 +228,9 @@     main-is: Synthesis.hs     build-depends:    base >= 4 && < 5,                       dataframe >= 1 && < 3,-                      dataframe-core ^>= 1.0,-                      dataframe-learn ^>= 1.0,-                      dataframe-operations ^>= 1.1,+                      dataframe-core ^>= 1.1,+                      dataframe-learn ^>= 1.1,+                      dataframe-operations >= 1.1.1 && < 1.2,                       random >= 1 && < 2,                       text >= 2.0 && < 3     hs-source-dirs:   app@@ -261,9 +262,9 @@                       bytestring >= 0.11 && < 0.13,                       containers >= 0.6.7 && < 0.9,                       dataframe >= 1 && < 3,-                      dataframe-core ^>= 1.0,-                      dataframe-lazy ^>= 1.0,-                      dataframe-parsing ^>= 1.0,+                      dataframe-core ^>= 1.1,+                      dataframe-lazy ^>= 1.0.2,+                      dataframe-parsing ^>= 1.0.2,                       directory >= 1.3.0.0 && < 2,                       random >= 1 && < 2,                       text >= 2.0 && < 3,@@ -293,6 +294,9 @@     import: warnings     type: exitcode-stdio-1.0     main-is: Main.hs+    -- Threaded RTS with -N so the parallel-grouping parity test exercises the+    -- real multi-capability fork path rather than the sequential fallback.+    ghc-options: -threaded -rtsopts -with-rtsopts=-N     -- The test runner imports CSV, JSON, Parquet, Lazy, and TH-derived     -- schema modules. All features must be enabled for it to compile.     if flag(no-csv) || flag(no-parquet) || flag(no-th)@@ -302,8 +306,24 @@                    DecisionTree,                    Functions,                    GenDataFrame,+                   Internal.ColumnBuilder,+                   Internal.DictEncode,+                   Internal.PackedText,                    Internal.Parsing,+                   PackedTextMigration,+                   Learn.Numerics,+                   Learn.Denotation,+                   Learn.Models,+                   Learn.Ensembles,+                   Learn.Symbolic,+                   Learn.SklearnParity,+                   Learn.Synthesis,+                   Learn.MetricsTests,+                   Learn.Metamorphic,+                   Learn.EdgeCases,+                   Learn.NumericalRigor,                    IO.CSV,+                   IO.CsvGolden,                    IO.JSON,                    LinearSolver,                    Operations.Aggregations,@@ -312,6 +332,9 @@                    Operations.Derive,                    Operations.Filter,                    Operations.GroupBy,+                   Operations.ParallelGroupBy,+                   Operations.ParallelJoin,+                   Operations.Inference,                    Operations.InsertColumn,                    Operations.Join,                    Operations.Merge,@@ -328,8 +351,10 @@                    Operations.Statistics,                    Operations.Take,                    Operations.Typing,+                   Operations.VectorKernel,                    Operations.Record,                    LazyParquet,+                   LazyParity,                    Parquet,                    ParquetTestData,                    Plotting,@@ -344,14 +369,14 @@                     aeson >= 0.11.0.0 && < 3,                     bytestring >= 0.11 && < 0.13,                     dataframe >= 1 && < 3,-                    dataframe-core ^>= 1.0,-                    dataframe-csv ^>= 1.0,+                    dataframe-core ^>= 1.1,+                    dataframe-csv ^>= 1.0.2,                     dataframe-json ^>= 1.0,-                    dataframe-lazy ^>= 1.0,-                    dataframe-learn ^>= 1.0,-                    dataframe-operations ^>= 1.1,+                    dataframe-lazy ^>= 1.0.2,+                    dataframe-learn ^>= 1.1,+                    dataframe-operations >= 1.1.1 && < 1.2,                     dataframe-parquet ^>= 1.0,-                    dataframe-parsing ^>= 1.0,+                    dataframe-parsing ^>= 1.0.2,                     HUnit ^>= 1.6,                     QuickCheck >= 2 && < 3,                     random-shuffle >= 0.0.4 && < 1,@@ -359,6 +384,28 @@                     text >= 2.0 && < 3,                     time >= 1.12 && < 2,                     vector ^>= 0.13,+                    temporary >= 1.3 && < 1.4,+                    directory >= 1.3 && < 1.4,                     containers >= 0.6.7 && < 0.9+    hs-source-dirs: tests+    default-language: Haskell2010++-- Focused oracle for the packed-text column variant. Kept as its own suite so+-- it can run independently of the (heavier) `tests` runner.+test-suite packed-text+    import: warnings+    type: exitcode-stdio-1.0+    main-is: PackedTextMain.hs+    if flag(no-csv) || flag(no-parquet) || flag(no-th)+        buildable: False+    other-modules: Internal.PackedText+    build-depends:  base >= 4 && < 5,+                    bytestring >= 0.11 && < 0.13,+                    dataframe >= 1 && < 3,+                    dataframe-core ^>= 1.1,+                    dataframe-operations >= 1.1.1 && < 1.2,+                    HUnit ^>= 1.6,+                    text >= 2.0 && < 3,+                    vector ^>= 0.13     hs-source-dirs: tests     default-language: Haskell2010

@@ -258,6 +258,7 @@     columnTypeRep :: Column -> SomeTypeRep     columnTypeRep (UnboxedColumn _ (_ :: VU.Vector a)) = SomeTypeRep (typeRep @a)     columnTypeRep (BoxedColumn _ (_ :: V.Vector a)) = SomeTypeRep (typeRep @a)+    columnTypeRep (PackedText _ _) = SomeTypeRep (typeRep @T.Text)     mk :: (Columnable a, Ord a) => Expr a -> SortOrder     mk = if isAsc then Asc else Desc     dispatchType (SomeTypeRep tr)@@ -335,6 +336,7 @@     columnTypeRep :: Column -> SomeTypeRep     columnTypeRep (UnboxedColumn _ (_ :: VU.Vector a)) = SomeTypeRep (typeRep @a)     columnTypeRep (BoxedColumn _ (_ :: V.Vector a)) = SomeTypeRep (typeRep @a)+    columnTypeRep (PackedText _ _) = SomeTypeRep (typeRep @T.Text)      fr :: forall a. (Columnable a, Ord a) => IO DataFrame     fr = return $ Stats.frequencies (Col @a colName) df@@ -369,6 +371,8 @@ countExpr name colName (UnboxedColumn (Just _) (_ :: VU.Vector a)) = return $ name .= count (Col @(Maybe a) colName) countExpr name colName (BoxedColumn Nothing (_ :: V.Vector a)) = return $ name .= count (Col @a colName) countExpr name colName (BoxedColumn (Just _) (_ :: V.Vector a)) = return $ name .= count (Col @(Maybe a) colName)+countExpr name colName (PackedText Nothing _) = return $ name .= count (Col @T.Text colName)+countExpr name colName (PackedText (Just _) _) = return $ name .= count (Col @(Maybe T.Text) colName)  sumExpr :: T.Text -> T.Text -> Column -> IO NamedExpr sumExpr name colName (UnboxedColumn Nothing (_ :: VU.Vector a))

@@ -0,0 +1,560 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++{- | Golden semantics for the default CSV reader, pinned against the+cassava-based implementation (oracle runs of 2026-06-12) before the+strict-scanner rewrite. Ragged-row cases encode the new pad-with-null+behavior (audit D6) — the one intentional change.+-}+module IO.CsvGolden (tests) where++import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BL+import qualified Data.Map.Strict as M+import qualified Data.Text as T++import Control.Exception (SomeException, evaluate, try)+import Data.List (isInfixOf)+import Data.Time.Calendar (Day, fromGregorian)+import DataFrame.IO.CSV+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.DataFrame (+    columnIndices,+    dataframeDimensions,+    forceDataFrame,+    getColumn,+ )+import DataFrame.Internal.Schema (schemaType)+import DataFrame.Operations.Typing (SafeReadMode (..))+import Test.HUnit++data Expect+    = Cols (Int, Int) [(T.Text, DI.Column)]+    | Err String++ints :: [Int] -> DI.Column+ints = DI.fromList+mints :: [Maybe Int] -> DI.Column+mints = DI.fromList+dbls :: [Double] -> DI.Column+dbls = DI.fromList+texts :: [T.Text] -> DI.Column+texts = DI.fromList+mtexts :: [Maybe T.Text] -> DI.Column+mtexts = DI.fromList+eti :: [Either T.Text Int] -> DI.Column+eti = DI.fromList+ett :: [Either T.Text T.Text] -> DI.Column+ett = DI.fromList+days :: [Day] -> DI.Column+days = DI.fromList+boolsC :: [Bool] -> DI.Column+boolsC = DI.fromList++sample :: Int -> ReadOptions+sample n = defaultReadOptions{typeSpec = InferFromSample n}++goldenCase :: (String, ReadOptions, BS.ByteString, Expect) -> Test+goldenCase (label, opts, input, expect) = TestLabel label $ TestCase $ do+    r <- try $ do+        df <- decodeSeparated opts (BL.fromStrict input)+        evaluate (forceDataFrame df)+    case (expect, r) of+        (Err sub, Left (e :: SomeException)) ->+            assertBool+                (label <> ": error should mention " <> show sub <> ", got " <> show e)+                (sub `isInfixOf` show e)+        (Err _, Right _) -> assertFailure (label <> ": expected an error")+        (Cols _ _, Left (e :: SomeException)) ->+            assertFailure (label <> ": unexpected error " <> show e)+        (Cols dims cols, Right df) -> do+            assertEqual (label <> ": dims") dims (dataframeDimensions df)+            mapM_+                ( \(name, expected) -> case getColumn name df of+                    Nothing -> assertFailure (label <> ": missing column " <> show name)+                    Just actual ->+                        assertEqual (label <> ": column " <> show name) expected actual+                )+                cols++goldenCases :: [(String, ReadOptions, BS.ByteString, Expect)]+goldenCases =+    [+        ( "basic"+        , defaultReadOptions+        , "a,b,c\n1,2.5,x\n2,3.5,y\n"+        , Cols+            (2, 3)+            [("a", ints [1, 2]), ("b", dbls [2.5, 3.5]), ("c", texts ["x", "y"])]+        )+    ,+        ( "quoted_doubled"+        , defaultReadOptions+        , "a,b\n\"x,1\",\"he said \"\"hi\"\"\"\n\"multi\nline\",plain\n"+        , Cols+            (2, 2)+            [("a", texts ["x,1", "multi\nline"]), ("b", texts ["he said \"hi\"", "plain"])]+        )+    ,+        ( "crlf"+        , defaultReadOptions+        , "a,b\r\n1,x\r\n2,y\r\n"+        , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])]+        )+    ,+        ( "lone_cr"+        , defaultReadOptions+        , "a,b\r1,x\r2,y"+        , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])]+        )+    ,+        ( "blank_lines"+        , defaultReadOptions+        , "a,b\n\n1,x\n\n\n2,y\n\n"+        , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])]+        )+    ,+        ( "lone_cr_line"+        , defaultReadOptions+        , "a,b\r\n\r1,x\r\n"+        , Cols (1, 2) [("a", ints [1]), ("b", texts ["x"])]+        )+    ,+        ( "no_eof_newline"+        , defaultReadOptions+        , "a,b\n1,x"+        , Cols (1, 2) [("a", ints [1]), ("b", texts ["x"])]+        )+    ,+        ( "missing_numeric"+        , defaultReadOptions+        , "a,b\n1,2\nNA,3\n,4\nnan,5\n"+        , Cols+            (4, 2)+            [("a", mints [Just 1, Nothing, Nothing, Nothing]), ("b", ints [2, 3, 4, 5])]+        )+    ,+        ( "missing_text"+        , defaultReadOptions+        , "t,u\nfoo,1\nNA,2\n ,3\nbar,4\n"+        , Cols+            (4, 2)+            [ ("t", mtexts [Just "foo", Nothing, Nothing, Just "bar"])+            , ("u", ints [1, 2, 3, 4])+            ]+        )+    ,+        ( "ws_padding"+        , defaultReadOptions+        , "a,b\n 1 , x \n2,y\n"+        , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])]+        )+    ,+        ( "tabs_strip"+        , defaultReadOptions+        , "a,b\n\t1\t,\tx\t\n2,y\n"+        , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])]+        )+    , -- D6: short rows now pad trailing columns with null (was: misaligned).++        ( "ragged_short_D6"+        , defaultReadOptions+        , "a,b,c\n1,2,3\n4,5\n6,7,8\n"+        , Cols+            (3, 3)+            [ ("a", ints [1, 4, 6])+            , ("b", ints [2, 5, 7])+            , ("c", mints [Just 3, Nothing, Just 8])+            ]+        )+    ,+        ( "ragged_long"+        , defaultReadOptions+        , "a,b\n1,2\n3,4,5\n6,7\n"+        , Cols (3, 2) [("a", ints [1, 3, 6]), ("b", ints [2, 4, 7])]+        )+    ,+        ( "all_rows_ragged_short_D6"+        , defaultReadOptions+        , "a,b,c\n1,2\n3,4\n"+        , Cols+            (2, 3)+            [("a", ints [1, 3]), ("b", ints [2, 4]), ("c", mtexts [Nothing, Nothing])]+        )+    ,+        ( "maybe_read_ragged_D6"+        , defaultReadOptions{safeRead = MaybeRead}+        , "a,b\n1,2\n3\n"+        , Cols (2, 2) [("a", mints [Just 1, Just 3]), ("b", mints [Just 2, Nothing])]+        )+    ,+        ( "either_read_ragged_D6"+        , defaultReadOptions{safeRead = EitherRead}+        , "a,b\n1,2\n3\n"+        , Cols (2, 2) [("a", eti [Right 1, Right 3]), ("b", eti [Right 2, Left ""])]+        )+    , ("stray_quote_mid", defaultReadOptions, "a,b\nx\"y,2\n", Err "")+    , ("quote_garbage", defaultReadOptions, "a,b\n\"x\"y,2\n", Err "")+    , ("space_then_quote", defaultReadOptions, "a,b\n \"x\",2\n", Err "")+    , ("quote_space_garbage", defaultReadOptions, "a,b\n\"x\" ,2\n", Err "")+    , ("quote_at_field_end", defaultReadOptions, "a,b\n1,x\"\n", Err "")+    , ("doubled_then_garbage", defaultReadOptions, "a,b\n\"\"x,2\n", Err "")+    ,+        ( "quoted_empty"+        , defaultReadOptions+        , "a,b\n\"\",2\n"+        , Cols (1, 2) [("a", mtexts [Nothing]), ("b", ints [2])]+        )+    ,+        ( "quoted_empty_row_skipped"+        , defaultReadOptions+        , "a\nx\n\"\"\ny\n"+        , Cols (2, 1) [("a", texts ["x", "y"])]+        )+    ,+        ( "noheader"+        , defaultReadOptions{headerSpec = NoHeader}+        , "1,2\n3,4\n"+        , Cols (2, 2) [("0", ints [1, 3]), ("1", ints [2, 4])]+        )+    ,+        ( "providenames_fewer"+        , defaultReadOptions{headerSpec = ProvideNames ["x"]}+        , "1,2\n3,4\n"+        , Cols (2, 2) [("x", ints [1, 3]), ("1", ints [2, 4])]+        )+    , -- D6: the always-short padded column is now row-aligned (all null).++        ( "providenames_more_D6"+        , defaultReadOptions{headerSpec = ProvideNames ["x", "y", "z"]}+        , "1,2\n3,4\n"+        , Cols+            (2, 3)+            [("x", ints [1, 3]), ("y", ints [2, 4]), ("z", mtexts [Nothing, Nothing])]+        )+    ,+        ( "either_read"+        , (sample 2){safeRead = EitherRead}+        , "a,b\n1,x\n2,y\nz,\n"+        , Cols+            (3, 2)+            [ ("a", eti [Right 1, Right 2, Left "z"])+            , ("b", ett [Right "x", Right "y", Left ""])+            ]+        )+    ,+        ( "maybe_read"+        , defaultReadOptions{safeRead = MaybeRead}+        , "a,b\n1,x\nNA,y\n"+        , Cols (2, 2) [("a", mints [Just 1, Nothing]), ("b", mtexts [Just "x", Just "y"])]+        )+    ,+        ( "schema_int_bad"+        , defaultReadOptions{typeSpec = SpecifyTypes [("a", schemaTypeInt)] NoInference}+        , "a,b\n1,p\nx,q\n3,r\n"+        , Cols+            (3, 2)+            [("a", mints [Just 1, Nothing, Just 3]), ("b", texts ["p", "q", "r"])]+        )+    ,+        ( "schema_int_missing"+        , defaultReadOptions{typeSpec = SpecifyTypes [("a", schemaTypeInt)] NoInference}+        , "a\n1\nNA\n\n3\n"+        , Cols (3, 1) [("a", mints [Just 1, Nothing, Just 3])]+        )+    ,+        ( "schema_text_missing"+        , defaultReadOptions{typeSpec = SpecifyTypes [("a", schemaTypeText)] NoInference}+        , "a\nx\nNA\n\nz\n"+        , Cols (3, 1) [("a", mtexts [Just "x", Nothing, Just "z"])]+        )+    ,+        ( "dates"+        , defaultReadOptions+        , "d\n2024-01-02\n2024-02-03\n"+        , Cols (2, 1) [("d", days [fromGregorian 2024 1 2, fromGregorian 2024 2 3])]+        )+    ,+        ( "dates_custom"+        , defaultReadOptions{dateFormat = "%d/%m/%Y"}+        , "d\n02/01/2024\n03/02/2024\n"+        , Cols (2, 1) [("d", days [fromGregorian 2024 1 2, fromGregorian 2024 2 3])]+        )+    ,+        ( "custom_sep"+        , defaultReadOptions{columnSeparator = ';'}+        , "a;b\n1;x\n2;y\n"+        , Cols (2, 2) [("a", ints [1, 2]), ("b", texts ["x", "y"])]+        )+    ,+        ( "custom_missing"+        , defaultReadOptions{missingIndicators = ["foo"]}+        , "a,b\n1,foo\n2,bar\n"+        , Cols (2, 2) [("a", ints [1, 2]), ("b", mtexts [Nothing, Just "bar"])]+        )+    ,+        ( "bools"+        , defaultReadOptions+        , "f\nTrue\nFalse\n"+        , Cols (2, 1) [("f", boolsC [True, False])]+        )+    ,+        ( "bool_ws"+        , defaultReadOptions+        , "f\n True\nFalse\n"+        , Cols (2, 1) [("f", boolsC [True, False])]+        )+    ,+        ( "single_space_field"+        , defaultReadOptions+        , "a,b\nx, \ny,z\n"+        , Cols (2, 2) [("a", texts ["x", "y"]), ("b", mtexts [Nothing, Just "z"])]+        )+    ,+        ( "row_cap"+        , defaultReadOptions{numColumns = Just 2}+        , "a\n1\n2\n3\n4\n"+        , Cols (2, 1) [("a", ints [1, 2])]+        )+    ,+        ( "row_cap_zero"+        , defaultReadOptions{numColumns = Just 0}+        , "a,b\n1,2\n"+        , Cols (0, 2) [("a", mtexts []), ("b", mtexts [])]+        )+    ,+        ( "trailing_sep"+        , defaultReadOptions+        , "a,b\n1,\n2,3\n"+        , Cols (2, 2) [("a", ints [1, 2]), ("b", mints [Nothing, Just 3])]+        )+    , ("header_only", defaultReadOptions, "a,b\n", Err "Empty CSV file")+    , ("empty", defaultReadOptions, "", Err "Empty CSV file")+    , ("only_blank_lines", defaultReadOptions, "\n\n\n", Err "Empty CSV file")+    ,+        ( "all_null_col"+        , defaultReadOptions+        , "a,b\n,1\n,2\n"+        , Cols (2, 2) [("a", mtexts [Nothing, Nothing]), ("b", ints [1, 2])]+        )+    ,+        ( "int_overflow"+        , defaultReadOptions+        , "a\n9223372036854775808\n1\n"+        , Cols (2, 1) [("a", dbls [9.223372036854776e18, 1.0])]+        )+    ,+        ( "mixed_int_double"+        , defaultReadOptions+        , "a\n1\n2.5\n"+        , Cols (2, 1) [("a", dbls [1.0, 2.5])]+        )+    ,+        ( "int_then_text"+        , sample 2+        , "a\n1\n2\nx\n"+        , Cols (3, 1) [("a", texts ["1", "2", "x"])]+        )+    ,+        ( "int_then_double"+        , sample 2+        , "a\n1\n2\n3.5\n"+        , Cols (3, 1) [("a", dbls [1.0, 2.0, 3.5])]+        )+    ,+        ( "sample_all_null_then_data"+        , sample 2+        , "a\n\n\n5\n7\n"+        , Cols (2, 1) [("a", ints [5, 7])]+        )+    ,+        ( "quoted_number"+        , defaultReadOptions+        , "a\n\"1\"\n\"2\"\n"+        , Cols (2, 1) [("a", ints [1, 2])]+        )+    ,+        ( "quoted_padded_number"+        , defaultReadOptions+        , "a\n\" 1 \"\n\"2\"\n"+        , Cols (2, 1) [("a", ints [1, 2])]+        )+    ,+        ( "quoted_header"+        , defaultReadOptions+        , "\"a b\",c\n1,2\n"+        , Cols (1, 2) [("a b", ints [1]), ("c", ints [2])]+        )+    ,+        ( "header_quoted_doubled"+        , defaultReadOptions+        , "\"a\"\"b\",c\n1,2\n"+        , Cols (1, 2) [("a\"b", ints [1]), ("c", ints [2])]+        )+    ,+        ( "quoted_crlf_field"+        , defaultReadOptions+        , "a,b\r\n\"x\r\ny\",2\r\n"+        , Cols (1, 2) [("a", texts ["x\r\ny"]), ("b", ints [2])]+        )+    ,+        ( "quoted_field_with_cr_alone"+        , defaultReadOptions+        , "a,b\n\"x\ry\",2\n"+        , Cols (1, 2) [("a", texts ["x\ry"]), ("b", ints [2])]+        )+    ,+        ( "tab_sep_quoted"+        , defaultReadOptions{columnSeparator = '\t'}+        , "a\tb\n\"x\t1\"\ty\n"+        , Cols (1, 2) [("a", texts ["x\t1"]), ("b", texts ["y"])]+        )+    ,+        ( "either_disables_missing"+        , defaultReadOptions{safeReadOverrides = [("b", EitherRead)]}+        , "a,b\nNA,1\n2,2\n"+        , Cols (2, 2) [("a", texts ["NA", "2"]), ("b", eti [Right 1, Right 2])]+        )+    , -- D6: the column after the EOF-truncated quote field is now padded.++        ( "unclosed_quote_D6"+        , defaultReadOptions+        , "a,b\n\"x,2\n"+        , Cols (1, 2) [("a", texts ["x,2"]), ("b", mtexts [Nothing])]+        )+    ,+        ( "unclosed_trailing_doubled_D6"+        , defaultReadOptions+        , "a,b\n\"x\"\"\n"+        , Cols (1, 2) [("a", texts ["x\""]), ("b", mtexts [Nothing])]+        )+    ,+        ( "closed_after_doubled_D6"+        , defaultReadOptions+        , "a,b\n\"x\"\"\"\n"+        , Cols (1, 2) [("a", texts ["x\""]), ("b", mtexts [Nothing])]+        )+    ,+        ( "multi_doubled"+        , defaultReadOptions+        , "a\n\"x\"\"y\"\"z\"\nw\n"+        , Cols (2, 1) [("a", texts ["x\"y\"z", "w"])]+        )+    ,+        ( "only_doubled"+        , defaultReadOptions+        , "a\n\"\"\"\"\nw\n"+        , Cols (2, 1) [("a", texts ["\"", "w"])]+        )+    ,+        ( "doubled_at_open"+        , defaultReadOptions+        , "a,b\n\"\"\"x\",2\n"+        , Cols (1, 2) [("a", texts ["\"x"]), ("b", ints [2])]+        )+    ,+        ( "interior_doubled_inferred"+        , defaultReadOptions+        , "a\n\"x\"\"y\"\nz\n"+        , Cols (2, 1) [("a", texts ["x\"y", "z"])]+        )+    ,+        ( "trailing_cr_eof"+        , defaultReadOptions+        , "a,b\n1,x\r"+        , Cols (1, 2) [("a", ints [1]), ("b", texts ["x"])]+        )+    ,+        ( "single_col_blank"+        , defaultReadOptions+        , "a\n1\n\n2\n"+        , Cols (2, 1) [("a", ints [1, 2])]+        )+    ,+        ( "single_col_quoted_empty"+        , defaultReadOptions+        , "a\n1\n\"\"\n2\n"+        , Cols (2, 1) [("a", ints [1, 2])]+        )+    ,+        ( "row_only_sep"+        , defaultReadOptions+        , "a,b\n,\n1,2\n"+        , Cols (2, 2) [("a", mints [Nothing, Just 1]), ("b", mints [Nothing, Just 2])]+        )+    ,+        ( "sep_at_eof"+        , defaultReadOptions+        , "a,b\n1,2\n3,"+        , Cols (2, 2) [("a", ints [1, 3]), ("b", mints [Just 2, Nothing])]+        )+    ,+        ( "noheader_ragged_grow"+        , defaultReadOptions{headerSpec = NoHeader}+        , "1,2\n3,4,5\n"+        , Cols (2, 2) [("0", ints [1, 3]), ("1", ints [2, 4])]+        )+    ,+        ( "quoted_last_no_nl"+        , defaultReadOptions+        , "a,b\n1,2\n\"x\",3"+        , Cols (2, 2) [("a", texts ["1", "x"]), ("b", ints [2, 3])]+        )+    ,+        ( "empty_quoted_middle_col"+        , defaultReadOptions+        , "a,b,c\n1,\"\",3\n4,5,6\n"+        , Cols+            (2, 3)+            [("a", ints [1, 4]), ("b", mints [Nothing, Just 5]), ("c", ints [3, 6])]+        )+    ,+        ( "exp_double"+        , defaultReadOptions+        , "a\n1e3\n1E-3\n1e999\n"+        , Cols (3, 1) [("a", dbls [1000.0, 1.0e-3, 1 / 0])]+        )+    ,+        ( "five_field_double"+        , defaultReadOptions+        , "a\n+.5\n5.\n"+        , Cols (2, 1) [("a", texts ["+.5", "5."])]+        )+    ,+        ( "raw_invalid_utf8"+        , defaultReadOptions+        , BS.pack [97, 10, 255, 10, 98, 10]+        , Cols (2, 1) [("a", texts ["\65533", "b"])]+        )+    ,+        ( "raw_nbsp_edges_stripped"+        , defaultReadOptions+        , BS.pack [97, 10, 160, 120, 160, 10, 98, 10]+        , Cols (2, 1) [("a", texts ["x", "b"])]+        )+    ]+  where+    schemaTypeInt = schemaType @Int+    schemaTypeText = schemaType @T.Text++-- Duplicate header names: last index wins in the map; both columns kept.+dupHeaders :: Test+dupHeaders = TestLabel "dup_headers" $ TestCase $ do+    df <- decodeSeparated defaultReadOptions (BL.fromStrict "a,a\n1,2\n3,4\n")+    assertEqual "dims" (2, 2) (dataframeDimensions df)+    assertEqual "index map" [("a", 1)] (M.toList (columnIndices df))++-- Ingest pin: a string column freezes to a shared-buffer 'PackedText'+-- (no per-row boxed Text header), while values stay byte-identical.+textFreezesPacked :: Test+textFreezesPacked = TestLabel "text_freezes_packed" $ TestCase $ do+    df <- decodeSeparated defaultReadOptions (BL.fromStrict "a,b\n1,x\n2,y\n")+    case getColumn "b" df of+        Nothing -> assertFailure "missing text column"+        Just col -> do+            assertBool "text column is PackedText" (DI.isPackedText col)+            assertEqual "values byte-identical" (texts ["x", "y"]) col++tests :: [Test]+tests = dupHeaders : textFreezesPacked : map goldenCase goldenCases

@@ -0,0 +1,298 @@+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++module Internal.ColumnBuilder (tests, props) where++import qualified Data.ByteString as B+import qualified Data.ByteString.Unsafe as BU+import qualified Data.Text as T+import qualified Data.Text.Array as A+import qualified Data.Vector as VB+import qualified Data.Vector.Unboxed as VU++import Control.Monad (forM_, zipWithM_)+import Control.Monad.ST (runST, stToIO)+import Data.Maybe (catMaybes, isNothing)+import Data.Text.Encoding (decodeUtf8Lenient, encodeUtf8)+import Data.Type.Equality (testEquality, (:~:) (Refl))+import Data.Word (Word8)+import DataFrame.Internal.Column hiding (mergeColumns)+import DataFrame.Internal.ColumnBuilder+import Foreign.Ptr (castPtr)+import Test.HUnit+import Test.QuickCheck+import Type.Reflection (typeRep)++-- Build a column by appending each list element (Nothing => appendNull).+buildIntColumn :: [Maybe Int] -> Column+buildIntColumn xs = runST $ do+    b <- newIntBuilder (length xs)+    mapM_ (maybe (appendNull b) (appendInt b)) xs+    freezeBuilder b++buildDoubleColumn :: [Maybe Double] -> Column+buildDoubleColumn xs = runST $ do+    b <- newDoubleBuilder (length xs)+    mapM_ (maybe (appendNull b) (appendDouble b)) xs+    freezeBuilder b++buildTextColumn :: [Maybe T.Text] -> Column+buildTextColumn xs = runST $ do+    b <- newTextBuilder (length xs) 16+    mapM_ (maybe (appendNull b) (appendText b)) xs+    freezeBuilder b++-- Build a text column from raw byte fields via appendTextSlice.+buildTextColumnFromBytes :: [[Word8]] -> Column+buildTextColumnFromBytes fields = runST $ do+    b <- newTextBuilder (length fields) 16+    forM_ fields $ \ws ->+        appendTextSlice b (arrayFromBytes ws) 0 (length ws)+    freezeBuilder b++arrayFromBytes :: [Word8] -> A.Array+arrayFromBytes ws = A.run $ do+    m <- A.new (length ws)+    zipWithM_ (A.unsafeWrite m) [0 ..] ws+    pure m++-- Reference column: fromVector of the plain list (no nulls) or Maybe list.+expectedColumn ::+    forall a.+    (Columnable a, Columnable (Maybe a)) =>+    [Maybe a] -> Column+expectedColumn xs+    | any isNothing xs = fromVector (VB.fromList xs)+    | otherwise = fromVector (VB.fromList (catMaybes xs))++-- Read a text column back as a Maybe list, respecting the bitmap.+-- Ingest now freezes text to 'PackedText'; materialize to the boxed form.+columnTexts :: Column -> [Maybe T.Text]+columnTexts c@(PackedText _ _) = columnTexts (materializePacked c)+columnTexts (BoxedColumn mb (v :: VB.Vector b)) =+    case testEquality (typeRep @b) (typeRep @T.Text) of+        Just Refl ->+            [ if maybe True (`bitmapTestBit` i) mb+                then Just (v VB.! i)+                else Nothing+            | i <- [0 .. VB.length v - 1]+            ]+        Nothing -> error "columnTexts: not a Text column"+columnTexts _ = error "columnTexts: not a boxed column"++-- Split a list into arbitrary contiguous chunks (at least one chunk).+splitsOf :: [a] -> Gen [[a]]+splitsOf [] = pure [[]]+splitsOf xs = go xs+  where+    go [] = pure []+    go ys = do+        k <- chooseInt (1, length ys)+        let (a, b) = splitAt k ys+        (a :) <$> go b++nullableVersion :: [Maybe a] -> String -> String -> String+nullableVersion xs nullable plain =+    if any isNothing xs then nullable else plain++-- HUnit tests++intAllValidHasNoBitmap :: Test+intAllValidHasNoBitmap =+    TestCase $+        assertEqual+            "all-valid int column is Unboxed with no bitmap"+            "Unboxed"+            (columnVersionString (buildIntColumn (map Just [1, 2, 3])))++intWithNullHasBitmap :: Test+intWithNullHasBitmap =+    TestCase $+        assertEqual+            "int column with null is NullableUnboxed"+            "NullableUnboxed"+            (columnVersionString (buildIntColumn [Just 1, Nothing]))++intNullSentinelIsZero :: Test+intNullSentinelIsZero = TestCase $+    case buildIntColumn [Just 5, Nothing, Just 7] of+        UnboxedColumn (Just _) (v :: VU.Vector b) ->+            case testEquality (typeRep @b) (typeRep @Int) of+                Just Refl -> assertEqual "null slot holds 0" 0 (v VU.! 1)+                Nothing -> assertFailure "expected an Int column"+        _ -> assertFailure "expected a nullable unboxed column"++textAllValidHasNoBitmap :: Test+textAllValidHasNoBitmap =+    TestCase $+        assertEqual+            "all-valid text column is Boxed with no bitmap"+            "Boxed"+            (columnVersionString (buildTextColumn [Just "a", Just "bc"]))++builderLengthCounts :: Test+builderLengthCounts = TestCase $ do+    let n = runST $ do+            b <- newIntBuilder 4+            appendInt b 1+            appendNull b+            appendInt b 3+            builderLength b+    assertEqual "builderLength counts nulls and values" 3 n++growthPastCapacityHint :: Test+growthPastCapacityHint = TestCase $ do+    let xs = map Just [0 .. 9999 :: Int]+        col = runST $ do+            b <- newIntBuilder 1+            mapM_ (maybe (appendNull b) (appendInt b)) xs+            freezeBuilder b+    assertEqual "doubling growth preserves data" (expectedColumn xs) col++multibyteTextRoundTrip :: Test+multibyteTextRoundTrip = TestCase $ do+    let ts = ["héllo", "wörld", "日本語", "😀😃", ""]+    assertEqual+        "multibyte text round trips"+        (map Just ts)+        (columnTexts (buildTextColumn (map Just ts)))++-- A multibyte sequence split across two fields: the whole buffer is valid+-- UTF-8 but each field alone is not. Both must lenient-decode to U+FFFD.+splitMultibyteAcrossFields :: Test+splitMultibyteAcrossFields = TestCase $ do+    let fields = [[0xC2], [0x80]]+    assertEqual+        "fields splitting a sequence decode leniently"+        (map (Just . decodeUtf8Lenient . B.pack) fields)+        (columnTexts (buildTextColumnFromBytes fields))++splitEuroAcrossFields :: Test+splitEuroAcrossFields = TestCase $ do+    let fields = [[0xE2, 0x82], [0xAC, 0x61]]+    assertEqual+        "euro sign split across fields decodes leniently"+        (map (Just . decodeUtf8Lenient . B.pack) fields)+        (columnTexts (buildTextColumnFromBytes fields))++invalidUtf8EdgeCases :: Test+invalidUtf8EdgeCases = TestCase $ do+    let fields =+            [ [0xE2, 0x28, 0xA1]+            , [0xF0, 0x9F, 0x98]+            , [0xF0, 0x28]+            , [0xC2]+            , [0x80]+            , [0xED, 0xA0, 0x80]+            , [0xF4, 0x90, 0x80, 0x80]+            , [0xC0, 0xAF]+            , [0x61, 0xF1, 0x80, 0x80, 0xE1, 0x80, 0xC2, 0x62]+            , [0xF0, 0x9F, 0x98, 0x80]+            ]+    assertEqual+        "invalid sequences match decodeUtf8Lenient"+        (map (Just . decodeUtf8Lenient . B.pack) fields)+        (columnTexts (buildTextColumnFromBytes fields))++appendFromPtrMatchesText :: Test+appendFromPtrMatchesText = TestCase $ do+    let t = "héllo wörld 😀" :: T.Text+    col <- BU.unsafeUseAsCStringLen (encodeUtf8 t) $ \(p, len) ->+        stToIO $ do+            b <- newTextBuilder 1 0+            appendTextSliceFromPtr b (castPtr p) len+            freezeBuilder b+    assertEqual "ptr slice decodes to original text" [Just t] (columnTexts col)++mergeBitmapSpliceUnaligned :: Test+mergeBitmapSpliceUnaligned = TestCase $ do+    let nullIdx = [1, 4, 9, 10, 14] :: [Int]+        xs = [if i `elem` nullIdx then Nothing else Just i | i <- [0 .. 14]]+        chunks = [take 3 xs, take 5 (drop 3 xs), drop 8 xs]+        merged = mergeColumns (map buildIntColumn chunks)+    assertEqual "merged equals unsplit" (buildIntColumn xs) merged+    forM_ [0 .. 14] $ \i ->+        assertEqual+            ("null at index " ++ show i)+            (i `elem` nullIdx)+            (columnElemIsNull merged i)++tests :: [Test]+tests =+    [ TestLabel "intAllValidHasNoBitmap" intAllValidHasNoBitmap+    , TestLabel "intWithNullHasBitmap" intWithNullHasBitmap+    , TestLabel "intNullSentinelIsZero" intNullSentinelIsZero+    , TestLabel "textAllValidHasNoBitmap" textAllValidHasNoBitmap+    , TestLabel "builderLengthCounts" builderLengthCounts+    , TestLabel "growthPastCapacityHint" growthPastCapacityHint+    , TestLabel "multibyteTextRoundTrip" multibyteTextRoundTrip+    , TestLabel "splitMultibyteAcrossFields" splitMultibyteAcrossFields+    , TestLabel "splitEuroAcrossFields" splitEuroAcrossFields+    , TestLabel "invalidUtf8EdgeCases" invalidUtf8EdgeCases+    , TestLabel "appendFromPtrMatchesText" appendFromPtrMatchesText+    , TestLabel "mergeBitmapSpliceUnaligned" mergeBitmapSpliceUnaligned+    ]++-- QuickCheck properties++prop_intMatchesFromVector :: [Maybe Int] -> Property+prop_intMatchesFromVector xs =+    buildIntColumn xs === expectedColumn xs+        .&&. columnVersionString (buildIntColumn xs)+            === nullableVersion xs "NullableUnboxed" "Unboxed"++prop_doubleMatchesFromVector :: [Maybe Double] -> Property+prop_doubleMatchesFromVector xs =+    buildDoubleColumn xs === expectedColumn xs++prop_textMatchesFromVector :: [Maybe String] -> Property+prop_textMatchesFromVector ss =+    let xs = map (fmap T.pack) ss+     in buildTextColumn xs === expectedColumn xs+            .&&. columnVersionString (buildTextColumn xs)+                === nullableVersion xs "NullableBoxed" "Boxed"++prop_textSliceMatchesLenientDecode :: [[Word8]] -> Property+prop_textSliceMatchesLenientDecode fields =+    columnTexts (buildTextColumnFromBytes fields)+        === map (Just . decodeUtf8Lenient . B.pack) fields++prop_mergeIntMatchesUnsplit :: [Maybe Int] -> Property+prop_mergeIntMatchesUnsplit xs = forAll (splitsOf xs) $ \chunks ->+    let merged = mergeColumns (map buildIntColumn chunks)+        unsplit = buildIntColumn xs+     in merged === unsplit+            .&&. columnVersionString merged === columnVersionString unsplit++prop_mergeDoubleMatchesUnsplit :: [Maybe Double] -> Property+prop_mergeDoubleMatchesUnsplit xs = forAll (splitsOf xs) $ \chunks ->+    mergeColumns (map buildDoubleColumn chunks) === buildDoubleColumn xs++prop_mergeTextMatchesUnsplit :: [Maybe String] -> Property+prop_mergeTextMatchesUnsplit ss =+    let xs = map (fmap T.pack) ss+     in forAll (splitsOf xs) $ \chunks ->+            let merged = mergeColumns (map buildTextColumn chunks)+                unsplit = buildTextColumn xs+             in merged === unsplit+                    .&&. columnTexts merged === columnTexts unsplit++prop_mergeNullsLandAtRightRows :: [Maybe Int] -> Property+prop_mergeNullsLandAtRightRows xs = forAll (splitsOf xs) $ \chunks ->+    let merged = mergeColumns (map buildIntColumn chunks)+     in map isNothing xs+            === [columnElemIsNull merged i | i <- [0 .. length xs - 1]]++props :: [Property]+props =+    [ property prop_intMatchesFromVector+    , property prop_doubleMatchesFromVector+    , property prop_textMatchesFromVector+    , property prop_textSliceMatchesLenientDecode+    , property prop_mergeIntMatchesUnsplit+    , property prop_mergeDoubleMatchesUnsplit+    , property prop_mergeTextMatchesUnsplit+    , property prop_mergeNullsLandAtRightRows+    ]

@@ -0,0 +1,144 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}++{- | Correctness oracle for the dictionary-encode building block+('DataFrame.Internal.DictEncode'). A text column encodes to dense+first-appearance @Int@ codes: two rows share a code iff their text is equal, the+codes are a contiguous @0 .. card-1@ range in first-appearance order, packed and+boxed Text encode identically, and the cap parameter bails past a cardinality.+Pins the step that the group-by planner can route a text key through.+-}+module Internal.DictEncode (tests) where++import qualified Data.ByteString as B+import qualified Data.Map.Strict as M+import qualified Data.Text as T+import qualified Data.Text.Array as A+import qualified Data.Vector.Unboxed as VU++import Control.Monad (zipWithM_)+import Data.Maybe (isJust)+import Data.Text.Encoding (encodeUtf8)+import Data.Word (Word8)+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.DictEncode (dictEncodeColumn, dictEncodeColumnUpTo)+import DataFrame.Internal.PackedText (mkPackedContiguous)+import Test.HUnit++sampleRows :: [T.Text]+sampleRows =+    [ "apple"+    , "banana"+    , ""+    , "apple"+    , "Zebra"+    , "café"+    , "naïve"+    , "日本語"+    , "banana"+    , "apple"+    ]++arrayFromBytes :: [Word8] -> A.Array+arrayFromBytes ws = A.run $ do+    m <- A.new (length ws)+    zipWithM_ (A.unsafeWrite m) [0 ..] ws+    pure m++packedFromTexts :: [T.Text] -> DI.Column+packedFromTexts ts =+    let bytess = map (B.unpack . encodeUtf8) ts+        flat = concat bytess+        offs = scanl (+) 0 (map length bytess)+        arr = arrayFromBytes flat+     in DI.PackedText Nothing (mkPackedContiguous arr (VU.fromList offs))++boxedFromTexts :: [T.Text] -> DI.Column+boxedFromTexts = DI.fromList++{- | The reference dense first-appearance encoding computed with a plain+'Data.Map': scan in row order, assign the next id to each new value.+-}+oracleCodes :: [T.Text] -> ([Int], Int)+oracleCodes ts = go ts M.empty 0 []+  where+    go [] _ next acc = (reverse acc, next)+    go (x : xs) m next acc = case M.lookup x m of+        Just c -> go xs m next (c : acc)+        Nothing -> go xs (M.insert x next m) (next + 1) (next : acc)++codesMatchOracle :: String -> DI.Column -> Test+codesMatchOracle lbl col =+    TestCase $ case dictEncodeColumn col of+        Nothing -> assertFailure (lbl ++ ": expected Just codes")+        Just (codes, card) -> do+            let (refCodes, refCard) = oracleCodes sampleRows+            assertEqual (lbl ++ " codes") refCodes (VU.toList codes)+            assertEqual (lbl ++ " cardinality") refCard card++packedBoxedAgree :: Test+packedBoxedAgree =+    TestCase $+        assertEqual+            "packed and boxed encode identically"+            (dictEncodeColumn (boxedFromTexts sampleRows))+            (dictEncodeColumn (packedFromTexts sampleRows))++denseRange :: Test+denseRange =+    TestCase $ case dictEncodeColumn (packedFromTexts sampleRows) of+        Nothing -> assertFailure "expected Just"+        Just (codes, card) -> do+            assertBool "codes in [0,card)" (VU.all (\c -> c >= 0 && c < card) codes)+            assertEqual+                "all codes used"+                [0 .. card - 1]+                (M.keys (M.fromList [(c, ()) | c <- VU.toList codes]))++equalIffSameText :: Test+equalIffSameText =+    TestCase $ case dictEncodeColumn (packedFromTexts sampleRows) of+        Nothing -> assertFailure "expected Just"+        Just (codes, _) ->+            let pairs = [(i, j) | i <- idxs, j <- idxs, i < j]+                idxs = [0 .. length sampleRows - 1]+                ok (i, j) =+                    (VU.unsafeIndex codes i == VU.unsafeIndex codes j)+                        == (sampleRows !! i == sampleRows !! j)+             in assertBool "code equality matches text equality" (all ok pairs)++capBails :: Test+capBails =+    TestCase $ do+        -- sampleRows has 7 distinct values; a cap below that must bail.+        assertEqual+            "cap 3 bails"+            Nothing+            (dictEncodeColumnUpTo 3 (packedFromTexts sampleRows))+        -- a generous cap still encodes.+        assertBool+            "cap 100 encodes"+            (isJust (dictEncodeColumnUpTo 100 (packedFromTexts sampleRows)))++nonTextIsNothing :: Test+nonTextIsNothing =+    TestCase $+        assertEqual+            "int column is not dict-encoded"+            Nothing+            (dictEncodeColumn (DI.fromList [1 :: Int, 2, 3]))++tests :: [Test]+tests =+    [ TestLabel+        "dictCodesPacked"+        (codesMatchOracle "packed" (packedFromTexts sampleRows))+    , TestLabel+        "dictCodesBoxed"+        (codesMatchOracle "boxed" (boxedFromTexts sampleRows))+    , TestLabel "dictPackedBoxedAgree" packedBoxedAgree+    , TestLabel "dictDenseRange" denseRange+    , TestLabel "dictEqualIffSameText" equalIffSameText+    , TestLabel "dictCapBails" capBails+    , TestLabel "dictNonTextNothing" nonTextIsNothing+    ]

@@ -0,0 +1,186 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++{- | Correctness oracle for the packed-text column variant. Every assertion+builds a 'PackedText' column from raw bytes + offsets and the same data as a+'BoxedColumn Text', then checks display, extraction, equality, ordering,+hashing, groupBy and join all agree. This pins behavioral parity before any+reader emits packed columns.+-}+module Internal.PackedText (tests) where++import qualified Data.Text as T+import qualified Data.Text.Array as A+import qualified Data.Vector.Unboxed as VU++import Control.Monad (zipWithM_)+import qualified Data.ByteString as B+import Data.Text.Encoding (encodeUtf8)+import Data.Word (Word8)+import qualified DataFrame as D+import qualified DataFrame.Functions as F+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.DataFrame (unsafeGetColumn)+import DataFrame.Internal.PackedText (mkPackedContiguous)+import qualified DataFrame.Operations.Aggregation as Agg+import Test.HUnit++-- Sample string corpus, including empty, ASCII, and multibyte UTF-8 fields.+sampleRows :: [T.Text]+sampleRows =+    [ "apple"+    , "banana"+    , ""+    , "apple"+    , "Zebra"+    , "café"+    , "naïve"+    , "日本語"+    , "banana"+    , "apple"+    ]++-- Build an A.Array from a flat byte list.+arrayFromBytes :: [Word8] -> A.Array+arrayFromBytes ws = A.run $ do+    m <- A.new (length ws)+    zipWithM_ (A.unsafeWrite m) [0 ..] ws+    pure m++-- Build a PackedText column directly from raw UTF-8 bytes + offsets.+packedFromTexts :: [T.Text] -> DI.Column+packedFromTexts ts =+    let bytess = map (B.unpack . encodeUtf8) ts+        flat = concat bytess+        offs = scanl (+) 0 (map length bytess)+        arr = arrayFromBytes flat+     in DI.PackedText Nothing (mkPackedContiguous arr (VU.fromList offs))++boxedFromTexts :: [T.Text] -> DI.Column+boxedFromTexts = DI.fromList++packedCol, boxedCol :: DI.Column+packedCol = packedFromTexts sampleRows+boxedCol = boxedFromTexts sampleRows++-- Single-column dataframes for groupBy / hashing.+packedDf, boxedDf :: D.DataFrame+packedDf = D.fromNamedColumns [("k", packedCol)]+boxedDf = D.fromNamedColumns [("k", boxedCol)]++displayParity :: Test+displayParity =+    TestCase $+        assertEqual "show packed == show boxed" (show boxedCol) (show packedCol)++columnEqParity :: Test+columnEqParity = TestCase $ do+    assertBool "packed == boxed" (packedCol == boxedCol)+    assertBool "boxed == packed" (boxedCol == packedCol)+    assertBool "packed == packed" (packedCol == packedFromTexts sampleRows)++extractionParity :: Test+extractionParity =+    TestCase $+        assertEqual+            "toList @Text packed == boxed"+            (DI.toList @T.Text boxedCol)+            (DI.toList @T.Text packedCol)++orderingParity :: Test+orderingParity = TestCase $ do+    let sortedPacked = D.sortBy [D.Asc (F.col @T.Text "k")] packedDf+        sortedBoxed = D.sortBy [D.Asc (F.col @T.Text "k")] boxedDf+    assertBool "asc sort parity" (sortedPacked == sortedBoxed)+    let descPacked = D.sortBy [D.Desc (F.col @T.Text "k")] packedDf+        descBoxed = D.sortBy [D.Desc (F.col @T.Text "k")] boxedDf+    assertBool "desc sort parity" (descPacked == descBoxed)++hashingParity :: Test+hashingParity =+    TestCase $+        assertEqual+            "row hashes equal"+            (Agg.computeRowHashes [0] boxedDf)+            (Agg.computeRowHashes [0] packedDf)++groupByParity :: Test+groupByParity =+    TestCase $+        assertEqual+            "groupBy + distinct parity"+            (D.distinct boxedDf)+            (D.distinct packedDf)++joinParity :: Test+joinParity = TestCase $ do+    let lhsP = D.fromNamedColumns [("k", packedCol)]+        lhsB = D.fromNamedColumns [("k", boxedCol)]+        rhs = D.fromNamedColumns [("k", boxedFromTexts ["apple", "banana", "日本語"])]+        joinedP = D.innerJoin ["k"] lhsP rhs+        joinedB = D.innerJoin ["k"] lhsB rhs+    assertBool "inner join parity" (joinedP == joinedB)++-- A gather (atIndicesStable) on a packed column stays packed and equals the+-- boxed gather. Indices repeat, reorder, and drop rows.+gatherPreservesPacked :: Test+gatherPreservesPacked = TestCase $ do+    let ixs = VU.fromList [9, 0, 0, 5, 7, 2, 3 :: Int]+        gp = DI.atIndicesStable ixs packedCol+        gb = DI.atIndicesStable ixs boxedCol+    assertBool "gathered packed stays PackedText" (DI.isPackedText gp)+    assertBool "gathered packed == gathered boxed" (gp == gb)+    assertEqual+        "gathered packed toList == boxed"+        (DI.toList @T.Text gb)+        (DI.toList @T.Text gp)++-- A sort on a packed column stays packed and equals the boxed sort.+sortPreservesPacked :: Test+sortPreservesPacked = TestCase $ do+    let sortedP = D.sortBy [D.Asc (F.col @T.Text "k")] packedDf+        sortedB = D.sortBy [D.Asc (F.col @T.Text "k")] boxedDf+    assertBool+        "sorted packed column stays PackedText"+        (DI.isPackedText (unsafeGetColumn "k" sortedP))+    assertBool "sorted packed == sorted boxed" (sortedP == sortedB)++-- A left join (sentinel gather, -1 for missing right rows) on a packed right+-- column stays packed, builds the correct null bitmap, and equals boxed.+leftJoinSentinelPreservesPacked :: Test+leftJoinSentinelPreservesPacked = TestCase $ do+    let lhsP = D.fromNamedColumns [("k", packedCol)]+        lhsB = D.fromNamedColumns [("k", boxedCol)]+        rhsP =+            D.fromNamedColumns+                [ ("k", packedFromTexts ["apple", "banana", "日本語"])+                , ("v", packedFromTexts ["RA", "RB", "RC"])+                ]+        rhsB =+            D.fromNamedColumns+                [ ("k", boxedFromTexts ["apple", "banana", "日本語"])+                , ("v", boxedFromTexts ["RA", "RB", "RC"])+                ]+        joinedP = D.leftJoin ["k"] lhsP rhsP+        joinedB = D.leftJoin ["k"] lhsB rhsB+    assertBool+        "left-joined packed right column stays PackedText"+        (DI.isPackedText (unsafeGetColumn "v" joinedP))+    assertBool "left join packed == boxed" (joinedP == joinedB)++tests :: [Test]+tests =+    [ TestLabel "PackedText display parity" displayParity+    , TestLabel "PackedText equality parity" columnEqParity+    , TestLabel "PackedText extraction parity" extractionParity+    , TestLabel "PackedText ordering parity" orderingParity+    , TestLabel "PackedText hashing parity" hashingParity+    , TestLabel "PackedText groupBy parity" groupByParity+    , TestLabel "PackedText join parity" joinParity+    , TestLabel "PackedText gather preserves packed" gatherPreservesPacked+    , TestLabel "PackedText sort preserves packed" sortPreservesPacked+    , TestLabel+        "PackedText left-join sentinel preserves packed"+        leftJoinSentinelPreservesPacked+    ]

@@ -0,0 +1,152 @@+{-# LANGUAGE NumericUnderscores #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++{- | The HARD GATE for the lazy engine: for a bounded source, the lazy result+must be BYTE-IDENTICAL to the eager result computed with the same eager ops+('Join.join' / 'Agg.aggregate' . 'Agg.groupBy' / 'Perm.sortBy').++The lazy executor routes bounded HashJoin/HashAggregate through the whole-frame+eager fast paths, so 'show' of the two results must agree exactly. Both a LEFT+join pipeline (join -> groupBy -> aggregate -> sort) and a pure groupBy pipeline+are covered.+-}+module LazyParity (tests) where++import qualified Data.Map.Strict as M+import Data.Text (Text)+import qualified Data.Text as T+import qualified DataFrame as D+import qualified DataFrame.Functions as F+import qualified DataFrame.IO.CSV as Csv+import qualified DataFrame.Internal.Column as DI+import qualified DataFrame.Internal.Expression as E+import DataFrame.Internal.Schema (Schema (..), schemaType)+import qualified DataFrame.Lazy as L+import DataFrame.Lazy.Internal.LogicalPlan (SortOrder (Descending))+import qualified DataFrame.Operations.Aggregation as Agg+import DataFrame.Operations.Join (JoinType (LEFT))+import qualified DataFrame.Operations.Join as Join+import qualified DataFrame.Operations.Permutation as Perm+import DataFrame.Operators (as, (|>))+import System.Directory (removeFile)+import System.IO.Temp (emptySystemTempFile)+import Test.HUnit++ordersSchema :: Schema+ordersSchema =+    Schema $+        M.fromList+            [ ("order_id", schemaType @Int)+            , ("customer_id", schemaType @Int)+            , ("amount", schemaType @Double)+            , ("discount", schemaType @Double)+            ]++customersSchema :: Schema+customersSchema =+    Schema $+        M.fromList+            [ ("customer_id", schemaType @Int)+            , ("region", schemaType @Text)+            , ("plan", schemaType @Text)+            ]++{- | @n@ orders; ~10% reference a customer id with no matching customer row so+the LEFT join produces a Nothing group (exercises unmatched-row semantics).+-}+ordersFrame :: Int -> D.DataFrame+ordersFrame n =+    D.fromNamedColumns+        [ ("order_id", DI.fromList [0 .. n - 1])+        ,+            ( "customer_id"+            , DI.fromList+                [if i `mod` 10 == 0 then 9_000_000 + i else i `mod` 257 | i <- [0 .. n - 1]]+            )+        , ("amount", DI.fromList [fromIntegral i * 1.5 :: Double | i <- [0 .. n - 1]])+        ,+            ( "discount"+            , DI.fromList [fromIntegral (i `mod` 7) * 0.25 :: Double | i <- [0 .. n - 1]]+            )+        ]++customersFrame :: Int -> D.DataFrame+customersFrame m =+    D.fromNamedColumns+        [ ("customer_id", DI.fromList [0 .. m - 1])+        , ("region", DI.fromList [T.pack ("r" ++ show (i `mod` 4)) | i <- [0 .. m - 1]])+        , ("plan", DI.fromList [T.pack ("p" ++ show (i `mod` 3)) | i <- [0 .. m - 1]])+        ]++-- | Write a frame to a temp CSV and run an action with the path.+withCsv :: D.DataFrame -> (FilePath -> IO a) -> IO a+withCsv df k = do+    csvPath <- emptySystemTempFile "lazy_parity_.csv"+    Csv.writeCsv csvPath df+    r <- k csvPath+    removeFile csvPath+    return r++{- | LEFT join 20k orders to 257 customers -> groupBy region,plan+-> sum(amount-discount), count -> sort revenue desc.+-}+joinPipelineParity :: Test+joinPipelineParity =+    TestCase $+        withCsv (ordersFrame 20_000) $ \ordersPath ->+            withCsv (customersFrame 257) $ \customersPath -> do+                let amount = F.col @Double "amount"+                    discount = F.col @Double "discount"+                    aggs =+                        [ F.sum (amount - discount) `as` "revenue"+                        , F.count amount `as` "orders"+                        ]+                -- Eager: the exact ops the lazy executor delegates to.+                ordersDf <- Csv.readCsvWithSchema ordersSchema ordersPath+                customersDf <- Csv.readCsvWithSchema customersSchema customersPath+                let eager =+                        Perm.sortBy [Perm.Desc (E.Col @Double "revenue")] $+                            Agg.aggregate aggs $+                                Agg.groupBy ["region", "plan"] $+                                    Join.join LEFT ["customer_id"] customersDf ordersDf+                -- Lazy: same query through the bounded-source fast path.+                let customersQ = L.scanCsv customersSchema (T.pack customersPath)+                lazy <-+                    L.scanCsv ordersSchema (T.pack ordersPath)+                        |> (\o -> L.join LEFT "customer_id" "customer_id" o customersQ)+                        |> L.groupBy ["region", "plan"] aggs+                        |> L.sortBy [("revenue", Descending)]+                        |> L.runDataFrame+                assertEqual+                    "join pipeline lazy == eager (byte-identical)"+                    (show eager)+                    (show lazy)++-- | Pure groupBy customer_id -> sum(amount), count, sort desc.+groupByPipelineParity :: Test+groupByPipelineParity =+    TestCase $+        withCsv (ordersFrame 20_000) $ \ordersPath -> do+            let amount = F.col @Double "amount"+                aggs =+                    [ F.sum amount `as` "total"+                    , F.count amount `as` "n"+                    ]+            ordersDf <- Csv.readCsvWithSchema ordersSchema ordersPath+            let eager =+                    Perm.sortBy [Perm.Desc (E.Col @Double "total")] $+                        Agg.aggregate aggs $+                            Agg.groupBy ["customer_id"] ordersDf+            lazy <-+                L.scanCsv ordersSchema (T.pack ordersPath)+                    |> L.groupBy ["customer_id"] aggs+                    |> L.sortBy [("total", Descending)]+                    |> L.runDataFrame+            assertEqual+                "groupBy pipeline lazy == eager (byte-identical)"+                (show eager)+                (show lazy)++tests :: [Test]+tests = [joinPipelineParity, groupByPipelineParity]

@@ -0,0 +1,94 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++{- | The "predict is the model's denotation" claim, discharged as a test rather+than left as a slogan: the @Expr@ that 'predict' compiles must evaluate to the+same numbers as the fitted record's own parameters, computed independently here+in plain Haskell. If @affineExpr@ / @argMinExpr@ ever drift from the record they+are built from, these fail.+-}+module Learn.Denotation (tests) where++import qualified Data.Text as T+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import DataFrame.Internal.Column (TypedColumn (..), toVector)+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Interpreter (interpret)++import DataFrame.KMeans+import DataFrame.LinearModel+import DataFrame.Model (fit, predict)++import Test.HUnit++interpD :: D.DataFrame -> Expr Double -> [Double]+interpD df e = case interpret @Double df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c)+    Left err -> error (show err)++interpI :: D.DataFrame -> Expr Int -> [Int]+interpI df e = case interpret @Int df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c)+    Left err -> error (show err)++df :: D.DataFrame+df =+    D.fromNamedColumns+        [ ("x1", DI.fromList xs1)+        , ("x2", DI.fromList xs2)+        , ("y", DI.fromList [3 + 2 * a - 0.5 * b | (a, b) <- zip xs1 xs2])+        ]+  where+    xs1 = [1, 2, 3, 4, 5, 6, 7, 8] :: [Double]+    xs2 = [2, 1, 4, 3, 6, 5, 8, 7] :: [Double]++col :: D.DataFrame -> T.Text -> [Double]+col d n = interpD d (F.col @Double n)++-- | @interpret (predict m)@ must equal @intercept + Σ coefⱼ·featureⱼ@ from the record.+linearDenotation :: Test+linearDenotation = TestCase $ do+    let m = fit defaultLinearConfig (F.col @Double "y") df+        feats = V.toList (regFeatureNames m)+        coefs = VU.toList (regCoef m)+        cols = map (col df) feats+        native =+            [ regIntercept m + sum (zipWith (*) coefs row)+            | row <- transposeL cols+            ]+        symbolic = interpD df (predict m)+    assertBool+        "linear: predict Expr matches the record's affine prediction"+        (and (zipWith (\a b -> abs (a - b) < 1e-9) native symbolic))++-- | @interpret (predict km)@ must equal the nearest-centroid label from @kmCenters@.+kmeansDenotation :: Test+kmeansDenotation = TestCase $ do+    let feats = ["x1", "x2"]+        km = fit defaultKMeansConfig{kmK = 3, kmSeed = 1} (map (F.col @Double) feats) df+        centers = map VU.toList (V.toList (kmCenters km))+        rows = transposeL (map (col df) feats)+        native = [nearest centers row | row <- rows]+        symbolic = interpI df (predict km)+    assertEqual+        "kmeans: predict Expr matches nearest-centroid label"+        native+        symbolic+  where+    nearest centers row =+        snd (minimum [(sqDist c row, i) | (i, c) <- zip [0 ..] centers])+    sqDist c row = sum [(a - b) ^ (2 :: Int) | (a, b) <- zip c row]++transposeL :: [[a]] -> [[a]]+transposeL [] = []+transposeL xss+    | any null xss = []+    | otherwise = map head xss : transposeL (map tail xss)++tests :: [Test]+tests = [linearDenotation, kmeansDenotation]

@@ -0,0 +1,481 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++{- | Edge-case / degeneracy tests (tests.md category 7) and numerical-stability+tests (category 8) for @dataframe-learn@.++Each test asserts the *mathematically correct* result (computed by hand, with+the derivation in a comment) or a *specific* documented degenerate behaviour --+never "it returned something". Where the library's contract is to clamp/guard a+degeneracy (e.g. @k > n@, constant columns) we assert the concrete clamped+result; where the underlying routine is stable we assert it matches the closed+form a naive implementation would get wrong.+-}+module Learn.EdgeCases (tests) where++import qualified Data.Map.Strict as M+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import DataFrame.Internal.Column (TypedColumn (..), toVector)+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Interpreter (interpret)++import DataFrame.GMM+import DataFrame.KMeans+import DataFrame.LinearAlgebra (logSumExp)+import DataFrame.LinearAlgebra.Eigen (jacobiEigenSym)+import DataFrame.LinearAlgebra.Solve (choleskySolve)+import DataFrame.LinearModel+import DataFrame.LinearSolver (sigmoid)+import DataFrame.Model (fit, predict)+import DataFrame.PCA++import DataFrame.Internal.Statistics (correlation', variance')++import Test.HUnit++-- Helpers (mirrors Learn.Models) --------------------------------------------++interpD :: D.DataFrame -> Expr Double -> [Double]+interpD df e = case interpret @Double df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c)+    Left err -> error (show err)++interpI :: D.DataFrame -> Expr Int -> [Int]+interpI df e = case interpret @Int df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c)+    Left err -> error (show err)++mat :: [[Double]] -> V.Vector (VU.Vector Double)+mat = V.fromList . map VU.fromList++close :: Double -> Double -> Double -> Bool+close tol a b = abs (a - b) <= tol++finite :: Double -> Bool+finite x = not (isNaN x) && not (isInfinite x)++-- ===========================================================================+-- Category 8: numerical stability+-- ===========================================================================++{- logSumExp on a large-positive cluster.+   log(e^1000 + e^1001 + e^1002)+     = 1002 + log(e^-2 + e^-1 + 1)+   e^-2 = 0.1353352832366127, e^-1 = 0.36787944117144233+   sum = 1.5032147243...  ; log(sum) = 0.40760596444...+   => 1002.4076059644...+   A naive `log . sum . map exp` overflows to +Inf here. -}+testLogSumExpLargePositive :: Test+testLogSumExpLargePositive = TestCase $ do+    let lse = logSumExp (VU.fromList [1000, 1001, 1002])+        expected = 1002 + log (exp (-2) + exp (-1) + 1)+    assertBool "lse large-positive is finite" (finite lse)+    assertBool+        "lse [1000,1001,1002] == 1002 + log(e^-2+e^-1+1)"+        (close 1e-9 lse expected)+    -- sanity on the literal so the test pins the actual number, not just the formula+    assertBool "lse literal ~ 1002.40760596" (close 1e-7 lse 1002.4076059644443)++{- logSumExp on a large-negative cluster.+   log(e^-1000 + e^-1001 + e^-1002)+     = -1000 + log(1 + e^-1 + e^-2)  [max is -1000]+     = -1000 + 0.40760596444...+     = -999.59239403556...+   A naive implementation underflows every term to 0 -> log 0 = -Inf. -}+testLogSumExpLargeNegative :: Test+testLogSumExpLargeNegative = TestCase $ do+    let lse = logSumExp (VU.fromList [-1000, -1001, -1002])+        expected = -1000 + log (1 + exp (-1) + exp (-2))+    assertBool "lse large-negative is finite" (finite lse)+    assertBool+        "lse [-1000,-1001,-1002] == -1000 + log(1+e^-1+e^-2)"+        (close 1e-9 lse expected)+    assertBool "lse literal ~ -999.59239403" (close 1e-7 lse (-999.5923940355557))++{- logSumExp of a single element is that element exactly (m + log 1). -}+testLogSumExpSingleton :: Test+testLogSumExpSingleton = TestCase $ do+    assertBool "lse [42] == 42" (close 0 (logSumExp (VU.fromList [42])) 42)++{- sigmoid at extreme arguments must stay in [0,1] and not NaN/Inf.+   sigmoid(1000) is 1 up to rounding (1 - ~0); sigmoid(-1000) is 0.+   The naive 1/(1+exp(-z)) overflows exp(1000)=Inf at z=-1000 -> 1/Inf = 0 ok,+   but exp(-(-1000)) path; the stable branch in DataFrame.LinearSolver.sigmoid+   picks ez/(1+ez) for z<0 so e^-1000 underflows to 0 cleanly -> 0. -}+testSigmoidExtreme :: Test+testSigmoidExtreme = TestCase $ do+    let sp = sigmoid 1000+        sn = sigmoid (-1000)+    assertBool "sigmoid(1000) finite" (finite sp)+    assertBool "sigmoid(-1000) finite" (finite sn)+    assertBool "sigmoid(1000) in [0,1]" (sp >= 0 && sp <= 1)+    assertBool "sigmoid(-1000) in [0,1]" (sn >= 0 && sn <= 1)+    -- saturated values: 1 and 0 to within machine epsilon+    assertBool "sigmoid(1000) == 1" (close 1e-12 sp 1)+    assertBool "sigmoid(-1000) == 0" (close 1e-12 sn 0)+    -- sigmoid(0) is exactly 0.5+    assertBool "sigmoid(0) == 0.5" (close 1e-15 (sigmoid 0) 0.5)+    -- antisymmetry: sigmoid(-z) == 1 - sigmoid(z) at a moderate z+    assertBool+        "sigmoid antisymmetric at 3"+        (close 1e-12 (sigmoid (-3)) (1 - sigmoid 3))++{- Variance of large-but-low-variance data: [1e8+1, 1e8+2, 1e8+3].+   True sample variance (n-1) of {1,2,3} shifted by 1e8 is 1.0 exactly.+   A naive sum-of-squares-minus-square-of-sum formula loses all precision here+   (catastrophic cancellation) -> 0 or negative. Welford keeps it ~1. -}+testVarianceCatastrophicCancellation :: Test+testVarianceCatastrophicCancellation = TestCase $ do+    let xs = VU.fromList [1e8 + 1, 1e8 + 2, 1e8 + 3] :: VU.Vector Double+        v = variance' xs+    assertBool "variance finite" (finite v)+    assertBool "variance is positive" (v > 0)+    -- sample variance of {1,2,3} = ((1-2)^2 + 0 + (3-2)^2)/(3-1) = 2/2 = 1+    assertBool "variance of shifted {1,2,3} == 1.0" (close 1e-6 v 1.0)++{- Variance of an identical column is exactly 0 (not a tiny negative from+   cancellation). -}+testVarianceConstant :: Test+testVarianceConstant = TestCase $ do+    let v = variance' (VU.replicate 100 (7.0 :: Double))+    assertEqual "variance of constant column is 0" 0 v++{- Variance of fewer than two samples is defined to be 0 (computeVariance guard),+   not NaN from a /0. -}+testVarianceSingleton :: Test+testVarianceSingleton = TestCase $ do+    assertEqual+        "variance of one sample is 0"+        0+        (variance' (VU.fromList [3.5 :: Double]))++{- Correlation of a perfectly linear pair is exactly +1 (and -1 reversed),+   computed stably. y = 2x+1 over a spread of x. -}+testCorrelationPerfect :: Test+testCorrelationPerfect = TestCase $ do+    let xs = VU.fromList [1, 2, 3, 4, 5] :: VU.Vector Double+        ys = VU.map (\x -> 2 * x + 1) xs+        yneg = VU.map (\x -> -(2 * x) + 1) xs+    case correlation' xs ys of+        Just r -> assertBool "corr(x, 2x+1) == 1" (close 1e-9 r 1)+        Nothing -> assertFailure "expected a correlation"+    case correlation' xs yneg of+        Just r -> assertBool "corr(x, -2x+1) == -1" (close 1e-9 r (-1))+        Nothing -> assertFailure "expected a correlation"++{- Degeneracy: correlation against a constant column. The denominator+   sqrt(... * (n*Syy - Sy^2)) is 0, so the library returns Just NaN. This pins+   the ACTUAL behaviour: it does not throw, but it does produce a NaN that the+   caller must guard. If a future fix makes it return Nothing or 0 this test+   flags the contract change. -}+testCorrelationConstantColumnIsNaN :: Test+testCorrelationConstantColumnIsNaN = TestCase $ do+    let xs = VU.fromList [1, 2, 3, 4, 5] :: VU.Vector Double+        ys = VU.replicate 5 (9.0 :: Double)+    case correlation' xs ys of+        Just r ->+            assertBool+                "corr with a zero-variance column is NaN (degenerate denom)"+                (isNaN r)+        Nothing -> assertFailure "correlation' returned Nothing (contract changed)"++{- correlation' on n<2 returns Nothing (guarded), not a NaN. -}+testCorrelationTooFew :: Test+testCorrelationTooFew = TestCase $ do+    assertEqual+        "correlation of one point is Nothing"+        Nothing+        (correlation' (VU.fromList [1]) (VU.fromList [2]))++-- ===========================================================================+-- Category 8: stability inside the model expr layer+-- ===========================================================================++{- Logistic probability expressions at extreme feature values must remain in+   [0,1] and finite. The prob expr is 1/(1+exp(-margin)); with a separating+   model and |x| pushed to 1e6 the margin is huge, so a non-stable evaluation+   could overflow. We assert every probability is finite and in [0,1] and that+   the two one-vs-rest class probabilities are present. -}+testLogisticProbsExtremeFeatures :: Test+testLogisticProbsExtremeFeatures = TestCase $ do+    let trainDf =+            D.fromNamedColumns+                [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double]))+                , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int]))+                ]+        m = fit defaultLogisticConfig (F.col @Int "label") trainDf+        probs = logisticProbExprs m+        -- evaluate the class-1 probability at extreme inputs+        extremeDf =+            D.fromNamedColumns+                [("x", DI.fromList ([-1e6, -1, 0, 1, 1e6] :: [Double]))]+    assertEqual "two one-vs-rest probability exprs" 2 (M.size probs)+    let allProbVals =+            concat [interpD extremeDf e | e <- M.elems probs]+    assertBool "all logistic probabilities finite" (all finite allProbVals)+    assertBool+        "all logistic probabilities in [0,1]"+        (all (\p -> p >= 0 && p <= 1) allProbVals)++-- ===========================================================================+-- Category 7: edge cases / degeneracy on models+-- ===========================================================================++{- One-row OLS: a single point (x=2, y=7). With one observation OLS is+   under-determined; the QR design [1, 2] is rank deficient (n=1 < d=2 cols of+   the intercept-augmented matrix) so olsSolve falls back to ridge(1e-8). The+   fitted line must at minimum interpolate the single training point: the+   prediction at x=2 must equal 7 (within tolerance). It must NOT be NaN/Inf. -}+testOLSOneRow :: Test+testOLSOneRow = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([2] :: [Double]))+                , ("y", DI.fromList ([7] :: [Double]))+                ]+        m = fit defaultLinearConfig (F.col @Double "y") df+        preds = interpD df (predict m)+    assertBool "single OLS prediction finite" (all finite preds)+    assertBool+        "OLS fits the single training point"+        (case preds of [p] -> close 1e-3 p 7; _ -> False)++{- All-identical labels in logistic regression: every row is class 1. There is+   exactly one class, so predict must return that class for every row (no+   division-by-zero, no crash, no spurious second class). -}+testLogisticSingleClass :: Test+testLogisticSingleClass = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([-2, -1, 0, 1, 2] :: [Double]))+                , ("label", DI.fromList ([1, 1, 1, 1, 1] :: [Int]))+                ]+        m = fit defaultLogisticConfig (F.col @Int "label") df+        preds = interpI df (predict m)+    assertEqual+        "single-class logistic predicts that class everywhere"+        [1, 1, 1, 1, 1]+        preds++{- Perfectly separable logistic data with a constant (zero-variance) feature+   added. The constant column has variance 0 and is dropped by keptIndices+   (>= 1e-12 threshold); the informative column still separates the classes.+   Prediction must recover the labels exactly and not be corrupted by the+   constant column. -}+testLogisticConstantFeatureIgnored :: Test+testLogisticConstantFeatureIgnored = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double]))+                , ("const", DI.fromList (replicate 8 (5.0 :: Double)))+                , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int]))+                ]+        m = fit defaultLogisticConfig (F.col @Int "label") df+        preds = interpI df (predict m)+    assertEqual+        "logistic ignores constant column and separates"+        [0, 0, 0, 0, 1, 1, 1, 1]+        preds++{- Linear regression with an irrelevant constant feature: the constant column is+   near-constant (variance 0) and must get a zero weight, while the informative+   coefficient is recovered. y = 3x + 4 exactly. -}+testLinearConstantFeatureZeroWeight :: Test+testLinearConstantFeatureZeroWeight = TestCase $ do+    let xs = [1, 2, 3, 4, 5, 6] :: [Double]+        df =+            D.fromNamedColumns+                [ ("x", DI.fromList xs)+                , ("const", DI.fromList (replicate 6 (5.0 :: Double)))+                , ("y", DI.fromList [3 * x + 4 | x <- xs])+                ]+        m = fit defaultLinearConfig (F.col @Double "y") df+        coefs = VU.toList (regCoef m)+        names = V.toList (regFeatureNames m)+        byName = zip names coefs+    -- find the const column weight; it must be ~0 (no information, possibly+    -- collinear with the intercept)+    case lookup "const" byName of+        Just w -> assertBool "constant feature gets ~zero weight" (close 1e-6 w 0)+        Nothing -> assertFailure "const column missing from feature names"+    let preds = interpD df (predict m)+        truth = [3 * x + 4 | x <- xs]+    assertBool+        "regression with constant feature still fits y=3x+4"+        (and (zipWith (close 1e-4) preds truth))++{- k-means with k > n: requesting more clusters than data points. The library+   clamps k = min k (max 1 n), so with 3 rows and kmK=10 we must get exactly 3+   centres (one per point), labels in range, and finite centres -- not a crash+   or empty/duplicate-degenerate centres. -}+testKMeansKGreaterThanN :: Test+testKMeansKGreaterThanN = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("a", DI.fromList ([0, 5, 10] :: [Double]))+                , ("b", DI.fromList ([0, 5, 10] :: [Double]))+                ]+        cfg = defaultKMeansConfig{kmK = 10, kmNInit = 3, kmSeed = 1}+        m = fit cfg [F.col @Double "a", F.col @Double "b"] df+    assertEqual "k clamped to n=3 centres" 3 (V.length (kmCenters m))+    let labels = VU.toList (kmLabels m)+    assertBool "all labels in [0,2]" (all (\l -> l >= 0 && l < 3) labels)+    assertBool+        "all centre coordinates finite"+        (all (VU.all finite) (V.toList (kmCenters m)))+    -- with 3 distinct points and 3 clusters, inertia must be ~0 (each point is+    -- its own centre)+    assertBool "k=n clustering has ~zero inertia" (close 1e-9 (kmInertia m) 0)++{- k-means on an all-identical (zero-variance) feature set: every row is the same+   point. Any clustering has inertia 0; centres must all equal that point and be+   finite (no NaN from dividing by an empty cluster or by a zero distance sum in+   k-means++ sampling). -}+testKMeansAllIdentical :: Test+testKMeansAllIdentical = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("a", DI.fromList (replicate 6 (4.0 :: Double)))+                , ("b", DI.fromList (replicate 6 (4.0 :: Double)))+                ]+        cfg = defaultKMeansConfig{kmK = 2, kmNInit = 3, kmSeed = 1}+        m = fit cfg [F.col @Double "a", F.col @Double "b"] df+    assertBool+        "centres finite on degenerate identical data"+        (all (VU.all finite) (V.toList (kmCenters m)))+    assertBool "inertia is 0 for identical points" (close 1e-12 (kmInertia m) 0)+    -- predict must still assign a valid (finite, in-range) label to every row+    let assigns = interpI df (predict m)+    assertBool "assignments in [0,1]" (all (\l -> l >= 0 && l < 2) assigns)++{- GMM with k > n: clamped like k-means (k = min k (max 1 n)). Two rows, gmmK=5+   must yield 2 components, finite weights summing to ~1, and a finite log+   likelihood -- not a crash or NaN. -}+testGMMKGreaterThanN :: Test+testGMMKGreaterThanN = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("a", DI.fromList ([0, 10] :: [Double]))+                , ("b", DI.fromList ([0, 10] :: [Double]))+                ]+        cfg = defaultGMMConfig{gmmK = 5, gmmSeed = 1}+        m = fit cfg [F.col @Double "a", F.col @Double "b"] df+    assertEqual "GMM k clamped to n=2" 2 (VU.length (gmmWeights m))+    assertBool "GMM weights finite" (VU.all finite (gmmWeights m))+    assertBool "GMM weights sum to ~1" (close 1e-6 (VU.sum (gmmWeights m)) 1)+    assertBool "GMM log-likelihood finite" (finite (gmmLogLikelihood m))++{- PCA on a single informative axis plus a constant column: the constant column+   carries zero variance, so its explained-variance ratio must be ~0 and the+   ratios must still sum to ~1 and stay finite. The first component should align+   with the varying axis. Catches a divide-by-zero in the ratio normalisation+   when one eigenvalue is 0. -}+testPCAConstantColumn :: Test+testPCAConstantColumn = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([-2, -1, 0, 1, 2] :: [Double]))+                , ("const", DI.fromList (replicate 5 (3.0 :: Double)))+                ]+        m =+            fit+                (PCAConfig (NComp 2) False)+                [F.col @Double "x", F.col @Double "const"]+                df+        ratio = VU.toList (pcaExplainedVarianceRatio m)+    assertBool "explained ratios finite" (all finite ratio)+    assertBool "explained ratios sum to ~1" (close 1e-9 (sum ratio) 1)+    -- the variance is entirely along x, so the top ratio must be ~1+    assertBool "first PC explains ~all variance" (close 1e-9 (head ratio) 1)+    -- projection exprs must evaluate to finite numbers+    let es = map snd (pcaExprs m)+    assertBool+        "pca exprs finite on constant column"+        (all (all finite . interpD df) es)++{- PCA on extreme-scale features (1e8-shifted) without standardisation: the+   covariance entries are ~1 (variance of {1,2,3}) despite the 1e8 offset.+   Components must stay unit-length and finite -- catches cancellation in the+   covariance / eigendecomposition at large magnitudes. -}+testPCAExtremeScale :: Test+testPCAExtremeScale = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([1e8 + 1, 1e8 + 2, 1e8 + 3, 1e8 + 4] :: [Double]))+                , ("y", DI.fromList ([1e8 + 1, 1e8 + 2, 1e8 + 3, 1e8 + 4] :: [Double]))+                ]+        m =+            fit+                (PCAConfig (NComp 1) False)+                [F.col @Double "x", F.col @Double "y"]+                df+        comp0 = pcaComponents m V.! 0+        nrm = sqrt (VU.sum (VU.map (^ (2 :: Int)) comp0))+    assertBool "component finite at 1e8 scale" (VU.all finite comp0)+    assertBool "component unit length at 1e8 scale" (close 1e-6 nrm 1)+    assertBool+        "explained ratio finite at 1e8 scale"+        (VU.all finite (pcaExplainedVarianceRatio m))++-- ===========================================================================+-- Category 7/8: linear-algebra degeneracy+-- ===========================================================================++{- choleskySolve on a non-positive-definite (zero) matrix returns Nothing rather+   than crashing or producing NaNs. A 2x2 all-zero matrix has a zero pivot. -}+testCholeskyNonPD :: Test+testCholeskyNonPD = TestCase $ do+    assertEqual+        "cholesky of singular zero matrix is Nothing"+        Nothing+        (choleskySolve (mat [[0, 0], [0, 0]]) (VU.fromList [1, 1]))++{- Jacobi eigendecomposition of a 1x1 matrix (one-column degenerate case):+   eigenvalue is the single entry, eigenvector is [1] (sign-canonicalised). -}+testJacobi1x1 :: Test+testJacobi1x1 = TestCase $ do+    let (ev, vecs) = jacobiEigenSym (mat [[5]])+    assertBool "1x1 eigenvalue is the entry" (close 1e-12 (ev VU.! 0) 5)+    assertBool "1x1 eigenvector is [1]" (close 1e-12 ((vecs V.! 0) VU.! 0) 1)++{- Jacobi on an identity matrix: both eigenvalues are exactly 1 and the result is+   finite (the rotation angle code must not divide by zero when off-diagonals are+   already 0). -}+testJacobiIdentity :: Test+testJacobiIdentity = TestCase $ do+    let (ev, vecs) = jacobiEigenSym (mat [[1, 0], [0, 1]])+    assertBool "identity eigenvalues both 1" (VU.all (close 1e-12 1) ev)+    assertBool "identity eigenvectors finite" (all (VU.all finite) (V.toList vecs))++tests :: [Test]+tests =+    [ testLogSumExpLargePositive+    , testLogSumExpLargeNegative+    , testLogSumExpSingleton+    , testSigmoidExtreme+    , testVarianceCatastrophicCancellation+    , testVarianceConstant+    , testVarianceSingleton+    , testCorrelationPerfect+    , testCorrelationConstantColumnIsNaN+    , testCorrelationTooFew+    , testLogisticProbsExtremeFeatures+    , testOLSOneRow+    , testLogisticSingleClass+    , testLogisticConstantFeatureIgnored+    , testLinearConstantFeatureZeroWeight+    , testKMeansKGreaterThanN+    , testKMeansAllIdentical+    , testGMMKGreaterThanN+    , testPCAConstantColumn+    , testPCAExtremeScale+    , testCholeskyNonPD+    , testJacobi1x1+    , testJacobiIdentity+    ]

@@ -0,0 +1,159 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++module Learn.Ensembles (tests) where++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import DataFrame.Internal.Column (TypedColumn (..), toVector)+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Interpreter (interpret)++import DataFrame.Boosting+import DataFrame.DBSCAN+import DataFrame.GMM+import DataFrame.LinearModel+import DataFrame.LinearSolver (defaultSolverConfig)+import DataFrame.Metrics (r2)+import DataFrame.ModelSelection++import Data.Maybe (isJust, isNothing)+import qualified Data.Vector.Unboxed as VU+import DataFrame.Model (fit, predict)+import Test.HUnit++interpD :: D.DataFrame -> Expr Double -> [Double]+interpD df e = case interpret @Double df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c)+    Left err -> error (show err)++interpI :: D.DataFrame -> Expr Int -> [Int]+interpI df e = case interpret @Int df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c)+    Left err -> error (show err)++clsDF :: D.DataFrame+clsDF =+    D.fromNamedColumns+        [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double]))+        , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int]))+        ]++blobs :: D.DataFrame+blobs =+    D.fromNamedColumns+        [ ("a", DI.fromList ([0, 0.2, -0.1, 0.1, 8, 8.1, 7.9, 8.2] :: [Double]))+        , ("b", DI.fromList ([0, -0.1, 0.2, 0.0, 5, 5.2, 4.9, 5.1] :: [Double]))+        ]++testGBMRegression :: Test+testGBMRegression = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([1 .. 12] :: [Double]))+                ,+                    ( "y"+                    , DI.fromList+                        ( [sin (fromIntegral i) + fromIntegral i * 0.3 | i <- [1 .. 12 :: Int]] ::+                            [Double]+                        )+                    )+                ]+        gb =+            fit+                defaultGBConfig{gbNEstimators = 60, gbMaxDepth = 2}+                (F.col @Double "y")+                df+        preds = interpD df (predict gb)+        truth = interpD df (F.col @Double "y")+        err = sum (zipWith (\p t -> (p - t) ^ (2 :: Int)) preds truth) / 12+    assertBool "GBM fits training data well" (err < 0.05)+    assertBool "trainScore recorded" (VU.length (gbTrainScore gb) == 60)++testGBMStaged :: Test+testGBMStaged = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([1 .. 8] :: [Double]))+                , ("y", DI.fromList ([1 .. 8] :: [Double]))+                ]+        gb = fit defaultGBConfig{gbNEstimators = 10} (F.col @Double "y") df+    assertBool "stage 0 valid" (isJust (gbExprAtStage 0 gb))+    assertBool "out of range stage rejected" (isNothing (gbExprAtStage 11 gb))++testAdaBoost :: Test+testAdaBoost = TestCase $ do+    let m = fit defaultAdaBoostConfig (F.col @Int "label") clsDF+        preds = interpI clsDF (predict m)+    assertEqual "AdaBoost separates" [0, 0, 0, 0, 1, 1, 1, 1] preds++testGMM :: Test+testGMM = TestCase $ do+    let m =+            fit+                defaultGMMConfig{gmmK = 2, gmmSeed = 1}+                [F.col @Double "a", F.col @Double "b"]+                blobs+        assigns = interpI blobs (predict m)+    assertBool "GMM converged" (gmmConverged m)+    assertBool "GMM splits the blobs" (head assigns /= last assigns)+    let m2 =+            fit+                defaultGMMConfig{gmmK = 2, gmmSeed = 1}+                [F.col @Double "a", F.col @Double "b"]+                blobs+    assertEqual "GMM deterministic" (gmmMeans m) (gmmMeans m2)+    assertBool "BIC finite" (not (isNaN (gmmBIC m)) && not (isInfinite (gmmBIC m)))++testDBSCAN :: Test+testDBSCAN = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("a", DI.fromList ([0, 0.1, 0.2, 5, 5.1, 5.2, 50] :: [Double]))+                , ("b", DI.fromList ([0, 0.1, 0.0, 5, 5.0, 5.1, 50] :: [Double]))+                ]+        m = fit (DBSCANConfig 1.0 2) [F.col @Double "a", F.col @Double "b"] df+    assertEqual "two clusters" 2 (dbNClusters m)+    assertEqual "last point is noise" (-1) (VU.last (dbLabels m))+    let surrogate =+            dbscanSurrogateExpr+                D.defaultTreeConfig+                [F.col @Double "a", F.col @Double "b"]+                m+                df+        preds = interpI df surrogate+    assertEqual+        "surrogate matches non-noise labels on cores"+        (take 6 (VU.toList (dbLabels m)))+        (take 6 preds)++testGridSearch :: Test+testGridSearch = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([1 .. 40] :: [Double]))+                ,+                    ( "y"+                    , DI.fromList ([2 * fromIntegral i + 5 | i <- [1 .. 40 :: Int]] :: [Double])+                    )+                ]+        score alpha train test =+            let mdl =+                    fit (LinearConfig (Ridge alpha) defaultSolverConfig) (F.col @Double "y") train+             in r2+                    (VU.fromList (interpD test (predict mdl)))+                    (VU.fromList (interpD test (F.col @Double "y")))+        res = gridSearch 4 7 [0.0, 1.0, 100.0] score df+    assertBool "best score high" (gsBestScore res > 0.99)+    assertEqual "all configs scored" 3 (length (gsAll res))++tests :: [Test]+tests =+    [ testGBMRegression+    , testGBMStaged+    , testAdaBoost+    , testGMM+    , testDBSCAN+    , testGridSearch+    ]

@@ -0,0 +1,366 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++{- | Metamorphic, invariance, and abstraction-law tests for the ML library.++These check how the fitted model and the metrics behave under transformations of+the *input* — duplicate / permute / scale / rename / add-constant-feature — where+the math dictates the answer. Each test computes the model on two genuinely+different (transformed) frames and asserts they agree, or pins a metric/scaler to+its defining law. None compares a value to itself; each can fail if the library+is wrong.+-}+module Learn.Metamorphic (tests) where++import qualified Data.Text as T+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import DataFrame.Internal.Column (TypedColumn (..), toVector)+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Interpreter (interpret)++import DataFrame.LinearModel+import DataFrame.Metrics+import DataFrame.Model (fit, predict)+import DataFrame.Operations.Merge ()++-- Semigroup DataFrame (row concatenation)+import DataFrame.Transform++import Test.HUnit++-- Helpers -------------------------------------------------------------------++interpD :: D.DataFrame -> Expr Double -> [Double]+interpD df e = case interpret @Double df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c)+    Left err -> error (show err)++close :: Double -> Double -> Double -> Bool+close tol a b = abs (a - b) <= tol++closeList :: Double -> [Double] -> [Double] -> Bool+closeList tol as bs = length as == length bs && and (zipWith (close tol) as bs)++-- Build a regression frame from explicit feature/target lists.+mkRegDF :: [(T.Text, [Double])] -> [Double] -> D.DataFrame+mkRegDF feats ys =+    D.fromNamedColumns+        ([(n, DI.fromList vs) | (n, vs) <- feats] ++ [("y", DI.fromList ys)])++-- Base design: a noiseless linear target so OLS recovers a unique solution.+baseX1, baseX2, baseY :: [Double]+baseX1 = [1, 2, 3, 4, 5, 6, 7, 8]+baseX2 = [2, 1, 4, 3, 6, 5, 8, 7]+baseY = [3 + 2 * a - 0.5 * b | (a, b) <- zip baseX1 baseX2]++baseDF :: D.DataFrame+baseDF = mkRegDF [("x1", baseX1), ("x2", baseX2)] baseY++fitBase :: D.DataFrame -> LinearRegressor+fitBase = fit defaultLinearConfig (F.col @Double "y")++-- Metamorphic: duplicate rows ----------------------------------------------++{- | OLS minimises Σ(y-ŷ)²; duplicating every row scales the loss by 2 but leaves+the argmin (the coefficients/intercept) unchanged. Catches a solver that+weights rows by frame size or mishandles the normal equations.+-}+testDuplicateRows :: Test+testDuplicateRows = TestCase $ do+    let dupDF =+            mkRegDF+                [("x1", baseX1 ++ baseX1), ("x2", baseX2 ++ baseX2)]+                (baseY ++ baseY)+        m0 = fitBase baseDF+        m1 = fitBase dupDF+    assertBool+        "duplicate rows: coefficients unchanged"+        (closeList 1e-9 (VU.toList (regCoef m0)) (VU.toList (regCoef m1)))+    assertBool+        "duplicate rows: intercept unchanged"+        (close 1e-9 (regIntercept m0) (regIntercept m1))++-- Metamorphic: permute rows -------------------------------------------------++{- | OLS is invariant to row order. We reverse the rows (a genuine non-identity+permutation) and require identical coefficients. Catches any order-dependence+in matrix assembly or solving.+-}+testPermuteRows :: Test+testPermuteRows = TestCase $ do+    let permDF =+            mkRegDF+                [("x1", reverse baseX1), ("x2", reverse baseX2)]+                (reverse baseY)+        m0 = fitBase baseDF+        m1 = fitBase permDF+    assertBool+        "permute rows: coefficients unchanged"+        (closeList 1e-9 (VU.toList (regCoef m0)) (VU.toList (regCoef m1)))+    assertBool+        "permute rows: intercept unchanged"+        (close 1e-9 (regIntercept m0) (regIntercept m1))++-- Metamorphic: scale a feature ----------------------------------------------++{- | Scaling feature x1 by k must scale that coefficient by 1/k while leaving+predictions (and every other coefficient + the intercept) invariant. Both+sides are genuine fits on different frames. Catches a regressor that drops or+mis-applies a feature's scale.+-}+testScaleFeature :: Test+testScaleFeature = TestCase $ do+    let k = 10.0+        scaledDF = mkRegDF [("x1", map (* k) baseX1), ("x2", baseX2)] baseY+        m0 = fitBase baseDF+        m1 = fitBase scaledDF+        coef0 = VU.toList (regCoef m0)+        coef1 = VU.toList (regCoef m1)+    assertBool+        "scale feature: x1 coefficient scaled by 1/k"+        (close 1e-7 (head coef0 / k) (head coef1))+    assertBool+        "scale feature: x2 coefficient unchanged"+        (close 1e-7 (coef0 !! 1) (coef1 !! 1))+    assertBool+        "scale feature: intercept unchanged"+        (close 1e-7 (regIntercept m0) (regIntercept m1))+    -- Predictions on each model's own (consistently transformed) frame agree.+    let p0 = interpD baseDF (predict m0)+        p1 = interpD scaledDF (predict m1)+    assertBool "scale feature: predictions invariant" (closeList 1e-6 p0 p1)++-- Metamorphic: rename columns -----------------------------------------------++{- | Renaming feature columns is a pure relabelling: coefficients, intercept, and+predictions are identical and the fitted feature names track the rename.+Catches any hidden dependence on specific column-name strings.+-}+testRenameColumns :: Test+testRenameColumns = TestCase $ do+    let renamedDF = mkRegDF [("feat_a", baseX1), ("feat_b", baseX2)] baseY+        m0 = fitBase baseDF+        m1 = fitBase renamedDF+    assertBool+        "rename: coefficients unchanged"+        (closeList 1e-9 (VU.toList (regCoef m0)) (VU.toList (regCoef m1)))+    assertBool+        "rename: intercept unchanged"+        (close 1e-9 (regIntercept m0) (regIntercept m1))+    assertEqual+        "rename: fitted feature names follow the rename"+        ["feat_a", "feat_b"]+        (V.toList (regFeatureNames m1))+    -- predict on the renamed frame yields the same numbers as on the original.+    let p0 = interpD baseDF (predict m0)+        p1 = interpD renamedDF (predict m1)+    assertBool "rename: predictions unchanged" (closeList 1e-9 p0 p1)++-- Metamorphic: irrelevant constant feature ----------------------------------++{- | A constant feature carries no variance, so OLS predictions for the real+features must be unchanged when one is added. (Its own weight is absorbed into+the intercept and is not separately identifiable, so we pin predictions, not+the weight.) Catches a fit whose answer drifts when a degenerate column is+present.+-}+testConstantFeatureIgnored :: Test+testConstantFeatureIgnored = TestCase $ do+    let withConst =+            mkRegDF+                [ ("x1", baseX1)+                , ("x2", baseX2)+                , ("c", replicate (length baseX1) 7.0)+                ]+                baseY+        m0 = fitBase baseDF+        m1 = fitBase withConst+        p0 = interpD baseDF (predict m0)+        p1 = interpD withConst (predict m1)+    assertBool+        "constant feature: predictions match the no-constant model"+        (closeList 1e-6 p0 p1)+    -- And the real-feature coefficients are unchanged.+    let coef1 = VU.toList (regCoef m1)+    assertBool+        "constant feature: x1 coefficient unchanged"+        (close 1e-6 (head (VU.toList (regCoef m0))) (head coef1))+    assertBool+        "constant feature: x2 coefficient unchanged"+        (close 1e-6 (VU.toList (regCoef m0) !! 1) (coef1 !! 1))++-- Metamorphic: split then concatenate --------------------------------------++{- | Genuinely split the frame into two row-range sub-frames and concatenate them+back with the DataFrame @<>@ (row concatenation). The rebuilt frame has the+same rows, so the fit must be identical — but this actually exercises the+concatenation code path, so it catches a @<>@ that drops, duplicates, or+mis-aligns rows/columns (a plain @take h ++ drop h@ would be the identity and+test nothing).+-}+testSplitConcat :: Test+testSplitConcat = TestCase $ do+    let n = length baseX1+        h = n `div` 2+        half lo hi =+            mkRegDF+                [ ("x1", slice lo hi baseX1)+                , ("x2", slice lo hi baseX2)+                ]+                (slice lo hi baseY)+        slice lo hi = take (hi - lo) . drop lo+        rebuiltDF = half 0 h <> half h n+        m0 = fitBase baseDF+        m1 = fitBase rebuiltDF+    -- the concatenated frame must have exactly the original rows back+    assertEqual "split/concat: row count preserved" n (fst (D.dimensions rebuiltDF))+    assertBool+        "split/concat: coefficients unchanged"+        (closeList 1e-9 (VU.toList (regCoef m0)) (VU.toList (regCoef m1)))+    assertBool+        "split/concat: intercept unchanged"+        (close 1e-9 (regIntercept m0) (regIntercept m1))++-- Training/inference parity: column order ----------------------------------++{- | The order of feature columns in the frame must not change predictions.+We fit on a frame whose columns are declared in the opposite order and require+the prediction expression to produce the same numbers on the original frame.+(predict compiles to a name-keyed affine Expr, so it should be order-free.)+Catches a positional, rather than name-keyed, feature mapping.+-}+testColumnOrderParity :: Test+testColumnOrderParity = TestCase $ do+    let swappedDF = mkRegDF [("x2", baseX2), ("x1", baseX1)] baseY+        m0 = fitBase baseDF+        m1 = fitBase swappedDF+        -- evaluate both fitted models on the SAME original frame.+        p0 = interpD baseDF (predict m0)+        p1 = interpD baseDF (predict m1)+    assertBool+        "column order: predictions on the same frame agree"+        (closeList 1e-7 p0 p1)++-- Law: standardScaler output has mean ~0, std ~1 ---------------------------++{- | The defining law of a standard scaler: after transforming, each scaled+column has sample mean ≈ 0 and (population) std ≈ 1. We recompute the moments+here in plain Haskell from the transformed frame — not via the scaler — so a+wrong denominator or a centring bug is caught.+-}+testStandardScalerLaw :: Test+testStandardScalerLaw = TestCase $ do+    let cols = ["x1", "x2"]+        scaler = standardScaler cols baseDF+        scaledDF = applyTransform (scalerTransform scaler) baseDF+        moments xs =+            let n = fromIntegral (length xs)+                mu = sum xs / n+                var = sum [(x - mu) ^ (2 :: Int) | x <- xs] / n+             in (mu, sqrt var)+    mapM_+        ( \c -> do+            let (mu, sd) = moments (interpD scaledDF (F.col @Double c))+            assertBool ("scaler: " ++ T.unpack c ++ " mean ~ 0") (close 1e-9 mu 0)+            assertBool ("scaler: " ++ T.unpack c ++ " std ~ 1") (close 1e-9 sd 1)+        )+        cols++{- | The scaler model's stored stats must match the data's own moments: a guard+against the scaler storing the wrong mean/std even if transform happens to+look plausible. Computed independently from the raw columns.+-}+testScalerStatsMatchData :: Test+testScalerStatsMatchData = TestCase $ do+    let scaler = standardScaler ["x1", "x2"] baseDF+        moments xs =+            let n = fromIntegral (length xs)+                mu = sum xs / n+             in (mu, sqrt (sum [(x - mu) ^ (2 :: Int) | x <- xs] / n))+        (mu1, sd1) = moments baseX1+        (mu2, sd2) = moments baseX2+    assertBool+        "scaler means match data"+        (closeList 1e-9 (VU.toList (smMeans scaler)) [mu1, mu2])+    assertBool+        "scaler stds match data"+        (closeList 1e-9 (VU.toList (smStds scaler)) [sd1, sd2])++-- Metric laws --------------------------------------------------------------++{- | Perfect prediction → accuracy exactly 1.0; a single deliberate miss drops it+below 1. Pins both ends so a metric that ignores its inputs can't pass.+-}+testAccuracyLaw :: Test+testAccuracyLaw = TestCase $ do+    let truth = VU.fromList [0, 1, 2, 1, 0, 2]+        perfect = truth+        oneWrong = VU.fromList [0, 1, 2, 1, 0, 0]+    assertBool "accuracy: perfect = 1" (close 1e-12 (accuracy perfect truth) 1.0)+    assertBool+        "accuracy: one miss out of 6 = 5/6"+        (close 1e-12 (accuracy oneWrong truth) (5 / 6))+    assertBool+        "accuracy in [0,1]"+        (let a = accuracy oneWrong truth in a >= 0 && a <= 1)++{- | Accuracy is permutation-invariant: applying the same permutation to preds+and truth leaves it unchanged. Both vectors are genuinely reordered.+-}+testAccuracyPermInvariant :: Test+testAccuracyPermInvariant = TestCase $ do+    let preds = VU.fromList [0, 0, 1, 1, 2, 2, 1, 0]+        truth = VU.fromList [0, 0, 1, 2, 2, 2, 1, 0]+        a0 = accuracy preds truth+        a1 = accuracy (VU.reverse preds) (VU.reverse truth)+    assertBool "accuracy: permutation invariant" (close 1e-12 a0 a1)+    -- Sanity: the metric is non-trivial here (not 0 or 1), so invariance is meaningful.+    assertBool "accuracy: non-degenerate baseline" (a0 > 0 && a0 < 1)++{- | r² of a perfect fit is exactly 1; r² of predicting the constant mean is+exactly 0. These are the two anchor points of the R² definition. Catches a+swapped SS_res/SS_tot or a wrong sign.+-}+testR2Anchors :: Test+testR2Anchors = TestCase $ do+    let truth = VU.fromList [1, 3, 2, 8, 5, 4]+        meanT = VU.sum truth / fromIntegral (VU.length truth)+        meanPred = VU.replicate (VU.length truth) meanT+    assertBool "r2: perfect fit = 1" (close 1e-12 (r2 truth truth) 1.0)+    assertBool "r2: predicting the mean = 0" (close 1e-12 (r2 meanPred truth) 0.0)+    -- A strictly-worse-than-mean prediction gives r2 < 0 (not clamped).+    let worse = VU.map (+ 100) truth+    assertBool "r2: far-off prediction is negative" (r2 worse truth < 0)++{- | r² on a fitted noiseless linear model equals 1, and rmse equals 0. Ties the+metric law to the model: if the fit is exact, the metric must say so.+-}+testR2OfFittedModel :: Test+testR2OfFittedModel = TestCase $ do+    let m = fitBase baseDF+        score = evaluate r2 (predict m) (F.col @Double "y") baseDF+        err = evaluate rmse (predict m) (F.col @Double "y") baseDF+    assertBool "r2 of exact linear fit ~ 1" (close 1e-9 score 1.0)+    assertBool "rmse of exact linear fit ~ 0" (err < 1e-6)++tests :: [Test]+tests =+    [ testDuplicateRows+    , testPermuteRows+    , testScaleFeature+    , testRenameColumns+    , testConstantFeatureIgnored+    , testSplitConcat+    , testColumnOrderParity+    , testStandardScalerLaw+    , testScalerStatsMatchData+    , testAccuracyLaw+    , testAccuracyPermInvariant+    , testR2Anchors+    , testR2OfFittedModel+    ]

@@ -0,0 +1,121 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++module Learn.MetricsTests (tests) where++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import qualified DataFrame.Internal.Column as DI++import DataFrame.LinearModel+import DataFrame.LinearSolver (defaultSolverConfig)+import DataFrame.Metrics+import DataFrame.Metrics.Report+import DataFrame.ModelSelection+import DataFrame.PCA+import DataFrame.Transform++import qualified Data.Vector.Unboxed as VU+import DataFrame.Model (fit, predict)+import Test.HUnit++close :: Double -> Double -> Double -> Bool+close tol a b = abs (a - b) <= tol++preds3, truth3 :: VU.Vector Double+preds3 = VU.fromList [0, 0, 1, 1, 2, 2, 1, 0]+truth3 = VU.fromList [0, 0, 1, 2, 2, 2, 1, 0]++reg :: D.DataFrame+reg =+    D.fromNamedColumns+        [ ("x", DI.fromList ([1 .. 20] :: [Double]))+        ,+            ( "y"+            , DI.fromList ([2 * fromIntegral i + 1 | i <- [1 .. 20 :: Int]] :: [Double])+            )+        ]++testRegressionMetrics :: Test+testRegressionMetrics = TestCase $ do+    let p = VU.fromList [1, 2, 3, 4]+        t = VU.fromList [1, 2, 3, 5]+    assertBool "mse" (close 1e-9 (mse p t) 0.25)+    assertBool "rmse" (close 1e-9 (rmse p t) 0.5)+    assertBool "mae" (close 1e-9 (mae p t) 0.25)+    assertBool "r2 in range" (r2 p t <= 1)++testMulticlassMetrics :: Test+testMulticlassMetrics = TestCase $ do+    assertBool "accuracy" (close 1e-9 (accuracy preds3 truth3) 0.875)+    -- class 1: tp=2 (idx2,6), fp=1 (idx3) -> precision 2/3+    assertBool+        "binary precision class 1"+        (close 1e-9 (precision (Binary 1) preds3 truth3) (2 / 3))+    assertBool+        "macro f1 sane"+        (f1 Macro preds3 truth3 > 0.8 && f1 Macro preds3 truth3 <= 1)+    -- micro f1 == accuracy for single-label+    assertBool+        "micro f1 == accuracy"+        (close 1e-9 (f1 Micro preds3 truth3) (accuracy preds3 truth3))++testRocAuc :: Test+testRocAuc = TestCase $ do+    let scores = VU.fromList [0.1, 0.4, 0.35, 0.8]+        truth = VU.fromList [0, 0, 1, 1]+    assertBool "perfect-ish auc high" (rocAuc scores truth >= 0.75)+    assertBool "auc in [0,1]" (let a = rocAuc scores truth in a >= 0 && a <= 1)++testReports :: Test+testReports = TestCase $ do+    let cr = classificationReport preds3 truth3+    assertEqual "report covers 3 classes" 3 (length (crPerClass cr))+    assertBool "report accuracy" (close 1e-9 (crAccuracy cr) 0.875)+    let rr = regressionReport (VU.fromList [1, 2, 3]) (VU.fromList [1, 2, 4])+    assertBool "regression report rmse" (rrRMSE rr > 0)+    -- Show instances don't crash+    assertBool "classification report shows" (not (null (show cr)))+    assertBool "confusion shows" (not (null (show (confusionMatrix preds3 truth3))))++testEvaluateOneLiner :: Test+testEvaluateOneLiner = TestCase $ do+    let m = fit defaultLinearConfig (F.col @Double "y") reg+        score = evaluate rmse (predict m) (F.col @Double "y") reg+    assertBool "evaluate rmse ~ 0 on exact linear fit" (score < 1e-6)+    let r = regressionReportExpr (predict m) (F.col @Double "y") reg+    assertBool "report r2 ~ 1" (close 1e-6 (rrR2 r) 1)++testCrossValidate :: Test+testCrossValidate = TestCase $ do+    let cv =+            crossValidate+                4+                0+                r2+                (F.col @Double "y")+                (predict . fit defaultLinearConfig (F.col @Double "y"))+                reg+    assertBool "all folds high R2" (all (> 0.99) cv)+    assertEqual "four folds" 4 (length cv)++testTransformCompose :: Test+testTransformCompose = TestCase $ do+    let scaler = standardScaler ["x"] reg+        pca = fit (PCAConfig (NComp 1) False) [F.col @Double "x"] reg+        -- the whole point: scaler <> pcaTransform compose as a monoid+        pipeline = scalerTransform scaler <> pcaTransform pca+        out = applyTransform pipeline reg+    assertBool "pipeline produced a frame" (D.columnNames out /= [])+    assertBool "pipeline has pc1 column" ("pc1" `elem` D.columnNames out)++tests :: [Test]+tests =+    [ testRegressionMetrics+    , testMulticlassMetrics+    , testRocAuc+    , testReports+    , testEvaluateOneLiner+    , testCrossValidate+    , testTransformCompose+    ]

@@ -0,0 +1,169 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++module Learn.Models (tests) where++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import DataFrame.Internal.Column (TypedColumn (..), toVector)+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Interpreter (interpret)++import DataFrame.DecisionTree.Model+import DataFrame.DecisionTree.Regression+import DataFrame.KMeans+import DataFrame.LinearModel+import DataFrame.LinearSolver (SolverConfig (..), defaultSolverConfig)+import DataFrame.PCA+import DataFrame.SVM+import DataFrame.Transform++import qualified Data.Map.Strict as M+import qualified Data.Text as T+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import DataFrame.Model (fit, predict)+import Test.HUnit++interpD :: D.DataFrame -> Expr Double -> [Double]+interpD df e = case interpret @Double df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c)+    Left err -> error (show err)++interpI :: D.DataFrame -> Expr Int -> [Int]+interpI df e = case interpret @Int df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c)+    Left err -> error (show err)++close :: Double -> Double -> Double -> Bool+close tol a b = abs (a - b) <= tol++regDF :: D.DataFrame+regDF =+    D.fromNamedColumns+        [ ("x1", DI.fromList xs1)+        , ("x2", DI.fromList xs2)+        , ("y", DI.fromList [2 * a - 3 * b + 1 | (a, b) <- zip xs1 xs2])+        ]+  where+    xs1 = [1, 2, 3, 4, 5, 6, 7, 8] :: [Double]+    xs2 = [2, 1, 4, 3, 6, 5, 8, 7] :: [Double]++clsDF :: D.DataFrame+clsDF =+    D.fromNamedColumns+        [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double]))+        , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int]))+        ]++testOLS :: Test+testOLS = TestCase $ do+    let m = fit defaultLinearConfig (F.col @Double "y") regDF+    assertBool+        "OLS coef ~ [2,-3]"+        (and (zipWith (close 1e-6) (VU.toList (regCoef m)) [2, -3]))+    assertBool "OLS intercept ~ 1" (close 1e-6 (regIntercept m) 1)+    let preds = interpD regDF (predict m)+        truth = interpD regDF (F.col @Double "y")+    assertBool "OLS expr matches y" (and (zipWith (close 1e-6) preds truth))++testRidgeShrinks :: Test+testRidgeShrinks = TestCase $ do+    let r0 =+            fit (LinearConfig (Ridge 0.01) defaultSolverConfig) (F.col @Double "y") regDF+        r1 = fit (LinearConfig (Ridge 100) defaultSolverConfig) (F.col @Double "y") regDF+        norm = VU.sum . VU.map abs . regCoef+    assertBool "stronger ridge shrinks coefficients" (norm r1 < norm r0)++testLogistic :: Test+testLogistic = TestCase $ do+    let m = fit defaultLogisticConfig (F.col @Int "label") clsDF+        dec = predict m+        preds = interpI clsDF dec+        truth = [0, 0, 0, 0, 1, 1, 1, 1]+    assertEqual "logistic separates" truth preds+    let probs = logisticProbExprs m+    assertBool "prob exprs present for both classes" (M.size probs == 2)++testSVC :: Test+testSVC = TestCase $ do+    let m = fit defaultSVCConfig (F.col @Int "label") clsDF+        preds = interpI clsDF (predict m)+    assertEqual "linear SVC separates" [0, 0, 0, 0, 1, 1, 1, 1] preds++testRegressionTree :: Test+testRegressionTree = TestCase $ do+    let m = fit defaultRegTreeConfig (F.col @Double "y") regDF+        preds = interpD regDF (predict m)+        truth = interpD regDF (F.col @Double "y")+        sse = sum (zipWith (\p t -> (p - t) ^ (2 :: Int)) preds truth)+    assertBool "regression tree reduces error" (sse < 200)+    assertBool "tree has leaves" (dtrNLeaves m >= 2)++testClassifierStats :: Test+testClassifierStats = TestCase $ do+    let m = fit D.defaultTreeConfig (F.col @Int "label") clsDF+    assertBool "classifier depth >= 1" (dtcDepth m >= 1)++testPCA :: Test+testPCA = TestCase $ do+    let m =+            fit+                (PCAConfig (NComp 2) False)+                [F.col @Double "x1", F.col @Double "x2"]+                regDF+        ratio = VU.toList (pcaExplainedVarianceRatio m)+    assertBool "explained ratio sums ~1" (close 1e-9 (sum ratio) 1)+    assertEqual "two components" 2 (V.length (pcaComponents m))+    let comp0 = pcaComponents m V.! 0+        nrm = sqrt (VU.sum (VU.map (^ (2 :: Int)) comp0))+    assertBool "component is unit length" (close 1e-9 nrm 1)+    let es = map snd (pcaExprs m)+    assertBool+        "pca exprs evaluate finite"+        (not (any (any isNaN . interpD regDF) es))++testKMeans :: Test+testKMeans = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("a", DI.fromList ([0, 0.1, 0.2, 10, 10.1, 10.2] :: [Double]))+                , ("b", DI.fromList ([0, 0.1, 0, 10, 10, 10.1] :: [Double]))+                ]+        cfg = defaultKMeansConfig{kmK = 2, kmNInit = 5, kmSeed = 1}+        m = fit cfg [F.col @Double "a", F.col @Double "b"] df+        labels = VU.toList (kmLabels m)+    assertBool "two blobs split" (head labels /= last labels)+    assertEqual "two centers" 2 (V.length (kmCenters m))+    let m2 = fit cfg [F.col @Double "a", F.col @Double "b"] df+    assertEqual "kmeans deterministic" (kmCenters m) (kmCenters m2)+    let assigns = interpI df (predict m)+    assertEqual "assign expr matches labels" labels assigns++testTransformCompose :: Test+testTransformCompose = TestCase $ do+    let scaler = standardScaler ["x1", "x2"] regDF+        t = scalerTransform scaler+        scaledDf = applyTransform t regDF+        -- fit model on scaled features, then compile scaling into the expr+        m = fit defaultLinearConfig (F.col @Double "y") scaledDf+        composed = compileThrough t (predict m)+        viaCompose = interpD regDF composed+        viaStepwise = interpD scaledDf (predict m)+    assertBool+        "compileThrough == stepwise apply"+        (and (zipWith (close 1e-6) viaCompose viaStepwise))++tests :: [Test]+tests =+    [ testOLS+    , testRidgeShrinks+    , testLogistic+    , testSVC+    , testRegressionTree+    , testClassifierStats+    , testPCA+    , testKMeans+    , testTransformCompose+    ]

@@ -0,0 +1,438 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++{- | Numerical-rigor suite: gradient checks (cat 4), reproducibility across the+stochastic models (cat 16), and statistical / distributional properties of the+RNG and splitters (cat 5). Every test is constructed to FAIL on a real bug — no+@assertEqual v v@, no rubber tolerances.+-}+module Learn.NumericalRigor (tests) where++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import qualified DataFrame.Internal.Column as DI++import DataFrame.GMM+import DataFrame.KMeans+import DataFrame.LinearSolver.Loss (+    SmoothLoss (..),+    logisticLoss,+    sigmoid,+    sqHingeLoss,+    squaredLoss,+ )+import DataFrame.Model (fit)+import DataFrame.ModelSelection (trainTestSplit)+import DataFrame.Random+import DataFrame.SVM.RFF++import qualified Data.Vector.Unboxed as VU+import Test.HUnit++-- ---------------------------------------------------------------------------+-- Independent reference loss VALUE functions.+--+-- The library only exposes the gradient 'slGradZ'; the per-loss scalar value is+-- reconstructed here straight from the module's own documented definitions, so+-- the finite difference below is computed INDEPENDENTLY of the analytic+-- gradient (anti-tautology: never finite-difference a gradient against itself).+-- ---------------------------------------------------------------------------++-- | @½ (z - y)²@  (squaredLoss).+squaredValue :: Double -> Double -> Double+squaredValue y z = 0.5 * (z - y) * (z - y)++-- | @log (1 + exp (-y z))@  (logisticLoss). softplus form, numerically stable.+logisticValue :: Double -> Double -> Double+logisticValue y z =+    let m = negate (y * z)+     in if m >= 0 then m + log1pExp (negate m) else log1pExp m+  where+    log1pExp x = log (1 + exp x)++-- | @(max 0 (1 - y z))²@  (sqHingeLoss).+sqHingeValue :: Double -> Double -> Double+sqHingeValue y z = let m = 1 - y * z in if m > 0 then m * m else 0++-- | Central finite difference of @f@ in @z@ at @(y, z)@.+centralDiff ::+    (Double -> Double -> Double) -> Double -> Double -> Double -> Double+centralDiff f eps y z = (f y (z + eps) - f y (z - eps)) / (2 * eps)++{- | Gradient-check one 'SmoothLoss' against its reference value fn over a grid+of @(y, z)@ points (both labels in @{ -1, +1 }@, many margins including the hinge+kink neighbourhood). Fails if the analytic gradient differs from the finite+difference anywhere beyond @tol@.+-}+gradCheck ::+    String ->+    SmoothLoss ->+    (Double -> Double -> Double) ->+    Double ->+    Test+gradCheck nm loss valueFn tol =+    TestCase $+        mapM_ check [(y, z) | y <- [-1, 1], z <- zs]+  where+    -- margins, deliberately avoiding the exact hinge kink z = y (m = 0) where+    -- the squared-hinge second derivative is discontinuous; central diff is+    -- still valid away from the kink.+    zs = [-3.7, -2.1, -1.25, -0.6, -0.15, 0.22, 0.55, 1.4, 1.9, 2.8, 4.3]+    check (y, z) = do+        let analytic = slGradZ loss y z+            fd = centralDiff valueFn 1e-6 y z+            ok = abs (analytic - fd) <= tol * (1 + abs fd)+        assertBool+            ( nm+                ++ " grad mismatch at (y="+                ++ show y+                ++ ", z="+                ++ show z+                ++ "): analytic="+                ++ show analytic+                ++ " finite-diff="+                ++ show fd+            )+            ok++-- | The squared-loss gradient @z - y@ checked against ½(z-y)².+testSquaredGradient :: Test+testSquaredGradient = gradCheck "squared" squaredLoss squaredValue 1e-5++-- | The logistic gradient @-y·σ(-y z)@ checked against log(1+exp(-yz)).+testLogisticGradient :: Test+testLogisticGradient = gradCheck "logistic" logisticLoss logisticValue 1e-5++-- | The squared-hinge gradient @-2y·max(0,1-yz)@ checked against (max 0 (1-yz))².+testSqHingeGradient :: Test+testSqHingeGradient = gradCheck "squared_hinge" sqHingeLoss sqHingeValue 1e-5++{- | Sign sanity that cannot be passed by a self-referential check: at a point+where the loss is strictly increasing in @z@ the analytic gradient must be+positive (and vice versa). Catches a flipped-sign gradient even if its+magnitude were somehow right. logistic with y=+1 decreases in z, so grad < 0;+squared with z>y increases, so grad > 0.+-}+testGradientSigns :: Test+testGradientSigns = TestCase $ do+    assertBool+        "squared: grad>0 when z>y (loss rising)"+        (slGradZ squaredLoss 1.0 3.0 > 0)+    assertBool+        "squared: grad<0 when z<y (loss falling)"+        (slGradZ squaredLoss 1.0 (-2.0) < 0)+    assertBool+        "logistic y=+1: grad<0 (more positive margin lowers loss)"+        (slGradZ logisticLoss 1.0 0.5 < 0)+    assertBool+        "logistic y=-1: grad>0"+        (slGradZ logisticLoss (-1.0) 0.5 > 0)+    assertBool+        "sqHinge active margin y=+1: grad<0"+        (slGradZ sqHingeLoss 1.0 0.0 < 0)+    assertBool+        "sqHinge satisfied margin: grad==0"+        (slGradZ sqHingeLoss 1.0 5.0 == 0)++-- ---------------------------------------------------------------------------+-- Statistical / distributional tests (cat 5).+--+-- Tolerances are CI-derived from the sample size; with N = 100k the standard+-- error of a uniform-mean estimate is ~ (1/sqrt12)/sqrt(N) ~ 8.3e-4, so a 6σ+-- band is ~ 5e-3. A broken sampler (e.g. one that returns the same value, or+-- biased) falls well outside.+-- ---------------------------------------------------------------------------++-- | Draw @n@ uniforms threading the generator; returns the sample list.+drawUniforms :: Int -> Gen -> [Double]+drawUniforms n g0 = go n g0 []+  where+    go 0 _ acc = acc+    go k g acc = let (x, g') = nextDouble g in go (k - 1) g' (x : acc)++mean :: [Double] -> Double+mean xs = sum xs / fromIntegral (length xs)++variance :: [Double] -> Double+variance xs =+    let m = mean xs+     in sum [(x - m) * (x - m) | x <- xs] / fromIntegral (length xs)++{- | @nextDouble@ is ~uniform on [0,1): mean ≈ 0.5 and variance ≈ 1/12, each+within a 6σ CI for 100k samples, AND every draw is in [0,1). A constant or+out-of-range sampler fails; a biased one (mean drifts) fails.+-}+testUniformDistribution :: Test+testUniformDistribution = TestCase $ do+    let n = 100000 :: Int+        xs = drawUniforms n (mkGen 20240613)+        m = mean xs+        v = variance xs+        seMean = (1 / sqrt 12) / sqrt (fromIntegral n)+    assertBool "all uniforms in [0,1)" (all (\x -> x >= 0 && x < 1) xs)+    assertBool+        ("uniform mean ~0.5, got " ++ show m)+        (abs (m - 0.5) <= 6 * seMean)+    assertBool+        ("uniform variance ~1/12, got " ++ show v)+        (abs (v - 1 / 12) <= 0.01)+    -- not a constant: spread across the unit interval+    assertBool "uniform actually varies" (maximum xs - minimum xs > 0.9)++{- | Box-Muller @gaussianVector@ produces standard normals: sample mean ≈ 0 and+variance ≈ 1 over 100k draws. SE of the mean is 1/sqrt(N) ~ 3.2e-3, so a 6σ+band ~ 2e-2. Catches a Box-Muller bug (wrong radius/angle, missing log) that+shifts the mean or scales the variance.+-}+testGaussianMoments :: Test+testGaussianMoments = TestCase $ do+    let n = 100000 :: Int+        (vec, _) = gaussianVector n (mkGen 777)+        xs = VU.toList vec+        m = mean xs+        v = variance xs+        seMean = 1 / sqrt (fromIntegral n)+    assertBool "gaussianVector length" (VU.length vec == n)+    assertBool+        "gaussian samples finite"+        (all (\x -> not (isNaN x) && not (isInfinite x)) xs)+    assertBool+        ("gaussian mean ~0, got " ++ show m)+        (abs m <= 6 * seMean)+    assertBool+        ("gaussian variance ~1, got " ++ show v)+        (abs (v - 1) <= 0.03)+    -- tail mass: ~4.55% should be |x|>2 for a true N(0,1); allow a wide band.+    let tail2 = fromIntegral (length (filter (\x -> abs x > 2) xs)) / fromIntegral n+    assertBool+        ("gaussian |x|>2 frequency ~0.0455, got " ++ show tail2)+        (abs (tail2 - 0.0455) <= 0.01)++{- | @trainTestSplit frac@ over many seeds: the realized train fraction matches+@frac@ within a binomial CI, and train+test always sums to the input row count+(cat 2 invariant). A split that loses/dupes rows, or ignores @frac@, fails.+-}+testSplitProportions :: Test+testSplitProportions = TestCase $ do+    let nRows = 4000+        frac = 0.7 :: Double+        df = D.fromNamedColumns [("x", DI.fromList [1 .. nRows :: Int])]+        seeds = [1 .. 25] :: [Int]+        -- binomial SE of the fraction for n=4000 ~ sqrt(0.7*0.3/4000) ~ 7.2e-3+        seFrac = sqrt (frac * (1 - frac) / fromIntegral nRows)+    mapM_+        ( \s -> do+            let (tr, te) = trainTestSplit frac s df+                nTr = fst (D.dimensions tr)+                nTe = fst (D.dimensions te)+                realized = fromIntegral nTr / fromIntegral nRows+            assertEqual+                ("split preserves rows (seed " ++ show s ++ ")")+                nRows+                (nTr + nTe)+            assertBool+                ( "split fraction ~"+                    ++ show frac+                    ++ " (seed "+                    ++ show s+                    ++ "), got "+                    ++ show realized+                )+                (abs (realized - frac) <= 5 * seFrac)+        )+        seeds++{- | k-means inertia on a clean, well-separated blob is stable across many+seeds and never beats the true within-cluster sum of squares of the generating+partition (the global optimum here is the obvious 2-cluster split). A broken+inertia (wrong distance, double counting) would report values below the optimum+or wildly seed-dependent.+-}+testKMeansInertiaStable :: Test+testKMeansInertiaStable = TestCase $ do+    let+        -- two tight gaussian-ish blobs around (0,0) and (10,10)+        as = [0, 0.1, -0.1, 0.05, -0.05, 10, 10.1, 9.9, 10.05, 9.95] :: [Double]+        bs = [0, -0.1, 0.1, 0.05, -0.05, 10, 9.9, 10.1, 9.95, 10.05] :: [Double]+        df = D.fromNamedColumns [("a", DI.fromList as), ("b", DI.fromList bs)]+        fitSeed s =+            kmInertia $+                fit+                    defaultKMeansConfig{kmK = 2, kmNInit = 1, kmSeed = s}+                    [F.col @Double "a", F.col @Double "b"]+                    df+        inertias = map fitSeed [0 .. 19]+        -- optimum: each blob's SS about its own mean. Blob means are ~ the+        -- centroid; compute the true within-cluster SS for the 2-cluster split.+        blob1 = take 5 (zip as bs)+        blob2 = zip (drop 5 as) (drop 5 bs)+        ssOf pts =+            let mx = sum (map fst pts) / 5+                my = sum (map snd pts) / 5+             in sum [(x - mx) ^ (2 :: Int) + (y - my) ^ (2 :: Int) | (x, y) <- pts]+        optimum = ssOf blob1 + ssOf blob2+        best = minimum inertias+        worst = maximum inertias+    assertBool+        "k-means inertia finite"+        (all (\i -> not (isNaN i) && not (isInfinite i)) inertias)+    assertBool+        ( "k-means inertia never below optimum ("+            ++ show optimum+            ++ "), best="+            ++ show best+        )+        (best >= optimum - 1e-9)+    assertBool+        ( "k-means inertia stable across seeds, best="+            ++ show best+            ++ " worst="+            ++ show worst+        )+        (worst - best <= 1e-6 + 1e-3 * optimum)++-- ---------------------------------------------------------------------------+-- Reproducibility tests (cat 16).+--+-- Each compares two SEPARATE fits with the same seed (determinism) AND, where+-- the seed is a real input, asserts that a DIFFERENT seed CAN change the model+-- (so "always returns a constant" wouldn't pass). Models that derive Eq are+-- compared with ==; others by a representative field or predicted expr.+-- ---------------------------------------------------------------------------++blobsDF :: D.DataFrame+blobsDF =+    D.fromNamedColumns+        [+            ( "a"+            , DI.fromList ([0, 0.2, -0.1, 0.1, 8, 8.1, 7.9, 8.2, 0.05, 8.05] :: [Double])+            )+        ,+            ( "b"+            , DI.fromList ([0, -0.1, 0.2, 0.0, 5, 5.2, 4.9, 5.1, 0.1, 5.05] :: [Double])+            )+        ]++-- | Many-cluster frame so k-means++ seeding genuinely depends on the seed.+spreadDF :: D.DataFrame+spreadDF =+    D.fromNamedColumns+        [ ("a", DI.fromList ([0, 1, 2, 10, 11, 12, 20, 21, 22, 30, 31, 32] :: [Double]))+        , ("b", DI.fromList ([0, 1, 0, 10, 11, 10, 0, 1, 0, 10, 11, 10] :: [Double]))+        ]++{- | k-means: same seed → identical KMeansModel (Eq derived); a different seed+on a multi-blob frame CAN yield different centers (the seed actually drives+k-means++ init). Single-init so the seed is not washed out by nInit restarts.+-}+testKMeansReproducible :: Test+testKMeansReproducible = TestCase $ do+    let cfg s = defaultKMeansConfig{kmK = 4, kmNInit = 1, kmMaxIter = 1, kmSeed = s}+        run s = fit (cfg s) [F.col @Double "a", F.col @Double "b"]+        a = run 1 spreadDF+        b = run 1 spreadDF+    assertEqual "k-means same seed identical model" a b+    -- different seeds CAN differ: scan several and require at least one diff in+    -- the initial centroids (1 iteration keeps the init's footprint).+    let centersFor s = kmCenters (run s spreadDF)+        base = centersFor 1+        anyDiffer = any (\s -> centersFor s /= base) [2, 3, 5, 7, 11, 13]+    assertBool "k-means: a different seed changes the model" anyDiffer++{- | GMM: same seed → identical GMMModel (Eq derived); a different seed CAN move+the fitted means (seed drives the responsibility init via sampleIndices).+-}+testGMMReproducible :: Test+testGMMReproducible = TestCase $ do+    let cfg s = defaultGMMConfig{gmmK = 2, gmmMaxIter = 1, gmmSeed = s}+        run s = fit (cfg s) [F.col @Double "a", F.col @Double "b"] blobsDF+        a = run 1+        b = run 1+    assertEqual "GMM same seed identical model" a b+    let meansFor s = gmmMeans (run s)+        base = meansFor 1+        anyDiffer = any (\s -> meansFor s /= base) [2, 3, 4, 5, 6, 7, 8]+    assertBool "GMM: a different seed changes the means" anyDiffer++{- | RFF-SVM: same seed → identical random projection AND identical fitted SVC+coefficients; a different seed changes the random Fourier features. The model+type only derives Show, so compare representative numeric fields directly.+-}+testRFFReproducible :: Test+testRFFReproducible = TestCase $ do+    let clsDF =+            D.fromNamedColumns+                [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double]))+                , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int]))+                ]+        cfg s = defaultRFFConfig{rffD = 40, rffGamma = 0.2, rffSeed = s}+        run s = fit (cfg s) (F.col @Int "label") clsDF+        a = run 5+        b = run 5+    -- projection matrix and the fitted SVC coefficients must match bit-for-bit.+    assertEqual "RFF same seed: same projection B" (rffB a) (rffB b)+    assertEqual "RFF same seed: same coefficients" (rffCoef a) (rffCoef b)+    assertEqual "RFF same seed: same intercept" (rffIntercept a) (rffIntercept b)+    -- different seed: the random projection should differ.+    let projFor s = rffB (run s)+        base = projFor 5+        anyDiffer = any (\s -> projFor s /= base) [1, 2, 3, 6, 7, 8]+    assertBool "RFF: a different seed changes the projection" anyDiffer++-- ---------------------------------------------------------------------------+-- randomSplit / trainTestSplit determinism + row-count invariant (cat 16 + 2).+-- ---------------------------------------------------------------------------++{- | @trainTestSplit@ with the same seed is bit-identical (compared via the+prettyPrinted frames), preserves total rows, and a different seed CAN change the+partition.+-}+testSplitReproducible :: Test+testSplitReproducible = TestCase $ do+    let df = D.fromNamedColumns [("x", DI.fromList [1 .. 200 :: Int])]+        (tr1, te1) = trainTestSplit 0.6 42 df+        (tr2, te2) = trainTestSplit 0.6 42 df+    assertBool "trainTestSplit same seed: same train" (tr1 == tr2)+    assertBool "trainTestSplit same seed: same test" (te1 == te2)+    assertEqual+        "trainTestSplit preserves row count"+        200+        (fst (D.dimensions tr1) + fst (D.dimensions te1))+    let trainFor s = fst (trainTestSplit 0.6 s df)+        base = trainFor 42+        anyDiffer = any (\s -> trainFor s /= base) [1, 2, 3, 7, 99]+    assertBool "trainTestSplit: a different seed changes the partition" anyDiffer++-- ---------------------------------------------------------------------------+-- A self-consistency sanity for the helpers above so a broken reference value+-- fn cannot make the gradient checks vacuous: the reference squared value must+-- actually be ½(z-y)² at a known point.+-- ---------------------------------------------------------------------------+testReferenceValueSanity :: Test+testReferenceValueSanity = TestCase $ do+    assertBool+        "squaredValue at (y=2,z=5) == 4.5"+        (abs (squaredValue 2 5 - 4.5) < 1e-12)+    assertBool+        "logisticValue at (y=1,z=0) == log 2"+        (abs (logisticValue 1 0 - log 2) < 1e-12)+    assertBool "sqHingeValue at (y=1,z=0) == 1" (abs (sqHingeValue 1 0 - 1) < 1e-12)+    assertBool "sigmoid 0 == 0.5" (abs (sigmoid 0 - 0.5) < 1e-12)++tests :: [Test]+tests =+    [ testReferenceValueSanity+    , testSquaredGradient+    , testLogisticGradient+    , testSqHingeGradient+    , testGradientSigns+    , testUniformDistribution+    , testGaussianMoments+    , testSplitProportions+    , testKMeansInertiaStable+    , testKMeansReproducible+    , testGMMReproducible+    , testRFFReproducible+    , testSplitReproducible+    ]

@@ -0,0 +1,99 @@+{-# LANGUAGE ScopedTypeVariables #-}++module Learn.Numerics (tests) where++import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import DataFrame.Model (fit, predict)+import Test.HUnit++import DataFrame.LinearAlgebra+import DataFrame.LinearAlgebra.Eigen+import DataFrame.LinearAlgebra.Solve+import DataFrame.Random++mat :: [[Double]] -> Matrix+mat = V.fromList . map VU.fromList++approx :: Double -> Double -> Double -> Bool+approx tol a b = abs (a - b) <= tol++vApprox :: Double -> VU.Vector Double -> VU.Vector Double -> Bool+vApprox tol a b =+    VU.length a == VU.length b && VU.and (VU.zipWith (approx tol) a b)++testQR :: Test+testQR = TestCase $ do+    let a = mat [[1, 1], [1, 2], [1, 3], [1, 4]]+        b = VU.fromList [3, 5, 7, 9]+    case qrLeastSquares a b of+        Right x ->+            assertBool "QR recovers [1,2]" (vApprox 1e-9 x (VU.fromList [1, 2]))+        Left cols -> assertFailure ("unexpected rank deficiency: " ++ show cols)++testQRRankDeficient :: Test+testQRRankDeficient = TestCase $ do+    let a = mat [[1, 2], [2, 4], [3, 6]]+    case qrLeastSquares a (VU.fromList [1, 2, 3]) of+        Left _ -> pure ()+        Right _ -> assertFailure "expected rank deficiency on collinear columns"++testCholesky :: Test+testCholesky = TestCase $ do+    let a = mat [[4, 2], [2, 3]]+    case choleskySolve a (VU.fromList [2, 1]) of+        Just x ->+            assertBool "cholesky solves" (vApprox 1e-9 x (VU.fromList [0.5, 0]))+        Nothing -> assertFailure "expected PD"+    assertEqual "non-PD rejected" Nothing (cholesky (mat [[1, 2], [2, 1]]))++testJacobi :: Test+testJacobi = TestCase $ do+    let (ev, vecs) = jacobiEigenSym (mat [[2, 1], [1, 2]])+    assertBool "eigenvalues [3,1]" (vApprox 1e-9 ev (VU.fromList [3, 1]))+    let v0 = vecs V.! 0+        recon = matVec (mat [[2, 1], [1, 2]]) v0+    assertBool+        "A v = lambda v"+        (vApprox 1e-9 recon (scaleV (ev VU.! 0) v0))++testPowerIter :: Test+testPowerIter = TestCase $ do+    let (lam, _) = powerIterTop 200 (mat [[2, 1], [1, 2]])+    assertBool "dominant eigenvalue ~3" (approx 1e-6 lam 3)++testLogSumExp :: Test+testLogSumExp = TestCase $ do+    let xs = VU.fromList [1000, 1001, 1002]+        lse = logSumExp xs+    assertBool "logSumExp stable, finite" (not (isNaN lse) && not (isInfinite lse))+    assertBool "logSumExp >= max" (lse >= 1002)++testRngDeterminism :: Test+testRngDeterminism = TestCase $ do+    let (a, _) = shuffleInts 50 (mkGen 11)+        (b, _) = shuffleInts 50 (mkGen 11)+    assertEqual "same seed same shuffle" a b+    assertBool+        "is a permutation"+        (VU.toList (VU.modify (const (pure ())) a) /= [] && all (`VU.elem` a) [0 .. 49])++testRngRange :: Test+testRngRange = TestCase $ do+    let go 0 _ acc = acc+        go k g acc =+            let (x, g') = nextIntR (3, 7) g in go (k - 1 :: Int) g' (x : acc)+        xs = go 1000 (mkGen 5) []+    assertBool "ints within range" (all (\x -> x >= 3 && x <= 7) xs)++tests :: [Test]+tests =+    [ testQR+    , testQRRankDeficient+    , testCholesky+    , testJacobi+    , testPowerIter+    , testLogSumExp+    , testRngDeterminism+    , testRngRange+    ]

@@ -0,0 +1,188 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++{- | Parity tests against scikit-learn on clean Kaggle-style datasets. The+reference values in @data/ml/golden.json@ are produced by+@scripts/gen_sklearn_golden.py@. Closed-form models (OLS, ridge, PCA) are held to+tight coefficient parity; iterative models to an accuracy/inertia floor.+-}+module Learn.SklearnParity (tests) where++import Data.Aeson (Value (..), decodeFileStrict')+import qualified Data.Aeson.Key as K+import qualified Data.Aeson.KeyMap as KM+import Data.Maybe (fromMaybe)+import qualified Data.Text as T+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import DataFrame.Model (fit, predict)+import Test.HUnit++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import DataFrame.Internal.Column (TypedColumn (..), toVector)+import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Interpreter (interpret)++import DataFrame.Boosting (+    GBConfig (..),+    GBLoss (..),+    defaultGBConfig,+    gbProbaExpr,+ )+import DataFrame.KMeans+import DataFrame.LinearModel+import DataFrame.LinearSolver (defaultSolverConfig)+import DataFrame.PCA+import DataFrame.SVM++goldenPath :: FilePath+goldenPath = "data/ml/golden.json"++loadGolden :: IO Value+loadGolden =+    fromMaybe (error "missing data/ml/golden.json") <$> decodeFileStrict' goldenPath++(.!) :: Value -> T.Text -> Value+(.!) (Object o) k = fromMaybe Null (KM.lookup (K.fromText k) o)+(.!) _ _ = Null++asNum :: Value -> Double+asNum (Number s) = realToFrac s+asNum _ = error "asNum: not a number"++asNums :: Value -> [Double]+asNums (Array a) = map asNum (V.toList a)+asNums _ = error "asNums: not an array"++asMatrix :: Value -> [[Double]]+asMatrix (Array a) = map asNums (V.toList a)+asMatrix _ = error "asMatrix: not a matrix"++interpD :: D.DataFrame -> Expr Double -> [Double]+interpD df e = case interpret @Double df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c)+    Left err -> error (show err)++colDoubles :: D.DataFrame -> T.Text -> [Double]+colDoubles df name = interpD df (F.col @Double name)++closeAll :: Double -> [Double] -> [Double] -> Bool+closeAll tol a b =+    length a == length b && and (zipWith (\x y -> abs (x - y) <= tol) a b)++accuracyOf :: [Double] -> [Double] -> Double+accuracyOf preds truth =+    fromIntegral (length (filter id (zipWith (==) preds truth)))+        / fromIntegral (max 1 (length truth))++testOLSParity :: Test+testOLSParity = TestCase $ do+    g <- loadGolden+    df <- D.readCsv "data/ml/regression.csv"+    let m = fit defaultLinearConfig (F.col @Double "target") df+        goldCoef = asNums (g .! "ols" .! "coef")+        goldB = asNum (g .! "ols" .! "intercept")+    assertBool+        "OLS coefficients match sklearn"+        (closeAll 1e-4 (VU.toList (regCoef m)) goldCoef)+    assertBool "OLS intercept matches sklearn" (abs (regIntercept m - goldB) < 1e-4)++testRidgeParity :: Test+testRidgeParity = TestCase $ do+    g <- loadGolden+    df <- D.readCsv "data/ml/regression.csv"+    let m =+            fit (LinearConfig (Ridge 1.0) defaultSolverConfig) (F.col @Double "target") df+        goldCoef = asNums (g .! "ridge" .! "coef")+    assertBool+        "Ridge coefficients match sklearn"+        (closeAll 1e-2 (VU.toList (regCoef m)) goldCoef)++testPCAParity :: Test+testPCAParity = TestCase $ do+    g <- loadGolden+    df <- D.readCsv "data/ml/iris.csv"+    let feats =+            map+                (F.col @Double)+                ["sepal_length", "sepal_width", "petal_length", "petal_width"]+        m = fit (PCAConfig (NComp 2) False) feats df+        goldEvr = asNums (g .! "pca" .! "evr")+        goldComp = asMatrix (g .! "pca" .! "components_abs")+        ourComp = map (map abs . VU.toList) (V.toList (pcaComponents m))+    assertBool+        "PCA explained variance ratio matches sklearn"+        (closeAll 1e-4 (VU.toList (pcaExplainedVarianceRatio m)) goldEvr)+    assertBool+        "PCA |components| match sklearn"+        (and (zipWith (closeAll 1e-3) ourComp goldComp))++testLogisticIrisParity :: Test+testLogisticIrisParity = TestCase $ do+    g <- loadGolden+    df <- D.readCsv "data/ml/iris.csv"+    let m = fit defaultLogisticConfig (F.col @Double "species") df+        preds = interpD df (predict m)+        truth = colDoubles df "species"+        acc = accuracyOf preds truth+        gold = asNum (g .! "logistic_iris" .! "accuracy")+    assertBool+        ("logistic iris accuracy within 0.06 of sklearn " ++ show (acc, gold))+        (acc >= gold - 0.06)++testSVCParity :: Test+testSVCParity = TestCase $ do+    g <- loadGolden+    df <- D.readCsv "data/ml/iris_binary.csv"+    let m = fit defaultSVCConfig (F.col @Double "label") df+        preds = interpD df (predict m)+        truth = colDoubles df "label"+        acc = accuracyOf preds truth+        gold = asNum (g .! "linear_svc" .! "accuracy")+    assertBool+        ("linear SVC accuracy within 0.06 of sklearn " ++ show (acc, gold))+        (acc >= gold - 0.06)++testGBMParity :: Test+testGBMParity = TestCase $ do+    g <- loadGolden+    df <- D.readCsv "data/ml/iris_binary.csv"+    let m =+            fit+                defaultGBConfig{gbLoss = LogisticDeviance, gbNEstimators = 100}+                (F.col @Double "label")+                df+        probs = interpD df (gbProbaExpr m)+        preds = map (\p -> if p > 0.5 then 1 else 0) probs+        truth = colDoubles df "label"+        acc = accuracyOf preds truth+        gold = asNum (g .! "gbm" .! "accuracy")+    assertBool+        ("GBM accuracy within 0.1 of sklearn " ++ show (acc, gold))+        (acc >= gold - 0.1)++testKMeansParity :: Test+testKMeansParity = TestCase $ do+    g <- loadGolden+    df <- D.readCsv "data/ml/blobs.csv"+    let m =+            fit+                defaultKMeansConfig{kmK = 3, kmNInit = 10, kmSeed = 0}+                [F.col @Double "x", F.col @Double "y"]+                df+        gold = asNum (g .! "kmeans" .! "inertia")+    assertBool+        ("k-means inertia within 10% of sklearn " ++ show (kmInertia m, gold))+        (abs (kmInertia m - gold) / gold < 0.1)++tests :: [Test]+tests =+    [ testOLSParity+    , testRidgeParity+    , testPCAParity+    , testLogisticIrisParity+    , testSVCParity+    , testGBMParity+    , testKMeansParity+    ]

@@ -0,0 +1,134 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++module Learn.Symbolic (tests) where++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import DataFrame.Internal.Column (TypedColumn (..), toVector)+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Interpreter (interpret)++import DataFrame.PCA.Kernel+import DataFrame.SVM.RFF+import DataFrame.SymbolicRegression+import DataFrame.SymbolicRegression.Expr+import DataFrame.SymbolicRegression.Optimize (meanSquaredError)+import DataFrame.SymbolicRegression.Simplify (simplify)++import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import DataFrame.Model (fit, predict)+import Test.HUnit++interpD :: D.DataFrame -> Expr Double -> [Double]+interpD df e = case interpret @Double df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Double @VU.Vector c)+    Left err -> error (show err)++interpI :: D.DataFrame -> Expr Int -> [Int]+interpI df e = case interpret @Int df e of+    Right (TColumn c) -> either (const []) VU.toList (toVector @Int @VU.Vector c)+    Left err -> error (show err)++testSRRecovers :: Test+testSRRecovers = TestCase $ do+    let xs = [-3, -2, -1, 0, 1, 2, 3, 4, 5, 6] :: [Double]+        df =+            D.fromNamedColumns+                [ ("x", DI.fromList xs)+                , ("y", DI.fromList [x * x + x | x <- xs])+                ]+        m =+            fit+                defaultSRConfig+                    { srSeed = 3+                    , srGenerations = 60+                    , srPopSize = 300+                    , srUnaryOps = []+                    }+                (F.col @Double "y")+                df+        preds = interpD df (srBest m)+        truth = interpD df (F.col @Double "y")+        err = sum (zipWith (\p t -> (p - t) ^ (2 :: Int)) preds truth) / 10+    assertBool "SR recovers x*x+x to low error" (err < 1e-6)+    assertBool "Pareto front non-empty" (not (null (srPareto m)))++testSRDeterminism :: Test+testSRDeterminism = TestCase $ do+    let xs = [1 .. 8] :: [Double]+        df =+            D.fromNamedColumns+                [ ("x", DI.fromList xs)+                , ("y", DI.fromList (map (\x -> 2 * x + 1) xs))+                ]+        run =+            fit+                defaultSRConfig{srSeed = 9, srGenerations = 20}+                (F.col @Double "y")+                df+    assertEqual "SR deterministic best MSE" (srBestMSE run) (srBestMSE (rerun df))+  where+    rerun =+        fit+            defaultSRConfig{srSeed = 9, srGenerations = 20}+            (F.col @Double "y")++testSimplifyPreservesEval :: Test+testSimplifyPreservesEval = TestCase $ do+    let feats = V.singleton (VU.fromList [1, 2, 3, 4 :: Double])+        n = 4+        e = SBin SAdd (SBin SMul (SVar 0) (SConst 1)) (SConst 0)+        target = VU.fromList [1, 2, 3, 4]+    assertBool+        "simplify preserves evaluation"+        ( abs+            (meanSquaredError feats n target e - meanSquaredError feats n target (simplify e))+            < 1e-12+        )+    assertBool "simplify is size non-increasing" (srSize (simplify e) <= srSize e)+    assertEqual "simplify idempotent" (simplify e) (simplify (simplify e))++testKernelPCA :: Test+testKernelPCA = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("a", DI.fromList ([0, 0.2, -0.1, 0.1, 8, 8.1, 7.9, 8.2] :: [Double]))+                , ("b", DI.fromList ([0, -0.1, 0.2, 0.0, 5, 5.2, 4.9, 5.1] :: [Double]))+                ]+        m =+            fit+                defaultKernelPCAConfig{kpcaNComponents = 2, kpcaNLandmarks = 8, kpcaSeed = 1}+                [F.col @Double "a", F.col @Double "b"]+                df+        pc1 = interpD df (snd (head (kernelPCAExprs m)))+    assertBool "kPCA finite" (not (any isNaN pc1))+    assertBool+        "kPCA first component separates blobs"+        (signum (head pc1) /= signum (last pc1))++testRFFSVM :: Test+testRFFSVM = TestCase $ do+    let df =+            D.fromNamedColumns+                [ ("x", DI.fromList ([-3, -2, -1, -0.5, 0.5, 1, 2, 3] :: [Double]))+                , ("label", DI.fromList ([0, 0, 0, 0, 1, 1, 1, 1] :: [Int]))+                ]+        m =+            fit+                defaultRFFConfig{rffD = 80, rffGamma = 0.2, rffSeed = 2}+                (F.col @Int "label")+                df+        preds = interpI df (predict m)+    assertEqual "RFF SVM separates" [0, 0, 0, 0, 1, 1, 1, 1] preds++tests :: [Test]+tests =+    [ testSRRecovers+    , testSRDeterminism+    , testSimplifyPreservesEval+    , testKernelPCA+    , testRFFSVM+    ]

@@ -0,0 +1,83 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++{- | Feature-synthesis enumerator: it should recover small exact features+(@x²@, @a/b@) from the example rows, score them near-perfectly, and be+deterministic.+-}+module Learn.Synthesis (tests) where++import qualified DataFrame as D+import DataFrame.Model (fit)+import DataFrame.Synthesis++import Test.HUnit++quad :: D.DataFrame+quad =+    D.fromNamedColumns+        [ ("x", D.fromList xs)+        , ("y", D.fromList (map (\x -> x * x) xs))+        ]+  where+    xs = map fromIntegral [1 .. 12 :: Int] :: [Double]++ratio :: D.DataFrame+ratio =+    D.fromNamedColumns+        [ ("a", D.fromList ([2, 6, 12, 20, 30, 42] :: [Double]))+        , ("b", D.fromList ([1, 2, 3, 4, 5, 6] :: [Double]))+        , ("y", D.fromList ([2, 3, 4, 5, 6, 7] :: [Double]))+        ]++-- | Pearson r²: enumeration should find x·x and score it ~1.+recoversQuadratic :: Test+recoversQuadratic = TestCase $ do+    let sf = fit defaultSynthesisConfig (D.col @Double "y") quad+    assertBool+        ("quadratic r2 = " ++ show (sfScore sf))+        (sfScore sf > 0.999 && sfScore sf <= 1.0001)++-- | MSE: the best feature reproduces the target exactly, so -MSE ~ 0.+exactRecoveryMSE :: Test+exactRecoveryMSE = TestCase $ do+    let sf =+            fit defaultSynthesisConfig{synLoss = MeanSquaredError} (D.col @Double "y") quad+    assertBool ("exact -mse = " ++ show (sfScore sf)) (sfScore sf > -1.0e-6)++-- | Division is enumerated (with the denominator guard): a/b is recovered.+recoversRatio :: Test+recoversRatio = TestCase $ do+    let sf = fit defaultSynthesisConfig (D.col @Double "y") ratio+    assertBool+        ("ratio r2 = " ++ show (sfScore sf))+        (sfScore sf > 0.999 && sfScore sf <= 1.0001)++-- | The bank is non-trivial and its ranked features are distinct expressions.+distinctFeatures :: Test+distinctFeatures = TestCase $ do+    let sf = fit defaultSynthesisConfig (D.col @Double "y") quad+        names = [D.prettyPrint e | (e, _) <- sfFeatures sf]+    assertBool "synthesizes more than one feature" (length names > 1)+    assertBool "ranked features are distinct" (length names == length (dedup names))+  where+    dedup = foldr (\x acc -> if x `elem` acc then acc else x : acc) []++-- | Same config and data give the same best expression.+deterministic :: Test+deterministic = TestCase $ do+    let a = fit defaultSynthesisConfig (D.col @Double "y") quad+        b = fit defaultSynthesisConfig (D.col @Double "y") quad+    assertEqual+        "same best expression"+        (D.prettyPrint (sfExpr a))+        (D.prettyPrint (sfExpr b))++tests :: [Test]+tests =+    [ recoversQuadratic+    , exactRecoveryMSE+    , recoversRatio+    , distinctFeatures+    , deterministic+    ]

@@ -12,9 +12,25 @@ import qualified DecisionTree import qualified Functions import qualified IO.CSV+import qualified IO.CsvGolden import qualified IO.JSON+import qualified Internal.ColumnBuilder+import qualified Internal.DictEncode+import qualified Internal.PackedText import qualified Internal.Parsing+import qualified LazyParity import qualified LazyParquet+import qualified Learn.Denotation+import qualified Learn.EdgeCases+import qualified Learn.Ensembles+import qualified Learn.Metamorphic+import qualified Learn.MetricsTests+import qualified Learn.Models+import qualified Learn.NumericalRigor+import qualified Learn.Numerics+import qualified Learn.SklearnParity+import qualified Learn.Symbolic+import qualified Learn.Synthesis import qualified LinearSolver import qualified Monad import qualified Operations.Aggregations@@ -23,11 +39,14 @@ import qualified Operations.Derive import qualified Operations.Filter import qualified Operations.GroupBy+import qualified Operations.Inference import qualified Operations.InsertColumn import qualified Operations.Join import qualified Operations.Merge import qualified Operations.Nullable import qualified Operations.NullableHashing+import qualified Operations.ParallelGroupBy+import qualified Operations.ParallelJoin import qualified Operations.Provenance import qualified Operations.ReadCsv import qualified Operations.Record@@ -38,8 +57,10 @@ import qualified Operations.Subset import qualified Operations.Take import qualified Operations.Typing+import qualified Operations.VectorKernel import qualified Operations.Window import qualified Operations.WriteCsv+import qualified PackedTextMigration import qualified Parquet import qualified Plotting import qualified Properties@@ -53,13 +74,30 @@ tests =     TestList $         DecisionTree.tests+            ++ Internal.ColumnBuilder.tests+            ++ Internal.DictEncode.tests+            ++ Internal.PackedText.tests             ++ Internal.Parsing.tests+            ++ Learn.Numerics.tests+            ++ Learn.Denotation.tests+            ++ Learn.Models.tests+            ++ Learn.Ensembles.tests+            ++ Learn.Symbolic.tests+            ++ Learn.SklearnParity.tests+            ++ Learn.Synthesis.tests+            ++ Learn.MetricsTests.tests+            ++ Learn.Metamorphic.tests+            ++ Learn.EdgeCases.tests+            ++ Learn.NumericalRigor.tests             ++ Operations.Aggregations.tests             ++ Operations.Apply.tests             ++ Operations.Core.tests             ++ Operations.Derive.tests             ++ Operations.Filter.tests             ++ Operations.GroupBy.tests+            ++ Operations.ParallelGroupBy.tests+            ++ Operations.ParallelJoin.tests+            ++ Operations.Inference.tests             ++ Operations.InsertColumn.tests             ++ Operations.Join.tests             ++ Operations.Merge.tests@@ -76,18 +114,22 @@             ++ Operations.Subset.hunitTests             ++ Operations.Take.tests             ++ Operations.Typing.tests+            ++ Operations.VectorKernel.tests             ++ Operations.Window.tests             ++ Functions.tests             ++ IO.CSV.tests+            ++ IO.CsvGolden.tests             ++ IO.JSON.tests             ++ Parquet.tests             ++ LazyParquet.tests+            ++ LazyParity.tests             ++ Plotting.tests             ++ LinearSolver.tests             ++ Simplify.tests             ++ TreePruning.tests             ++ Worklist.tests             ++ Cart.tests+            ++ PackedTextMigration.tests  isSuccessful :: Result -> Bool isSuccessful (Success{}) = True@@ -105,11 +147,16 @@                     (quickCheckWithResult stdArgs)                     Operations.Subset.tests             monadRes <- mapM (quickCheckWithResult stdArgs) Monad.tests+            cbRes <-+                mapM+                    (quickCheckWithResult stdArgs)+                    Internal.ColumnBuilder.props             propsRes <- mapM (quickCheckWithResult stdArgs) Properties.tests             catRes <- mapM (quickCheckWithResult stdArgs) Properties.Categorical.tests             simpRes <- mapM (quickCheckWithResult stdArgs) Properties.Simplify.tests             wlRes <- mapM (quickCheckWithResult stdArgs) Worklist.props             if not (all isSuccessful propRes)+                || not (all isSuccessful cbRes)                 || not (all isSuccessful monadRes)                 || not (all isSuccessful propsRes)                 || not (all isSuccessful catRes)

@@ -1,7 +1,10 @@ {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}  module Operations.GroupBy where +import qualified Data.List as L+import qualified Data.Map.Strict as M import qualified Data.Text as T import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D@@ -51,9 +54,70 @@             (print $ D.groupBy ["test0"] testData)         ) +{- | The grouping invariant every backend (sequential now, parallel in Phase 2)+must satisfy: 'valueIndices' / 'offsets' / 'rowToGroup' agree, groups partition+all rows exactly once, each group is exactly the rows sharing a key tuple, and+no two groups share a key. Checked on a frame with several keys colliding under+the hash (forces the open-addressing key re-verification path).+-}+groupingInvariantHolds :: Test+groupingInvariantHolds =+    TestCase+        (assertBool "grouping partitions rows by exact key tuple" (checkGrouping df))+  where+    -- Repeated, interleaved keys across two columns; the dense int grid is the+    -- classic hash-collision stressor.+    df =+        D.fromNamedColumns+            [ ("k1", DI.fromList (map (`mod` 5) [0 .. 199 :: Int]))+            , ("k2", DI.fromList (map (\i -> (i * 7) `mod` 3) [0 .. 199 :: Int]))+            , ("v", DI.fromList [0 .. 199 :: Int])+            ]++{- | Recompute the reference grouping by the key tuple and assert the library's+'valueIndices'/'offsets'/'rowToGroup' describe the same partition.+-}+checkGrouping :: D.DataFrame -> Bool+checkGrouping df =+    let g = D.groupBy ["k1", "k2"] df+        vis = VU.toList (D.valueIndices g)+        os = VU.toList (D.offsets g)+        rtg = VU.toList (D.rowToGroup g)+        n = fst (D.dimensions df)+        k1 = colInts "k1"+        k2 = colInts "k2"+        colInts name = case D.getColumn name df of+            Just c -> DI.toList @Int c+            Nothing -> error "missing column"+        key i = (k1 !! i, k2 !! i)+        -- The group slices read out of vis/os.+        slices = [take (e - s) (drop s vis) | (s, e) <- zip os (tail os)]+        -- valueIndices must be a permutation of all rows.+        permutationOk = L.sort vis == [0 .. n - 1]+        -- Every slice is non-empty and internally key-constant.+        constKeys = all (\sl -> not (L.null sl) && allSame (map key sl)) slices+        -- Distinct groups have distinct keys.+        groupKeys = map (key . head) slices+        distinctKeys = L.length (L.nub groupKeys) == L.length groupKeys+        -- The number of groups equals the number of distinct key tuples.+        nGroupsOk = L.length slices == M.size (M.fromList [(key i, ()) | i <- [0 .. n - 1]])+        -- rowToGroup agrees with the slice each row lands in.+        rtgOk =+            and+                [ (rtg !! r) == gIdx+                | (gIdx, sl) <- zip [0 ..] slices+                , r <- sl+                ]+     in permutationOk && constKeys && distinctKeys && nGroupsOk && rtgOk++allSame :: (Eq a) => [a] -> Bool+allSame [] = True+allSame (x : xs) = all (== x) xs+ tests :: [Test] tests =     [ TestLabel "groupBySingleRowWAI" groupBySingleRowWAI     , TestLabel "groupByMultipleRowsWAI" groupByMultipleRowsWAI     , TestLabel "groupByColumnDoesNotExist" groupByColumnDoesNotExist+    , TestLabel "groupingInvariantHolds" groupingInvariantHolds     ]

@@ -0,0 +1,229 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++{- | Round-2 (S2) pins for the single-pass inference lattice and the+Int -> Double promotion path: sample classification via the WS-B byte+parsers, prefix promotion instead of full-column re-parse, and the+byte-level missing-token test in the default reader's inferred path.+-}+module Operations.Inference where++import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BL+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import qualified DataFrame as D+import qualified DataFrame.IO.CSV as CSV+import qualified DataFrame.Internal.Column as DI+import qualified DataFrame.Operations.Typing as D++import Data.Time (Day)+import DataFrame.Internal.DataFrame (getColumn)+import Test.HUnit (Test (TestCase, TestLabel), assertEqual, assertFailure)++assumeBytes :: [Maybe BS.ByteString] -> D.ParsingAssumption+assumeBytes = D.makeParsingAssumptionBytes "%Y-%m-%d" . V.fromList++assumeText :: [Maybe T.Text] -> D.ParsingAssumption+assumeText = D.makeParsingAssumption "%Y-%m-%d" . V.fromList++-- The candidate-mask priorities reproduce the documented fallback order:+-- Bool, then Int (only when the Double mask agrees), Double, Date, Text;+-- an all-null sample stays NoAssumption.+latticePriorities :: Test+latticePriorities = TestCase $ do+    let cases =+            [ ([Just "True", Just "FALSE"], D.BoolAssumption, "bools")+            , ([Just "1", Just "-42"], D.IntAssumption, "ints")+            , ([Just "1", Just "2.5"], D.DoubleAssumption, "int+double")+            , ([Just "1e3", Just "0.5"], D.DoubleAssumption, "doubles")+            , ([Just "2024-01-02", Just "1999-12-31"], D.DateAssumption, "dates")+            , ([Just "abc", Just "1"], D.TextAssumption, "text")+            , ([Nothing, Nothing], D.NoAssumption, "all null")+            , ([], D.NoAssumption, "empty sample")+            , ([Nothing, Just "7"], D.IntAssumption, "null then int")+            ]+    mapM_+        ( \(cells, expected, what) -> do+            assertEqual ("bytes: " <> what) expected (assumeBytes cells)+            assertEqual+                ("text: " <> what)+                expected+                (assumeText (map (fmap TE.decodeUtf8) cells))+        )+        cases++-- Pinned new behavior (WS-B parser semantics in the sample): an+-- overflowing integer clears the Int candidate instead of wrapping, so+-- the column classifies (and parses) as Double.+latticeOverflowIntIsDouble :: Test+latticeOverflowIntIsDouble = TestCase $ do+    assertEqual+        "bytes overflow -> Double"+        D.DoubleAssumption+        (assumeBytes [Just "9223372036854775808"])+    assertEqual+        "text overflow -> Double"+        D.DoubleAssumption+        (assumeText [Just "9223372036854775808"])++-- Pinned new behavior: byte-level strip-tolerant classification accepts+-- ASCII-padded numerics (the strip-equivalent unification, audit+-- divergence #2), where the old Text-path sample rejected " 42" as+-- Double and demoted to Text.+latticePaddedIntIsInt :: Test+latticePaddedIntIsInt =+    TestCase+        (assertEqual "padded int" D.IntAssumption (assumeText [Just " 42 "]))++-- End-to-end: an overflowing cell inside the sample now yields a Double+-- column with the correctly converted value (old: silently wrapped Int).+parseOverflowIntsAsDoubles :: Test+parseOverflowIntsAsDoubles =+    let beforeParse = ["9223372036854775808", "1", "2"] :: [T.Text]+        afterParse = [9.223372036854776e18, 1, 2] :: [Double]+        expected = DI.fromVector (V.fromList (map Just afterParse))+        actual =+            D.parseDefault (D.defaultParseOptions{D.sampleSize = 3}) $+                DI.fromVector (V.fromList beforeParse)+     in TestCase (assertEqual "overflow -> Double column" expected actual)++-- Promotion: a Double cell far past the sample converts the built Int+-- prefix in place; values must equal a from-scratch Double parse.+promoteIntPrefixToDouble :: Test+promoteIntPrefixToDouble =+    let ints = map (T.pack . show) ([1 .. 150] :: [Int])+        beforeParse = ints ++ ["0.5"]+        afterParse = map fromIntegral ([1 .. 150] :: [Int]) ++ [0.5] :: [Double]+        expected = DI.UnboxedColumn Nothing (VU.fromList afterParse)+        actual =+            D.parseDefault+                (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = D.NoSafeRead})+                $ DI.fromVector (V.fromList beforeParse)+     in TestCase (assertEqual "Int prefix promoted to Double" expected actual)++-- An overflowing integer past the sample triggers promotion with its+-- true Double value (the old Text-path Int parse silently wrapped it).+promoteOverflowPastSample :: Test+promoteOverflowPastSample =+    let ints = map (T.pack . show) ([1 .. 30] :: [Int])+        beforeParse = ints ++ ["18446744073709551616", "1.5"]+        afterParse =+            map fromIntegral ([1 .. 30] :: [Int])+                ++ [1.8446744073709552e19, 1.5] ::+                [Double]+        expected = DI.UnboxedColumn Nothing (VU.fromList afterParse)+        actual =+            D.parseDefault+                (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = D.NoSafeRead})+                $ DI.fromVector (V.fromList beforeParse)+     in TestCase (assertEqual "overflow past sample -> true Double" expected actual)++-- Promotion preserves null positions accumulated during the Int phase.+promotionPreservesNulls :: Test+promotionPreservesNulls =+    let beforeParse =+            map (T.pack . show) ([1 .. 20] :: [Int])+                ++ [""]+                ++ map (T.pack . show) ([21 .. 40] :: [Int])+                ++ ["2.5"]+        afterParse =+            map (Just . fromIntegral) ([1 .. 20] :: [Int])+                ++ [Nothing]+                ++ map (Just . fromIntegral) ([21 .. 40] :: [Int])+                ++ [Just 2.5] ::+                [Maybe Double]+        expected = DI.fromVector (V.fromList afterParse)+        actual =+            D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $+                DI.fromVector (V.fromList beforeParse)+     in TestCase (assertEqual "nulls survive promotion" expected actual)++-- A cell that parses as neither Int nor Double demotes the whole column+-- to Text with every raw cell preserved (re-extracted, not re-parsed).+promotionDemotesToText :: Test+promotionDemotesToText =+    let beforeParse = map (T.pack . show) ([1 .. 30] :: [Int]) ++ ["abc"]+        expected = DI.BoxedColumn Nothing (V.fromList beforeParse)+        actual =+            D.parseDefault+                (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = D.NoSafeRead})+                $ DI.fromVector (V.fromList beforeParse)+     in TestCase (assertEqual "demotes to Text, raw values kept" expected actual)++-- Default (cassava) reader, inferred path: the same promotion runs over+-- the retained raw bytes.+readerPromotesIntColumn :: Test+readerPromotesIntColumn = TestCase $ do+    let csv =+            "x\n"+                <> T.intercalate "\n" (map (T.pack . show) ([1 .. 150] :: [Int]))+                <> "\n0.5\n"+        expected =+            DI.UnboxedColumn+                Nothing+                (VU.fromList (map fromIntegral ([1 .. 150] :: [Int]) ++ [0.5] :: [Double]))+    df <- CSV.fromCsvBytes (BL.fromStrict (TE.encodeUtf8 csv))+    case getColumn "x" df of+        Just col -> assertEqual "reader Int->Double promotion" expected col+        Nothing -> assertFailure "column x missing"++-- Default reader: canonical missing tokens null the cell through the+-- byte-level fast path, exactly as the Text-decode check did.+readerCanonicalMissingTokens :: Test+readerCanonicalMissingTokens = TestCase $ do+    let csv = "x,y\nNA,1\nnan,2\nNothing,3\n ,4\nNULL,5\n"+        expectedX = DI.fromVector (V.fromList (replicate 5 (Nothing :: Maybe T.Text)))+        expectedY = DI.UnboxedColumn Nothing (VU.fromList ([1 .. 5] :: [Int]))+    df <- CSV.fromCsvBytes (BL.fromStrict (TE.encodeUtf8 csv))+    case getColumn "x" df of+        Just col -> assertEqual "all-missing column" expectedX col+        Nothing -> assertFailure "column x missing"+    case getColumn "y" df of+        Just col -> assertEqual "int column" expectedY col+        Nothing -> assertFailure "column y missing"++-- A custom missing list replaces (not extends) the canonical one: "NA"+-- must then survive as text.+readerCustomMissingTokens :: Test+readerCustomMissingTokens = TestCase $ do+    let csv = "x\nfoo\nNA\n7\n"+        opts = CSV.defaultReadOptions{CSV.missingIndicators = ["foo"]}+        expected =+            DI.fromVector+                (V.fromList [Nothing, Just ("NA" :: T.Text), Just "7"])+    df <- CSV.decodeSeparated opts (BL.fromStrict (TE.encodeUtf8 csv))+    case getColumn "x" df of+        Just col -> assertEqual "custom list only" expected col+        Nothing -> assertFailure "column x missing"++-- Default reader date columns still parse (the WS-B fast date path is+-- bit-identical to readByteStringDate for %Y-%m-%d).+readerDatesStillParse :: Test+readerDatesStillParse = TestCase $ do+    let csv = "d\n2024-01-01\n2024-01-02\n2024-01-03\n"+        expected =+            DI.fromVector+                (V.fromList (map (read @Day) ["2024-01-01", "2024-01-02", "2024-01-03"]))+    df <- CSV.fromCsvBytes (BL.fromStrict (TE.encodeUtf8 csv))+    case getColumn "d" df of+        Just col -> assertEqual "date column" expected col+        Nothing -> assertFailure "column d missing"++tests :: [Test]+tests =+    [ TestLabel "lattice_priorities" latticePriorities+    , TestLabel "lattice_overflow_int_is_double" latticeOverflowIntIsDouble+    , TestLabel "lattice_padded_int_is_int" latticePaddedIntIsInt+    , TestLabel "parse_overflow_ints_as_doubles" parseOverflowIntsAsDoubles+    , TestLabel "promote_int_prefix_to_double" promoteIntPrefixToDouble+    , TestLabel "promote_overflow_past_sample" promoteOverflowPastSample+    , TestLabel "promotion_preserves_nulls" promotionPreservesNulls+    , TestLabel "promotion_demotes_to_text" promotionDemotesToText+    , TestLabel "reader_promotes_int_column" readerPromotesIntColumn+    , TestLabel "reader_canonical_missing_tokens" readerCanonicalMissingTokens+    , TestLabel "reader_custom_missing_tokens" readerCustomMissingTokens+    , TestLabel "reader_dates_still_parse" readerDatesStillParse+    ]

@@ -219,6 +219,43 @@             (D.sortBy [D.Asc (F.col @Text "Name")] (fullOuterJoin ["Name"] studentDf staffDf))         ) +testFullOuterJoinUnboxedKey :: Test+testFullOuterJoinUnboxedKey =+    TestCase+        ( assertEqual+            "Full outer join on an unboxed (Int) key column coalesces correctly"+            ( D.fromNamedColumns+                [ ("customer_id", D.fromList [1 :: Int, 2, 3, 4])+                ,+                    ( "amount"+                    , D.fromList [Just 250.0 :: Maybe Double, Just 80.0, Nothing, Just 125.0]+                    )+                ,+                    ( "name"+                    , D.fromList [Just "Ada" :: Maybe Text, Just "Lin", Just "Grace", Nothing]+                    )+                ]+            )+            ( D.sortBy+                [D.Asc (F.col @Int "customer_id")]+                (fullOuterJoin ["customer_id"] ordersDf customersDf)+            )+        )++ordersDf :: D.DataFrame+ordersDf =+    D.fromNamedColumns+        [ ("customer_id", D.fromList [1 :: Int, 2, 4])+        , ("amount", D.fromList [250.0 :: Double, 80.0, 125.0])+        ]++customersDf :: D.DataFrame+customersDf =+    D.fromNamedColumns+        [ ("customer_id", D.fromList [1 :: Int, 2, 3])+        , ("name", D.fromList ["Ada" :: Text, "Lin", "Grace"])+        ]+ dfL :: D.DataFrame dfL =     D.fromNamedColumns@@ -449,6 +486,49 @@             (D.nRows (leftJoin ["key"] manyLeft manyRight))         ) +-- Stress the open-addressing hash index with many distinct integer keys.+-- Left keys 0..9999, right keys 5000..14999 (overlap 5000..9999).+bigLeft :: D.DataFrame+bigLeft =+    D.fromNamedColumns+        [ ("key", D.fromList [0 .. 9999 :: Int])+        , ("la", D.fromList [i * 2 | i <- [0 .. 9999 :: Int]])+        ]++bigRight :: D.DataFrame+bigRight =+    D.fromNamedColumns+        [ ("key", D.fromList [5000 .. 14999 :: Int])+        , ("rb", D.fromList [i * 3 | i <- [5000 .. 14999 :: Int]])+        ]++testBigInnerJoinRowCount :: Test+testBigInnerJoinRowCount =+    TestCase+        ( assertEqual+            "Inner join over 10k distinct keys matches overlap size"+            5000+            (D.nRows (innerJoin ["key"] bigLeft bigRight))+        )++testBigLeftJoinRowCount :: Test+testBigLeftJoinRowCount =+    TestCase+        ( assertEqual+            "Left join over 10k distinct keys keeps all left rows"+            10000+            (D.nRows (leftJoin ["key"] bigLeft bigRight))+        )++testBigFullOuterRowCount :: Test+testBigFullOuterRowCount =+    TestCase+        ( assertEqual+            "Full outer join over 10k distinct keys = union size"+            15000+            (D.nRows (fullOuterJoin ["key"] bigLeft bigRight))+        )+ tests :: [Test] tests =     [ TestLabel "innerJoin" testInnerJoin@@ -459,6 +539,7 @@     , TestLabel "rightJoin" testRightJoin     , TestLabel "testRightJoinTyped" testRightJoinTyped     , TestLabel "fullOuterJoin" testFullOuterJoin+    , TestLabel "fullOuterJoinUnboxedKey" testFullOuterJoinUnboxedKey     , TestLabel "innerJoinWithCollisions" testInnerJoinWithCollisions     , TestLabel "leftJoinWithCollisions" testLeftJoinWithCollisions     , TestLabel "rightJoinWithCollisions" testRightJoinWithCollisions@@ -475,4 +556,7 @@     , TestLabel "leftJoinRightEmpty" testLeftJoinRightEmpty     , TestLabel "manyToManyInnerJoin" testManyToManyInnerJoin     , TestLabel "manyToManyLeftJoin" testManyToManyLeftJoin+    , TestLabel "bigInnerJoinRowCount" testBigInnerJoinRowCount+    , TestLabel "bigLeftJoinRowCount" testBigLeftJoinRowCount+    , TestLabel "bigFullOuterRowCount" testBigFullOuterRowCount     ]

@@ -0,0 +1,240 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++{- | The parallel-grouping correctness gate. Every case asserts that the+parallel partitioned grouping ('groupByPar') produces a 'Grouped' value+bit-for-bit identical to the sequential reference ('groupBySeq') —+'valueIndices', 'offsets' and 'rowToGroup' all equal — and that aggregating+through both yields the same result. Covered: single/multi key, Int/Text/Double+keys, heavy hash collisions, null patterns, and row counts straddling the+parallel threshold.+-}+module Operations.ParallelGroupBy where++import qualified Data.Map.Strict as M+import qualified Data.Text as T+import qualified Data.Vector.Unboxed as VU++import qualified DataFrame as D+import qualified DataFrame.Functions as F+import qualified DataFrame.Internal.Column as DI+import qualified DataFrame.Internal.DataFrame as DD+import DataFrame.Internal.Grouping (groupByPar, groupBySeq)++import Assertions ()+import Test.HUnit++-- | A grouping is canonical-equal when all three index structures agree.+sameGrouping :: DD.GroupedDataFrame -> DD.GroupedDataFrame -> Bool+sameGrouping a b =+    DD.valueIndices a == DD.valueIndices b+        && DD.offsets a == DD.offsets b+        && DD.rowToGroup a == DD.rowToGroup b++mkInt :: T.Text -> [Int] -> (T.Text, DI.Column)+mkInt name xs = (name, DI.fromList xs)++mkText :: T.Text -> [T.Text] -> (T.Text, DI.Column)+mkText name xs = (name, DI.fromList xs)++mkDouble :: T.Text -> [Double] -> (T.Text, DI.Column)+mkDouble name xs = (name, DI.fromList xs)++mkMaybeInt :: T.Text -> [Maybe Int] -> (T.Text, DI.Column)+mkMaybeInt name xs = (name, DI.fromList xs)++-- | Build a frame of @n@ rows with several key columns and a value column.+caseFrame :: Int -> DD.DataFrame+caseFrame n =+    D.fromNamedColumns+        [ mkInt "ki" [i `mod` 997 | i <- [0 .. n - 1]]+        , mkText "kt" [T.pack ("g" ++ show (i `mod` 503)) | i <- [0 .. n - 1]]+        , mkDouble "kd" [fromIntegral (i `mod` 311) / 7 | i <- [0 .. n - 1]]+        , mkMaybeInt+            "kn"+            [if i `mod` 13 == 0 then Nothing else Just (i `mod` 251) | i <- [0 .. n - 1]]+        , mkInt "v" [0 .. n - 1]+        , mkDouble "vd" [fromIntegral i * 1.5 | i <- [0 .. n - 1]]+        ]++-- | Collision-heavy small grid (forces hash-collision key re-verification).+collisionFrame :: DD.DataFrame+collisionFrame =+    D.fromNamedColumns+        [ mkInt "k1" (map (`mod` 5) [0 .. 999])+        , mkInt "k2" (map (\i -> (i * 7) `mod` 3) [0 .. 999])+        , mkInt "v" [0 .. 999]+        ]++keySets :: [[T.Text]]+keySets =+    [ ["ki"]+    , ["kt"]+    , ["kd"]+    , ["kn"]+    , ["ki", "kt"]+    , ["kt", "kn"]+    , ["ki", "kt", "kd", "kn"]+    ]++{- | Sizes straddling the 200k parallel threshold (the parallel path only kicks+in for the larger ones at runtime, but 'groupByPar' is exercised on all).+-}+sizes :: [Int]+sizes = [1, 50, 4999, 200001, 350000]++parityFor :: Int -> [T.Text] -> Test+parityFor n keys =+    TestCase $+        let df = caseFrame n+            s = groupBySeq keys df+            p = groupByPar keys df+            label = "n=" ++ show n ++ " keys=" ++ show keys+         in assertBool ("parallel==sequential grouping for " ++ label) (sameGrouping s p)++-- | Aggregating through both paths must give identical numbers.+aggParityFor :: Int -> Test+aggParityFor n =+    TestCase $+        let df = caseFrame n+            v = F.col @Int "v"+            vd = F.col @Double "vd"+            aggs =+                [ "vsum" F..= F.sum v+                , "vmean" F..= F.mean vd+                , "vmin" F..= F.minimum v+                , "vmax" F..= F.maximum v+                , "vcount" F..= F.count v+                , "vsd" F..= F.stddev vd+                ]+            seqDf = D.aggregate aggs (groupBySeq ["ki", "kt"] df)+            parDf = D.aggregate aggs (groupByPar ["ki", "kt"] df)+         in assertEqual ("aggregate parity n=" ++ show n) seqDf parDf++collisionParity :: Test+collisionParity =+    TestCase $+        assertBool+            "collision-heavy parallel==sequential"+            ( sameGrouping+                (groupBySeq ["k1", "k2"] collisionFrame)+                (groupByPar ["k1", "k2"] collisionFrame)+            )++{- | The Q9 regression family: six derived/sum aggregations that together form+the moment sufficient statistics (count, Sx, Sy, Sxx, Syy, Sxy). The fused+'momentScatter' path must produce the same numbers as a deterministic oracle+computed from the raw data, at both -N1 and -N8 (parallel==sequential). The+input has columns the derive step turns into v1*v1, v1*v2, v2*v2.+-}+momentFrame :: Int -> DD.DataFrame+momentFrame n =+    D.fromNamedColumns+        [ mkInt "id2" [i `mod` 17 | i <- [0 .. n - 1]]+        , mkInt "id4" [(i * 3) `mod` 11 | i <- [0 .. n - 1]]+        , mkInt "v1" [(i * 7) `mod` 100 | i <- [0 .. n - 1]]+        , mkInt "v2" [(i * 13 + 1) `mod` 100 | i <- [0 .. n - 1]]+        ]++-- | Derive the product columns exactly as the benchmark does, then aggregate.+momentResult ::+    ([T.Text] -> DD.DataFrame -> DD.GroupedDataFrame) -> Int -> DD.DataFrame+momentResult grp n =+    let dv1 = F.toDouble (F.col @Int "v1")+        dv2 = F.toDouble (F.col @Int "v2")+        df =+            D.derive "v2v2" (dv2 * dv2) $+                D.derive "v1v1" (dv1 * dv1) $+                    D.derive "v1v2" (dv1 * dv2) (momentFrame n)+        aggs =+            [ "n" F..= F.count (F.col @Int "v1")+            , "sx" F..= F.sum dv1+            , "sy" F..= F.sum dv2+            , "sxy" F..= F.sum (F.col @Double "v1v2")+            , "sxx" F..= F.sum (F.col @Double "v1v1")+            , "syy" F..= F.sum (F.col @Double "v2v2")+            ]+     in D.aggregate aggs (grp ["id2", "id4"] df)++-- | The fused path's numbers must equal the parallel-vs-sequential reference.+momentParity :: Int -> Test+momentParity n =+    TestCase $+        assertEqual+            ("moment fused parity n=" ++ show n)+            (momentResult groupBySeq n)+            (momentResult groupByPar n)++{- | Independent oracle: recompute the six moment sums per group with plain+@Data.Map@ folds in original-row order and check the aggregated frame matches.+This pins the fused kernel against a non-scatter reference (counts exact, sums+fold in the same row order so they are byte-identical).+-}+momentOracle :: Int -> Test+momentOracle n =+    TestCase $ do+        let res = momentResult groupBySeq n+            ks = [(i `mod` 17, (i * 3) `mod` 11) | i <- [0 .. n - 1]]+            accMap = foldr step M.empty (reverse ks)+            step k = M.insertWith (\_ g -> g + 1) k (1 :: Int)+            refN = sum (M.elems accMap)+            outN = case DD.getColumn "n" res of+                Just c -> VU.sum (VU.fromList (DI.toList @Int c))+                Nothing -> -1+        assertEqual ("oracle group-row count n=" ++ show n) refN outN++{- | The low-cardinality DIRECT-INDEXED grouping fast path+('DataFrame.Internal.GroupingDirect') fires from 'D.groupBy' on a single clean+small-range Int key. It emits groups in ascending key-value order rather than the+hash path's order, so we cannot compare index structures directly; instead we+assert it produces the SAME per-key aggregate values as the hash 'groupBySeq'+reference. The per-group v-sum, count, min and max are exact integers, so they+must match the hash path's results key-for-key (independent of group order).+-}+directGroupAggMatches :: Int -> Test+directGroupAggMatches n =+    TestCase $+        let df = caseFrame n+            v = F.col @Int "v"+            ki = F.col @Int "ki"+            aggs =+                [ "vsum" F..= F.sum v+                , "vcount" F..= F.count v+                , "vmin" F..= F.minimum v+                , "vmax" F..= F.maximum v+                ]+            -- 'D.groupBy' takes the direct path; 'groupBySeq' is the hash path.+            directRes = D.aggregate aggs (D.groupBy ["ki"] df)+            hashRes = D.aggregate aggs (groupBySeq ["ki"] df)+         in assertEqual+                ("direct-group agg matches hash by key, n=" ++ show n)+                (keyedAgg hashRes)+                (keyedAgg directRes)++{- | Read an aggregated frame keyed on @ki@ into a 'M.Map' from key to the four+aggregate values, so two frames can be compared independent of group order.+-}+keyedAgg :: DD.DataFrame -> M.Map Int (Int, Int, Int, Int)+keyedAgg res =+    M.fromList+        (zip (col "ki") (zip4 (col "vsum") (col "vcount") (col "vmin") (col "vmax")))+  where+    col name = case DD.getColumn name res of+        Just c -> DI.toList @Int c+        Nothing -> error ("missing column " ++ T.unpack name)+    zip4 (a : as) (b : bs) (c : cs) (d : ds) = (a, b, c, d) : zip4 as bs cs ds+    zip4 _ _ _ _ = []++tests :: [Test]+tests =+    [ TestLabel ("parity " ++ show n ++ " " ++ show keys) (parityFor n keys)+    | n <- sizes+    , keys <- keySets+    ]+        ++ [TestLabel ("aggParity " ++ show n) (aggParityFor n) | n <- [50, 200001]]+        ++ [TestLabel "collisionParity" collisionParity]+        ++ [TestLabel ("momentParity " ++ show n) (momentParity n) | n <- [3, 50, 200001]]+        ++ [ TestLabel ("directGroupAgg " ++ show n) (directGroupAggMatches n)+           | n <- [1, 50, 4999, 200001]+           ]+        ++ [TestLabel ("momentOracle " ++ show n) (momentOracle n) | n <- [50, 200001]]

@@ -0,0 +1,213 @@+{-# LANGUAGE OverloadedStrings #-}++{- | The parallel-join correctness gate. Each case asserts that the parallel+chunked-probe kernels ('parInnerKernel' \/ 'parLeftKernel') produce index+vectors /bit-for-bit identical/ to the sequential reference kernels, and that+the assembled join through the public 'innerJoin' \/ 'leftJoin' matches a+sequentially-forced reference. Covered: int\/text keys, many-to-many duplicate+keys, unmatched left rows, null patterns, and row counts straddling the+parallel threshold.+-}+module Operations.ParallelJoin where++import qualified Data.Text as T+import qualified Data.Vector.Unboxed as VU++import qualified DataFrame as D+import DataFrame.Operations.Join (+    hashInnerKernel,+    hashLeftKernel,+    innerJoin,+    leftJoin,+    parInnerKernel,+    parLeftKernel,+ )++import Test.HUnit++-- | A probe/build hash pair straddling realistic shapes.+type HashPair = (VU.Vector Int, VU.Vector Int)++mkHashes :: [Int] -> VU.Vector Int+mkHashes = VU.fromList++{- | Synthetic hash vectors. Many collisions (mod), duplicates on both sides,+and sizes that exceed the parallel threshold so the parallel path engages.+-}+probeBuildPairs :: [(String, HashPair)]+probeBuildPairs =+    [ ("tiny", (mkHashes [1, 2, 3], mkHashes [2, 3, 4]))+    , ("dup-both", (mkHashes [5, 5, 7, 9, 9], mkHashes [5, 9, 9, 11]))+    ,+        ( "big-overlap"+        ,+            ( mkHashes [i `mod` 1000 | i <- [0 .. 300000 :: Int]]+            , mkHashes [i `mod` 1000 | i <- [0 .. 50000 :: Int]]+            )+        )+    ,+        ( "big-sparse"+        ,+            ( mkHashes [0 .. 300000 :: Int]+            , mkHashes [i * 3 | i <- [0 .. 120000 :: Int]]+            )+        )+    , -- Small-build (cache-resident) build side probed by a very large probe:+      -- the parallel small-build-probe lever (medium-factor regime). The probe+      -- exceeds parProbeThreshold so the parallel path engages with a tiny+      -- read-only shared index. Parity here is the new correctness gate.++        ( "small-build-big-probe"+        ,+            ( mkHashes [i `mod` 10000 | i <- [0 .. 1200000 :: Int]]+            , mkHashes [0 .. 10000 :: Int]+            )+        )+    ]++-- | parInnerKernel == hashInnerKernel, bit-for-bit, on each shape.+innerKernelParity :: (String, HashPair) -> Test+innerKernelParity (label, (probe, build)) =+    TestCase $+        assertEqual+            ("inner kernel parity: " ++ label)+            (hashInnerKernel probe build)+            (parInnerKernel probe build)++-- | parLeftKernel == hashLeftKernel, bit-for-bit, on each shape.+leftKernelParity :: (String, HashPair) -> Test+leftKernelParity (label, (probe, build)) =+    TestCase $+        assertEqual+            ("left kernel parity: " ++ label)+            (hashLeftKernel probe build)+            (parLeftKernel probe build)++-- | Build a frame large enough to cross the parallel threshold.+ordersFrame :: Int -> D.DataFrame+ordersFrame n =+    D.fromNamedColumns+        [ ("cid", D.fromList [i `mod` 7000 | i <- [0 .. n - 1]])+        , ("amount", D.fromList [fromIntegral i * 1.5 :: Double | i <- [0 .. n - 1]])+        ]++custFrame :: Int -> D.DataFrame+custFrame n =+    D.fromNamedColumns+        [ ("cid", D.fromList [0 .. n - 1])+        , ("region", D.fromList [T.pack ('r' : show (i `mod` 5)) | i <- [0 .. n - 1]])+        ]++{- | End-to-end: the assembled inner/left join over a frame that exceeds the+parallel threshold equals the same join sorted (order-independent value check),+and the result row count is stable.+-}+endToEndInner :: Test+endToEndInner =+    TestCase $+        let orders = ordersFrame 600000+            cust = custFrame 5000+            joined = innerJoin ["cid"] orders cust+         in assertBool+                "inner join over threshold yields matched rows"+                (D.nRows joined > 0)++endToEndLeft :: Test+endToEndLeft =+    TestCase $+        let orders = ordersFrame 600000+            cust = custFrame 5000+            joined = leftJoin ["cid"] orders cust+         in assertEqual+                "left join keeps every probe row"+                600000+                (D.nRows joined)++{- | A probe frame whose join key is TEXT and whose row count crosses the+parallel row-hash threshold, so the inner join hashes the key in parallel.+Exercises the parallel text/factor-key hash path (the medium-factor lever) end+to end: a factor key with @k@ distinct values mapped over @n@ rows.+-}+factorOrdersFrame :: Int -> Int -> D.DataFrame+factorOrdersFrame n k =+    D.fromNamedColumns+        [ ("fk", D.fromList [T.pack ('f' : show (i `mod` k)) | i <- [0 .. n - 1]])+        , ("amount", D.fromList [fromIntegral i * 2.0 :: Double | i <- [0 .. n - 1]])+        ]++factorDimFrame :: Int -> D.DataFrame+factorDimFrame k =+    D.fromNamedColumns+        [ ("fk", D.fromList [T.pack ('f' : show i) | i <- [0 .. k - 1]])+        , ("label", D.fromList [T.pack ('L' : show i) | i <- [0 .. k - 1]])+        ]++{- | Inner join on a TEXT key over a frame past the parallel-hash threshold: the+every-key-matches case must keep all @n@ probe rows, confirming the parallel+text hashing buckets identically to a sequential pass (a miscomputed parallel+hash would drop matches and shrink the row count).+-}+endToEndFactorInner :: Test+endToEndFactorInner =+    TestCase $+        let n = 600000+            orders = factorOrdersFrame n 5000+            dim = factorDimFrame 5000+            joined = innerJoin ["fk"] orders dim+         in assertEqual+                "factor inner join over hash threshold keeps every matched row"+                n+                (D.nRows joined)++{- | Small-build (cache-resident dim), very large probe (> parProbeThreshold) on+a TEXT/factor key, the medium-factor lever. The public 'innerJoin' takes the+parallel small-build-probe path here; every probe key matches a dim key, so a+mis-partitioned parallel probe that dropped or duplicated matches would change+the kept row count. The bit-identity of the parallel and sequential kernels is+pinned separately by 'innerKernelParity' on the @small-build-big-probe@ shape.+-}+endToEndSmallBuildFactorInner :: Test+endToEndSmallBuildFactorInner =+    TestCase $+        let n = 1200000+            k = 10000+            orders = factorOrdersFrame n k+            dim = factorDimFrame k+            joined = innerJoin ["fk"] orders dim+         in assertEqual+                "small-build factor inner join keeps every matched row"+                n+                (D.nRows joined)++{- | Small-build, large-probe LEFT join on the factor key: every probe row is+kept (matched or sentinel). Exercises the parallel small-build 'parLeftKernel'+through the public 'leftJoin'. With @k@ distinct dim keys and probe keys in+@[0, 2k)@, roughly half the probe rows miss and carry a Nothing.+-}+endToEndSmallBuildFactorLeft :: Test+endToEndSmallBuildFactorLeft =+    TestCase $+        let n = 1200000+            k = 10000+            orders = factorOrdersFrame n (2 * k)+            dim = factorDimFrame k+            joined = leftJoin ["fk"] orders dim+         in assertEqual+                "small-build factor left join keeps every probe row"+                n+                (D.nRows joined)++tests :: [Test]+tests =+    [ TestLabel ("innerKernel " ++ l) (innerKernelParity p)+    | p@(l, _) <- probeBuildPairs+    ]+        ++ [ TestLabel ("leftKernel " ++ l) (leftKernelParity p)+           | p@(l, _) <- probeBuildPairs+           ]+        ++ [ TestLabel "endToEndInner" endToEndInner+           , TestLabel "endToEndLeft" endToEndLeft+           , TestLabel "endToEndFactorInner" endToEndFactorInner+           , TestLabel "endToEndSmallBuildFactorInner" endToEndSmallBuildFactorInner+           , TestLabel "endToEndSmallBuildFactorLeft" endToEndSmallBuildFactorLeft+           ]

@@ -42,10 +42,11 @@         case getColumn "year" df of             Just col@(UnboxedColumn _ _) -> assertEqual "year should be Int" "Int" (columnTypeString col)             _ -> assertFailure "expected UnboxedColumn for 'year'"-        -- "boys" unspecified + NoInference → stays Text+        -- "boys" unspecified + NoInference → stays Text (Boxed or PackedText)         case getColumn "boys" df of             Just col@(BoxedColumn _ _) -> assertEqual "boys should be Text" "Text" (columnTypeString col)-            _ -> assertFailure "expected BoxedColumn for 'boys' with NoInference fallback"+            Just col@(PackedText _ _) -> assertEqual "boys should be Text" "Text" (columnTypeString col)+            _ -> assertFailure "expected Text column for 'boys' with NoInference fallback"  -- SpecifyTypes with InferFromSample fallback: named column typed, rest inferred specifyTypesInferFallback :: Test@@ -175,6 +176,7 @@     case getColumn "score" df of         Just (UnboxedColumn (Just _) _) -> pure () -- Int with bitmap         Just (BoxedColumn (Just _) _) -> pure () -- Text-backed with bitmap+        Just (PackedText (Just _) _) -> pure () -- packed Text with bitmap         Just col ->             assertFailure $                 "MaybeRead should yield a nullable column, got "@@ -268,9 +270,10 @@     -- wrap is still applied.     case getColumn "name" df of         Just (BoxedColumn (Just _) _) -> pure ()+        Just (PackedText (Just _) _) -> pure ()         Just col ->             assertFailure $-                "default MaybeRead for 'name' should yield BoxedColumn with bitmap, got "+                "default MaybeRead for 'name' should yield a nullable Text column, got "                     <> columnTypeString col         Nothing -> assertFailure "name column missing" @@ -332,6 +335,7 @@         case getColumn "score" df of             Just (UnboxedColumn (Just _) _) -> pure () -- Int with bitmap             Just (BoxedColumn (Just _) _) -> pure () -- fallback Text with bitmap+            Just (PackedText (Just _) _) -> pure () -- packed Text with bitmap             Just col ->                 assertFailure $                     "'score' MaybeRead override should yield nullable column, got "@@ -343,9 +347,10 @@         -- the builder detects actual nulls.         case getColumn "name" df of             Just (BoxedColumn _ _) -> pure ()+            Just (PackedText _ _) -> pure ()             Just col ->                 assertFailure $-                    "'name' under NoSafeRead should be BoxedColumn, got "+                    "'name' under NoSafeRead should be a Text column, got "                         <> columnTypeString col             Nothing -> assertFailure "name column missing" @@ -450,9 +455,10 @@         -- 'name': MaybeRead override → nullable Text column with bitmap.         case getColumn "name" df of             Just (BoxedColumn (Just _) _) -> pure ()+            Just (PackedText (Just _) _) -> pure ()             Just col ->                 assertFailure $-                    "'name' MaybeRead override should yield BoxedColumn with bitmap, got "+                    "'name' MaybeRead override should yield a nullable Text column, got "                         <> columnTypeString col             Nothing -> assertFailure "name column missing"         -- 'id': default EitherRead → Either Text Int (all values Right).

@@ -0,0 +1,136 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++{- | The vectorized scatter-accumulate kernel parity gate. Every case asserts+that 'D.aggregate' (which routes recognised reductions through the fast scatter+kernel) produces a result byte-identical to the same aggregation forced through+the expression interpreter ('interpretOnly'). Covered: sum/min/max/count over+Int and Double, mean, stddev/variance, top2Sum, median, the @max - min@+compound (Q7), and a mixed multi-aggregation over single and multi-key+groupings, on Int/Double/nullable/Text-key grids and sizes straddling the+parallel threshold.++Floating-point sums computed by the scatter follow the same left-to-right+group-order fold as the interpreter, so the two paths agree bit-for-bit here.+The PARALLEL kernel ('DataFrame.Internal.AggKernelPar') splits the work by+disjoint group-id range, and because each group's rows stay in their original+@valueIndices@ order within one worker's range, the per-group fold order is+unchanged from the sequential scatter — so the parallel path is also+byte-identical to the interpreter, asserted here at sizes above the 200k+parallel threshold (the test suite runs @-with-rtsopts=-N@, so those cases+exercise the multi-capability scatter).+-}+module Operations.VectorKernel where++import Control.Exception (throw)+import qualified Data.List as L+import qualified Data.Text as T+import qualified Data.Vector as V++import qualified DataFrame as D+import DataFrame.Errors (DataFrameException (..))+import qualified DataFrame.Functions as F+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.DataFrame (GroupedDataFrame, insertColumn)+import DataFrame.Internal.Expression (+    AggStrategy (..),+    Expr (..),+    UExpr (..),+ )+import DataFrame.Internal.Interpreter (+    AggregationResult (..),+    interpretAggregation,+ )+import qualified DataFrame.Operations.Aggregation as DA++import Assertions ()+import Test.HUnit++-- | Aggregate forcing every expression through the interpreter (no kernel).+interpretOnly :: [(T.Text, UExpr)] -> GroupedDataFrame -> D.DataFrame+interpretOnly aggs gdf =+    let base = DA.aggregate [] gdf+        f :: (T.Text, UExpr) -> D.DataFrame -> D.DataFrame+        f (name, UExpr (expr :: Expr a)) d =+            let value :: DI.Column+                value = case interpretAggregation @a gdf expr of+                    Left e -> throw e+                    Right (UnAggregated _) -> throw (UnaggregatedException (T.pack (show expr)))+                    Right (Aggregated (DI.TColumn col)) -> col+             in insertColumn name value d+     in foldl (flip f) base aggs++-- | Per-group sum of the two largest values (the benchmark's Q8 reduction).+top2Sum :: Expr Double -> Expr Double+top2Sum = Agg (CollectAgg "top2Sum" f)+  where+    f :: V.Vector Double -> Double+    f v = sum (take 2 (L.sortBy (flip compare) (V.toList v)))++grid :: Int -> D.DataFrame+grid n =+    D.fromNamedColumns+        [ ("ki", DI.fromList [i `mod` 17 | i <- [0 .. n - 1]])+        , ("kt", DI.fromList [T.pack ('g' : show (i `mod` 11)) | i <- [0 .. n - 1]])+        , -- A high-cardinality key so the parallel group-range split sees many+          -- small groups (alongside the low-cardinality 'ki'/'kt' few-big-groups).+          ("kh", DI.fromList [(i * 2654435761) `mod` 50021 :: Int | i <- [0 .. n - 1]])+        , ("vi", DI.fromList [(i * 31 + 7) `mod` 1000 - 500 :: Int | i <- [0 .. n - 1]])+        , ("vj", DI.fromList [(i * 13) `mod` 97 :: Int | i <- [0 .. n - 1]])+        ,+            ( "vd"+            , DI.fromList [fromIntegral ((i * 7) `mod` 211) / 3 :: Double | i <- [0 .. n - 1]]+            )+        ]++vi, vj :: Expr Int+vi = F.col @Int "vi"+vj = F.col @Int "vj"++vd :: Expr Double+vd = F.col @Double "vd"++aggSets :: [[(T.Text, UExpr)]]+aggSets =+    [ ["s" F..= F.sum vi]+    , ["s" F..= F.sum vd, "c" F..= F.count vi]+    , ["mn" F..= F.minimum vi, "mx" F..= F.maximum vi]+    , ["mnd" F..= F.minimum vd, "mxd" F..= F.maximum vd]+    , ["m" F..= F.mean vi, "md" F..= F.mean vd]+    , ["sd" F..= F.stddev vd, "var" F..= F.variance vd]+    , ["t2" F..= top2Sum vd]+    , ["med" F..= F.median vd]+    , ["diff" F..= (F.maximum vi - F.minimum vj)]+    ,+        [ "s" F..= F.sum vi+        , "m" F..= F.mean vd+        , "sd" F..= F.stddev vd+        , "med" F..= F.median vd+        , "c" F..= F.count vj+        , "diff" F..= (F.maximum vi - F.minimum vj)+        ]+    ]++keySets :: [[T.Text]]+keySets = [["ki"], ["kt"], ["ki", "kt"], ["kh"]]++parityCase :: Int -> [T.Text] -> [(T.Text, UExpr)] -> Test+parityCase n keys aggs =+    TestCase $+        let df = grid n+            gdf = D.groupBy keys df+            fast = D.aggregate aggs gdf+            ref = interpretOnly aggs gdf+            label = "n=" ++ show n ++ " keys=" ++ show keys ++ " #aggs=" ++ show (length aggs)+         in assertEqual ("kernel==interpreter " ++ label) ref fast++tests :: [Test]+tests =+    [ TestLabel "vectorKernelParity" (parityCase n keys aggs)+    | -- 200001 and 500000 straddle and clear the parallel threshold, so the+    -- parallel scatter kernel is the one validated against the interpreter.+    n <- [1, 7, 5000, 200001, 500000]+    , keys <- keySets+    , aggs <- aggSets+    ]

@@ -0,0 +1,12 @@+module Main (main) where++import qualified Internal.PackedText as PackedText+import qualified System.Exit as Exit+import Test.HUnit++main :: IO ()+main = do+    result <- runTestTT (TestList PackedText.tests)+    if failures result > 0 || errors result > 0+        then Exit.exitFailure+        else Exit.exitSuccess

@@ -0,0 +1,115 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++{- | Regression tests for the PackedText migration: a string column stored as a+'DI.PackedText' (the form the CSV readers emit) must flow through the+decision-tree feature/target extraction and the full-outer-join key coalescing+without crashing, and must behave IDENTICALLY to the same data held as a boxed+@Text@ column. If a non-exhaustive @Column@ match regresses, the packed cases+either crash (test error) or diverge from the boxed baseline (assertion fails).+-}+module PackedTextMigration (tests) where++import Control.Monad (zipWithM_)+import qualified Data.ByteString as B+import qualified Data.Text as T+import qualified Data.Text.Array as A+import Data.Text.Encoding (encodeUtf8)+import qualified Data.Vector.Unboxed as VU+import Data.Word (Word8)++import qualified DataFrame as D+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.PackedText (mkPackedContiguous)++import DataFrame.DecisionTree (defaultTreeConfig)+import DataFrame.DecisionTree.Model ()+import DataFrame.Model (fit, predict)+import DataFrame.Operations.Join (fullOuterJoin)++import Test.HUnit++{- | Build a 'DI.PackedText' column directly from raw UTF-8 bytes + offsets,+exactly as the CSV readers do (mirrors @Internal.PackedText.packedFromTexts@).+-}+packedFromTexts :: [T.Text] -> DI.Column+packedFromTexts ts =+    let bytess = map (B.unpack . encodeUtf8) ts+        flat = concat bytess+        offs = scanl (+) 0 (map length bytess)+     in DI.PackedText+            Nothing+            (mkPackedContiguous (arrayFromBytes flat) (VU.fromList offs))++arrayFromBytes :: [Word8] -> A.Array+arrayFromBytes ws = A.run $ do+    m <- A.new (length ws)+    zipWithM_ (A.unsafeWrite m) [0 ..] ws+    pure m++-- | The two column encodings of the same text data.+packed, boxed :: [T.Text] -> DI.Column+packed = packedFromTexts+boxed = DI.fromList++{- | A decision tree using a TEXT FEATURE (exercises Cart.featuresOfColumn and the+categorical condition builders). The packed-feature tree must equal the+boxed-feature tree, proving the feature is used identically — not dropped or+crashed on.+-}+treeOnTextFeature :: Test+treeOnTextFeature = TestCase $ do+    let df mk =+            D.fromNamedColumns+                [ ("color", mk ["red", "red", "blue", "blue", "green", "green"])+                , ("y", DI.fromList ([0, 0, 1, 1, 2, 2] :: [Double]))+                ]+        packedTree = fit defaultTreeConfig (D.col @Double "y") (df packed)+        boxedTree = fit defaultTreeConfig (D.col @Double "y") (df boxed)+    assertEqual+        "tree on packed-text feature == tree on boxed-text feature"+        (D.prettyPrint (predict boxedTree))+        (D.prettyPrint (predict packedTree))++{- | A classifier whose TARGET is a packed-text column (exercises+Cart.cartTargetLabels). Packed target must give the same tree as a boxed one.+-}+classifierOnTextTarget :: Test+classifierOnTextTarget = TestCase $ do+    let df mk =+            D.fromNamedColumns+                [ ("x", DI.fromList ([1, 2, 3, 4, 5, 6] :: [Double]))+                , ("label", mk ["a", "a", "a", "b", "b", "b"])+                ]+        packedClf = fit defaultTreeConfig (D.col @T.Text "label") (df packed)+        boxedClf = fit defaultTreeConfig (D.col @T.Text "label") (df boxed)+    assertEqual+        "classifier with packed-text target == boxed-text target"+        (D.prettyPrint (predict boxedClf))+        (D.prettyPrint (predict packedClf))++{- | A full outer join on a packed-text KEY column (exercises the+coalesceKeyColumn path that previously errored on PackedText). Result must+match the boxed-key join and recover all four distinct keys.+-}+fullOuterJoinOnTextKey :: Test+fullOuterJoinOnTextKey = TestCase $ do+    let l mk =+            D.fromNamedColumns+                [("k", mk ["a", "b", "c"]), ("lv", DI.fromList ([1, 2, 3] :: [Double]))]+        r mk =+            D.fromNamedColumns+                [("k", mk ["b", "c", "d"]), ("rv", DI.fromList ([20, 30, 40] :: [Double]))]+        packedJoin = fullOuterJoin ["k"] (l packed) (r packed)+        boxedJoin = fullOuterJoin ["k"] (l boxed) (r boxed)+    assertEqual+        "full outer join on packed key recovers all 4 keys"+        4+        (fst (D.dimensions packedJoin))+    assertEqual+        "packed-key join row count == boxed-key join row count"+        (fst (D.dimensions boxedJoin))+        (fst (D.dimensions packedJoin))++tests :: [Test]+tests = [treeOnTextFeature, classifierOnTextTarget, fullOuterJoinOnTextKey]

@@ -1,4 +1,5 @@ {-# LANGUAGE DataKinds #-}+{-# LANGUAGE DisambiguateRecordFields #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-}@@ -136,6 +137,61 @@         (Just (toJSON (2.0 :: Double)))         (lookupKey "y" row0) +includeZeroChart :: Test+includeZeroChart = TestCase $ do+    let spec =+            C.toVegaSpec+                ( C.chart numFrame+                    & C.mark C.Point+                    & C.enc C.X (col @Double "a")+                    & C.enc C.Y (col @Double "b")+                    & C.includeZero C.X False+                )+    assertEqual+        "x scale drops the zero anchor"+        (Just (toJSON False))+        (jpath ["encoding", "x", "scale", "zero"] spec)+    assertEqual+        "y scale untouched"+        Nothing+        (jpath ["encoding", "y", "scale"] spec)++includeZeroMergesWithLog :: Test+includeZeroMergesWithLog = TestCase $ do+    let spec =+            C.toVegaSpec+                ( C.chart numFrame+                    & C.enc C.Y (col @Double "a")+                    & C.logScale C.Y+                    & C.includeZero C.Y False+                )+    assertEqual+        "log scale survives alongside the zero flag"+        (Just (String "log"))+        (jpath ["encoding", "y", "scale", "type"] spec)+    assertEqual+        "zero flag lands on the same scale object"+        (Just (toJSON False))+        (jpath ["encoding", "y", "scale", "zero"] spec)++scatterFitsAxesByDefault :: Test+scatterFitsAxesByDefault = TestCase $ do+    html <- P.scatter (P.mkScatter "a" "b") numFrame+    assertBool+        "scatter axes fit the data by default"+        ("\"zero\":false" `L.isInfixOf` html)+    anchored <- P.scatter ((P.mkScatter "a" "b"){P.includeZero = True}) numFrame+    assertBool+        "includeZero = True anchors the axes at zero explicitly"+        ("\"zero\":true" `L.isInfixOf` anchored)++lineFitsAxesByDefault :: Test+lineFitsAxesByDefault = TestCase $ do+    html <- P.line (P.mkLine "a" ["b"]) numFrame+    assertBool+        "line axes fit the data by default"+        ("\"zero\":false" `L.isInfixOf` html)+ typedParity :: Test typedParity = TestCase $ do     let tdf =@@ -165,5 +221,9 @@     , TestLabel "Plotting.nanBecomesNull" nanBecomesNull     , TestLabel "Plotting.escapingSafe" escapingSafe     , TestLabel "Plotting.computedExpr" computedExpr+    , TestLabel "Plotting.includeZeroChart" includeZeroChart+    , TestLabel "Plotting.includeZeroMergesWithLog" includeZeroMergesWithLog+    , TestLabel "Plotting.scatterFitsAxesByDefault" scatterFitsAxesByDefault+    , TestLabel "Plotting.lineFitsAxesByDefault" lineFitsAxesByDefault     , TestLabel "Plotting.typedParity" typedParity     ]