From 74ae891766d5bd015ce881c0d305ecbf5328a438 Mon Sep 17 00:00:00 2001 From: skymanbp <57272723+skymanbp@users.noreply.github.com> Date: Wed, 19 Aug 2026 03:00:05 -0400 Subject: [PATCH] fix: statistics ignored the null bitmap Stats read the raw payload of nullable columns, so the sentinel stored at null slots entered every result (a zero for unboxed columns, an error thunk for boxed ones): - mean/median/percentile/genericPercentile/stddev/skewness/variance/ IQR/sum now drop null slots via a shared dropNulls view. - correlation does listwise deletion over both columns. - valueCounts/valueProportions (and so frequencies) no longer count the sentinel as a category. - Maybe-typed views are untouched: their nulls are real values. --- .../src-internal/DataFrame/Internal/Column.hs | 18 +- .../src/DataFrame/Operations/Core.hs | 20 +- .../src/DataFrame/Operations/Statistics.hs | 64 +++--- tests/Operations/Statistics.hs | 183 ++++++++++++++++++ 4 files changed, 254 insertions(+), 31 deletions(-) diff --git a/dataframe-core/src-internal/DataFrame/Internal/Column.hs b/dataframe-core/src-internal/DataFrame/Internal/Column.hs index 1395b9b4..6f76f1df 100644 --- a/dataframe-core/src-internal/DataFrame/Internal/Column.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Column.hs @@ -39,7 +39,7 @@ import Data.Bits ( ) import Data.Kind (Type) import Data.Maybe -import Data.Type.Equality (TestEquality (..)) +import Data.Type.Equality (TestEquality (..), type (:~~:) (HRefl)) import Data.Word (Word8) import DataFrame.Errors import DataFrame.Internal.PackedText ( @@ -209,6 +209,22 @@ columnBitmap (UnboxedColumn bm _) = bm columnBitmap (PackedText bm _) = bm columnBitmap (MergedColumn _ _) = Nothing +{- | Drop the null slots of a payload-typed view of a nullable column: those +slots hold a sentinel, not a value. A @Maybe@-typed view already encodes the +nulls, so it is returned untouched. Backpermute never forces the kept-out +slots, so boxed error thunks at null slots are safe. +-} +dropNulls :: + forall v a. + (Typeable a, VG.Vector v a, VG.Vector v Int) => Maybe Bitmap -> v a -> v a +dropNulls Nothing xs = xs +dropNulls (Just bm) xs = case typeRep @a of + App m _ | Just HRefl <- eqTypeRep m (typeRep @Maybe) -> xs + _ -> VG.backpermute xs keep + where + keep = VG.fromList [i | i <- [0 .. VG.length xs - 1], bitmapTestBit bm i] +{-# INLINE dropNulls #-} + {- | Decode a 'PackedText' into a @BoxedColumn Text@ (bit-identical to materializing at freeze). Identity on every other column. -} diff --git a/dataframe-operations/src/DataFrame/Operations/Core.hs b/dataframe-operations/src/DataFrame/Operations/Core.hs index 77df934f..ef7e66c9 100644 --- a/dataframe-operations/src/DataFrame/Operations/Core.hs +++ b/dataframe-operations/src/DataFrame/Operations/Core.hs @@ -67,8 +67,10 @@ import DataFrame.Internal.Column ( Column (..), Columnable, TypedColumn (..), + columnBitmap, columnLength, columnTypeString, + dropNulls, fromList, fromVector, materializeMerged, @@ -688,7 +690,7 @@ valueCounts :: forall a. (Ord a, Columnable a) => Expr a -> DataFrame -> [(a, Int)] valueCounts expr df | null df = throw (EmptyDataSetException "valueCounts") - | otherwise = case columnAsVector expr df of + | otherwise = case columnAsVectorNonNull expr df of Left e -> throw e Right column' -> let @@ -696,6 +698,20 @@ valueCounts expr df in M.toAscList column +-- | As 'columnAsVector', minus null slots: a sentinel is not a category. +columnAsVectorNonNull :: + forall a. + (Columnable a) => Expr a -> DataFrame -> Either DataFrameException (V.Vector a) +columnAsVectorNonNull expr df = case expr of + Col name -> case getColumn name df of + Just col -> withColumnName name (dropNulls (columnBitmap col) <$> toVector col) + Nothing -> + Left $ + ColumnsNotFoundException [name] "valueCounts" (M.keys $ columnIndices df) + _ -> case interpret df expr of + Left e -> throw e + Right (TColumn col) -> dropNulls (columnBitmap col) <$> toVector col + {- | O (k * n) Shows the proportions of each value in a given column. ==== __Example__ @@ -712,7 +728,7 @@ valueProportions :: forall a. (Ord a, Columnable a) => Expr a -> DataFrame -> [(a, Double)] valueProportions expr df | null df = throw (EmptyDataSetException "valueCounts") - | otherwise = case columnAsVector expr df of + | otherwise = case columnAsVectorNonNull expr df of Left e -> throw e Right column' -> let diff --git a/dataframe-operations/src/DataFrame/Operations/Statistics.hs b/dataframe-operations/src/DataFrame/Operations/Statistics.hs index 7322c0dc..c67d6d96 100644 --- a/dataframe-operations/src/DataFrame/Operations/Statistics.hs +++ b/dataframe-operations/src/DataFrame/Operations/Statistics.hs @@ -120,7 +120,7 @@ mean expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> mean' xs + Right xs -> mean' (dropNulls (columnBitmap col) xs) meanMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double @@ -137,13 +137,13 @@ meanMaybe expr df = case interpret @(Maybe a) df expr of median :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double median (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> median' xs + Right xs -> median' (dropNulls (colBitmap name df) xs) Left e -> throw e median expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> median' xs + Right xs -> median' (dropNulls (columnBitmap col) xs) -- | Calculates the median of a given column (containing optional values) as a standalone value. medianMaybe :: @@ -162,50 +162,50 @@ percentile :: forall a. (Columnable a, Real a, VU.Unbox a) => Int -> Expr a -> DataFrame -> Double percentile n (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> percentile' n xs + Right xs -> percentile' n (dropNulls (colBitmap name df) xs) Left e -> throw e percentile n expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> percentile' n xs + Right xs -> percentile' n (dropNulls (columnBitmap col) xs) -- | Calculates the nth percentile of a given column as a standalone value. genericPercentile :: forall a. (Columnable a, Ord a) => Int -> Expr a -> DataFrame -> a genericPercentile n (Col name) df = case columnAsVector (Col @a name) df of - Right xs -> percentileOrd' n xs + Right xs -> percentileOrd' n (dropNulls (colBitmap name df) xs) Left e -> throw e genericPercentile n expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toVector @a col of Left e -> throw e - Right xs -> percentileOrd' n xs + Right xs -> percentileOrd' n (dropNulls (columnBitmap col) xs) -- | Calculates the standard deviation of a given column as a standalone value. standardDeviation :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double standardDeviation (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> (sqrt . variance') xs + Right xs -> (sqrt . variance') (dropNulls (colBitmap name df) xs) Left e -> throw e standardDeviation expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> (sqrt . variance') xs + Right xs -> (sqrt . variance') (dropNulls (columnBitmap col) xs) -- | Calculates the skewness of a given column as a standalone value. skewness :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double skewness (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> skewness' xs + Right xs -> skewness' (dropNulls (colBitmap name df) xs) Left e -> throw e skewness expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> skewness' xs + Right xs -> skewness' (dropNulls (columnBitmap col) xs) -- | Calculates the variance of a given column as a standalone value. variance :: @@ -217,36 +217,44 @@ variance expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> variance' xs + Right xs -> variance' (dropNulls (columnBitmap col) xs) -- | Calculates the inter-quartile range of a given column as a standalone value. interQuartileRange :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double interQuartileRange (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> interQuartileRange' xs + Right xs -> interQuartileRange' (dropNulls (colBitmap name df) xs) Left e -> throw e interQuartileRange expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> interQuartileRange' xs + Right xs -> interQuartileRange' (dropNulls (columnBitmap col) xs) -- | Calculates the Pearson's correlation coefficient between two given columns as a standalone value. correlation :: T.Text -> T.Text -> DataFrame -> Maybe Double correlation first second df = do - f <- _getColumnAsDouble first df - s <- _getColumnAsDouble second df + -- Listwise deletion: a null in either column drops the pair. + let df' = filterJust first (filterJust second df) + f <- _getColumnAsDouble first df' + s <- _getColumnAsDouble second df' correlation' f s +-- | Bitmap of a named column, if it has one. +colBitmap :: T.Text -> DataFrame -> Maybe Bitmap +colBitmap name df = columnBitmap =<< getColumn name df + _getColumnAsDouble :: T.Text -> DataFrame -> Maybe (VU.Vector Double) _getColumnAsDouble name df = case getColumn name df of - Just (UnboxedColumn _ (f :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of - Just Refl -> Just f - Nothing -> case sIntegral @a of - STrue -> Just (VU.map fromIntegral f) - SFalse -> case sFloating @a of - STrue -> Just (VU.map realToFrac f) - SFalse -> Nothing + Just (UnboxedColumn bm (f' :: VU.Vector a)) -> + let f = dropNulls bm f' + in case testEquality (typeRep @a) (typeRep @Double) of + Just Refl -> Just f + Nothing -> case sIntegral @a of + STrue -> Just (VU.map fromIntegral f) + SFalse -> case sFloating @a of + STrue -> Just (VU.map realToFrac f) + SFalse -> Nothing Nothing -> throw $ ColumnsNotFoundException [name] "_getColumnAsDouble" (M.keys $ columnIndices df) @@ -265,11 +273,11 @@ sum :: forall a. (Columnable a, Num a) => Expr a -> DataFrame -> a sum (Col name) df = case getColumn name df of Nothing -> throw $ ColumnsNotFoundException [name] "sum" (M.keys $ columnIndices df) - Just ((UnboxedColumn _ (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of - Just Refl -> VG.sum column + Just ((UnboxedColumn bm (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of + Just Refl -> VG.sum (dropNulls bm column) Nothing -> 0 - Just ((BoxedColumn _ (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of - Just Refl -> VG.sum column + Just ((BoxedColumn bm (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of + Just Refl -> VG.sum (dropNulls bm column) Nothing -> 0 Just (PackedText _ _) -> 0 Just (MergedColumn _ _) -> 0 -- matches the old eager These column (type never Num) @@ -277,7 +285,7 @@ sum expr df = case interpret df expr of Left e -> throw e Right (TColumn xs) -> case toVector @a @V.Vector xs of Left e -> throw e - Right xs' -> VG.sum xs' + Right xs' -> VG.sum (dropNulls (columnBitmap xs) xs') {- | /O(n)/ Impute missing values in a column using a derived scalar. diff --git a/tests/Operations/Statistics.hs b/tests/Operations/Statistics.hs index db907654..dc67124f 100644 --- a/tests/Operations/Statistics.hs +++ b/tests/Operations/Statistics.hs @@ -6,6 +6,7 @@ module Operations.Statistics where import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D +import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI import qualified DataFrame.Internal.Statistics as D @@ -254,6 +255,168 @@ correlationMissingColumn = (print $ D.correlation "x" "missingcol" correlationDf) ) +-- Nullable columns: statistics must skip null slots, not read the sentinel +-- stored there. + +nullableDf :: D.DataFrame +nullableDf = + D.fromNamedColumns [("x", DI.fromList [Just (10 :: Double), Nothing, Just 20])] + +nullableIntDf :: D.DataFrame +nullableIntDf = + D.fromNamedColumns [("n", DI.fromList [Just (10 :: Int), Nothing, Just 20])] + +nullableBoxedDf :: D.DataFrame +nullableBoxedDf = + D.fromNamedColumns [("b", DI.fromList [Just (10 :: Integer), Nothing, Just 20])] + +meanIgnoresNulls :: Test +meanIgnoresNulls = + TestCase + (assertEqual "mean skips nulls" 15.0 (D.mean (F.col @Double "x") nullableDf)) + +meanExprIgnoresNulls :: Test +meanExprIgnoresNulls = + TestCase + ( assertEqual + "mean over a derived nullable expression skips nulls" + 30.0 + (D.mean (F.lift (* 2) (F.col @Double "x")) nullableDf) + ) + +medianIgnoresNulls :: Test +medianIgnoresNulls = + TestCase + (assertEqual "median skips nulls" 15.0 (D.median (F.col @Double "x") nullableDf)) + +percentileIgnoresNulls :: Test +percentileIgnoresNulls = + TestCase + ( assertEqual + "percentile skips nulls" + 15.0 + (D.percentile 50 (F.col @Double "x") nullableDf) + ) + +stdDevIgnoresNulls :: Test +stdDevIgnoresNulls = + TestCase + ( assertBool + "standard deviation skips nulls" + ( abs (D.standardDeviation (F.col @Double "x") nullableDf - 7.0710678118654755) + < 1e-12 + ) + ) + +varianceIgnoresNulls :: Test +varianceIgnoresNulls = + TestCase + ( assertEqual + "variance skips nulls" + 50.0 + (D.variance (F.col @Double "x") nullableDf) + ) + +varianceExprIgnoresNulls :: Test +varianceExprIgnoresNulls = + TestCase + ( assertEqual + "variance over a derived nullable expression skips nulls" + 200.0 + (D.variance (F.lift (* 2) (F.col @Double "x")) nullableDf) + ) + +iqrIgnoresNulls :: Test +iqrIgnoresNulls = + TestCase + ( assertEqual + "inter-quartile range skips nulls" + 5.0 + (D.interQuartileRange (F.col @Double "x") nullableDf) + ) + +skewnessIgnoresNulls :: Test +skewnessIgnoresNulls = + TestCase + ( let skewDf = + D.fromNamedColumns + [("s", DI.fromList [Just (10 :: Double), Nothing, Just 20, Just 100, Just 11])] + in assertBool + "skewness skips nulls" + ( abs + ( D.skewness (F.col @Double "s") skewDf + - D.skewness' (VU.fromList [10 :: Double, 20, 100, 11]) + ) + < 1e-12 + ) + ) + +genericPercentileIgnoresNulls :: Test +genericPercentileIgnoresNulls = + TestCase + ( assertEqual + "genericPercentile skips the sentinel" + 10 + (D.genericPercentile 10 (F.col @Int "n") nullableIntDf) + ) + +genericPercentileBoxedNullableDoesNotThrow :: Test +genericPercentileBoxedNullableDoesNotThrow = + TestCase + ( assertEqual + "genericPercentile on a boxed nullable column skips the error thunk" + 20 + (D.genericPercentile 100 (F.col @Integer "b") nullableBoxedDf) + ) + +genericPercentileMaybeViewKeepsNothing :: Test +genericPercentileMaybeViewKeepsNothing = + TestCase + ( assertEqual + "a Maybe-typed view still sees its Nothings" + (Nothing :: Maybe Int) + (D.genericPercentile 0 (F.col @(Maybe Int) "n") nullableIntDf) + ) + +sumUnboxedNullable :: Test +sumUnboxedNullable = + TestCase + (assertEqual "sum skips null slots" 30.0 (D.sum (F.col @Double "x") nullableDf)) + +sumBoxedNullableDoesNotThrow :: Test +sumBoxedNullableDoesNotThrow = + TestCase + ( assertEqual + "sum on a boxed nullable column skips the error thunk" + (30 :: Integer) + (D.sum (F.col @Integer "b") nullableBoxedDf) + ) + +correlationIgnoresNullRows :: Test +correlationIgnoresNullRows = + TestCase + ( let dfc = + D.fromNamedColumns + [ ("a", DI.fromList [Just (1 :: Double), Nothing, Just 3]) + , ("c", DI.fromList [1 :: Double, 2, 3]) + ] + in case D.correlation "a" "c" dfc of + Nothing -> assertFailure "Expected Just 1.0, got Nothing" + Just r -> + assertBool + "null rows are dropped pairwise" + (abs (r - 1.0) < 1e-10) + ) + +frequenciesSkipsNulls :: Test +frequenciesSkipsNulls = + TestCase + ( assertEqual + "frequencies has no sentinel category" + 3 -- Statistic, 10 and 20 + (D.nColumns (D.frequencies (F.col @Int "n") nullableIntDf)) + ) + tests :: [Test] tests = [ TestLabel "medianOfOddLengthDataSet" medianOfOddLengthDataSet @@ -281,4 +444,24 @@ tests = , TestLabel "correlationPerfectNegative" correlationPerfectNegative , TestLabel "correlationSelfIdentity" correlationSelfIdentity , TestLabel "correlationMissingColumn" correlationMissingColumn + , TestLabel "meanIgnoresNulls" meanIgnoresNulls + , TestLabel "meanExprIgnoresNulls" meanExprIgnoresNulls + , TestLabel "medianIgnoresNulls" medianIgnoresNulls + , TestLabel "percentileIgnoresNulls" percentileIgnoresNulls + , TestLabel "stdDevIgnoresNulls" stdDevIgnoresNulls + , TestLabel "varianceIgnoresNulls" varianceIgnoresNulls + , TestLabel "varianceExprIgnoresNulls" varianceExprIgnoresNulls + , TestLabel "iqrIgnoresNulls" iqrIgnoresNulls + , TestLabel "skewnessIgnoresNulls" skewnessIgnoresNulls + , TestLabel "genericPercentileIgnoresNulls" genericPercentileIgnoresNulls + , TestLabel + "genericPercentileBoxedNullableDoesNotThrow" + genericPercentileBoxedNullableDoesNotThrow + , TestLabel + "genericPercentileMaybeViewKeepsNothing" + genericPercentileMaybeViewKeepsNothing + , TestLabel "sumUnboxedNullable" sumUnboxedNullable + , TestLabel "sumBoxedNullableDoesNotThrow" sumBoxedNullableDoesNotThrow + , TestLabel "correlationIgnoresNullRows" correlationIgnoresNullRows + , TestLabel "frequenciesSkipsNulls" frequenciesSkipsNulls ]