diff --git a/dataframe-core/src-internal/DataFrame/Internal/Column.hs b/dataframe-core/src-internal/DataFrame/Internal/Column.hs index 1395b9b4..6f76f1df 100644 --- a/dataframe-core/src-internal/DataFrame/Internal/Column.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Column.hs @@ -39,7 +39,7 @@ import Data.Bits ( ) import Data.Kind (Type) import Data.Maybe -import Data.Type.Equality (TestEquality (..)) +import Data.Type.Equality (TestEquality (..), type (:~~:) (HRefl)) import Data.Word (Word8) import DataFrame.Errors import DataFrame.Internal.PackedText ( @@ -209,6 +209,22 @@ columnBitmap (UnboxedColumn bm _) = bm columnBitmap (PackedText bm _) = bm columnBitmap (MergedColumn _ _) = Nothing +{- | Drop the null slots of a payload-typed view of a nullable column: those +slots hold a sentinel, not a value. A @Maybe@-typed view already encodes the +nulls, so it is returned untouched. Backpermute never forces the kept-out +slots, so boxed error thunks at null slots are safe. +-} +dropNulls :: + forall v a. + (Typeable a, VG.Vector v a, VG.Vector v Int) => Maybe Bitmap -> v a -> v a +dropNulls Nothing xs = xs +dropNulls (Just bm) xs = case typeRep @a of + App m _ | Just HRefl <- eqTypeRep m (typeRep @Maybe) -> xs + _ -> VG.backpermute xs keep + where + keep = VG.fromList [i | i <- [0 .. VG.length xs - 1], bitmapTestBit bm i] +{-# INLINE dropNulls #-} + {- | Decode a 'PackedText' into a @BoxedColumn Text@ (bit-identical to materializing at freeze). Identity on every other column. -} diff --git a/dataframe-operations/src/DataFrame/Operations/Core.hs b/dataframe-operations/src/DataFrame/Operations/Core.hs index 77df934f..ef7e66c9 100644 --- a/dataframe-operations/src/DataFrame/Operations/Core.hs +++ b/dataframe-operations/src/DataFrame/Operations/Core.hs @@ -67,8 +67,10 @@ import DataFrame.Internal.Column ( Column (..), Columnable, TypedColumn (..), + columnBitmap, columnLength, columnTypeString, + dropNulls, fromList, fromVector, materializeMerged, @@ -688,7 +690,7 @@ valueCounts :: forall a. (Ord a, Columnable a) => Expr a -> DataFrame -> [(a, Int)] valueCounts expr df | null df = throw (EmptyDataSetException "valueCounts") - | otherwise = case columnAsVector expr df of + | otherwise = case columnAsVectorNonNull expr df of Left e -> throw e Right column' -> let @@ -696,6 +698,20 @@ valueCounts expr df in M.toAscList column +-- | As 'columnAsVector', minus null slots: a sentinel is not a category. +columnAsVectorNonNull :: + forall a. + (Columnable a) => Expr a -> DataFrame -> Either DataFrameException (V.Vector a) +columnAsVectorNonNull expr df = case expr of + Col name -> case getColumn name df of + Just col -> withColumnName name (dropNulls (columnBitmap col) <$> toVector col) + Nothing -> + Left $ + ColumnsNotFoundException [name] "valueCounts" (M.keys $ columnIndices df) + _ -> case interpret df expr of + Left e -> throw e + Right (TColumn col) -> dropNulls (columnBitmap col) <$> toVector col + {- | O (k * n) Shows the proportions of each value in a given column. ==== __Example__ @@ -712,7 +728,7 @@ valueProportions :: forall a. (Ord a, Columnable a) => Expr a -> DataFrame -> [(a, Double)] valueProportions expr df | null df = throw (EmptyDataSetException "valueCounts") - | otherwise = case columnAsVector expr df of + | otherwise = case columnAsVectorNonNull expr df of Left e -> throw e Right column' -> let diff --git a/dataframe-operations/src/DataFrame/Operations/Statistics.hs b/dataframe-operations/src/DataFrame/Operations/Statistics.hs index 7322c0dc..c67d6d96 100644 --- a/dataframe-operations/src/DataFrame/Operations/Statistics.hs +++ b/dataframe-operations/src/DataFrame/Operations/Statistics.hs @@ -120,7 +120,7 @@ mean expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> mean' xs + Right xs -> mean' (dropNulls (columnBitmap col) xs) meanMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double @@ -137,13 +137,13 @@ meanMaybe expr df = case interpret @(Maybe a) df expr of median :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double median (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> median' xs + Right xs -> median' (dropNulls (colBitmap name df) xs) Left e -> throw e median expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> median' xs + Right xs -> median' (dropNulls (columnBitmap col) xs) -- | Calculates the median of a given column (containing optional values) as a standalone value. medianMaybe :: @@ -162,50 +162,50 @@ percentile :: forall a. (Columnable a, Real a, VU.Unbox a) => Int -> Expr a -> DataFrame -> Double percentile n (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> percentile' n xs + Right xs -> percentile' n (dropNulls (colBitmap name df) xs) Left e -> throw e percentile n expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> percentile' n xs + Right xs -> percentile' n (dropNulls (columnBitmap col) xs) -- | Calculates the nth percentile of a given column as a standalone value. genericPercentile :: forall a. (Columnable a, Ord a) => Int -> Expr a -> DataFrame -> a genericPercentile n (Col name) df = case columnAsVector (Col @a name) df of - Right xs -> percentileOrd' n xs + Right xs -> percentileOrd' n (dropNulls (colBitmap name df) xs) Left e -> throw e genericPercentile n expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toVector @a col of Left e -> throw e - Right xs -> percentileOrd' n xs + Right xs -> percentileOrd' n (dropNulls (columnBitmap col) xs) -- | Calculates the standard deviation of a given column as a standalone value. standardDeviation :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double standardDeviation (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> (sqrt . variance') xs + Right xs -> (sqrt . variance') (dropNulls (colBitmap name df) xs) Left e -> throw e standardDeviation expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> (sqrt . variance') xs + Right xs -> (sqrt . variance') (dropNulls (columnBitmap col) xs) -- | Calculates the skewness of a given column as a standalone value. skewness :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double skewness (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> skewness' xs + Right xs -> skewness' (dropNulls (colBitmap name df) xs) Left e -> throw e skewness expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> skewness' xs + Right xs -> skewness' (dropNulls (columnBitmap col) xs) -- | Calculates the variance of a given column as a standalone value. variance :: @@ -217,36 +217,44 @@ variance expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> variance' xs + Right xs -> variance' (dropNulls (columnBitmap col) xs) -- | Calculates the inter-quartile range of a given column as a standalone value. interQuartileRange :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double interQuartileRange (Col name) df = case columnAsUnboxedVector (Col @a name) df of - Right xs -> interQuartileRange' xs + Right xs -> interQuartileRange' (dropNulls (colBitmap name df) xs) Left e -> throw e interQuartileRange expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of Left e -> throw e - Right xs -> interQuartileRange' xs + Right xs -> interQuartileRange' (dropNulls (columnBitmap col) xs) -- | Calculates the Pearson's correlation coefficient between two given columns as a standalone value. correlation :: T.Text -> T.Text -> DataFrame -> Maybe Double correlation first second df = do - f <- _getColumnAsDouble first df - s <- _getColumnAsDouble second df + -- Listwise deletion: a null in either column drops the pair. + let df' = filterJust first (filterJust second df) + f <- _getColumnAsDouble first df' + s <- _getColumnAsDouble second df' correlation' f s +-- | Bitmap of a named column, if it has one. +colBitmap :: T.Text -> DataFrame -> Maybe Bitmap +colBitmap name df = columnBitmap =<< getColumn name df + _getColumnAsDouble :: T.Text -> DataFrame -> Maybe (VU.Vector Double) _getColumnAsDouble name df = case getColumn name df of - Just (UnboxedColumn _ (f :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of - Just Refl -> Just f - Nothing -> case sIntegral @a of - STrue -> Just (VU.map fromIntegral f) - SFalse -> case sFloating @a of - STrue -> Just (VU.map realToFrac f) - SFalse -> Nothing + Just (UnboxedColumn bm (f' :: VU.Vector a)) -> + let f = dropNulls bm f' + in case testEquality (typeRep @a) (typeRep @Double) of + Just Refl -> Just f + Nothing -> case sIntegral @a of + STrue -> Just (VU.map fromIntegral f) + SFalse -> case sFloating @a of + STrue -> Just (VU.map realToFrac f) + SFalse -> Nothing Nothing -> throw $ ColumnsNotFoundException [name] "_getColumnAsDouble" (M.keys $ columnIndices df) @@ -265,11 +273,11 @@ sum :: forall a. (Columnable a, Num a) => Expr a -> DataFrame -> a sum (Col name) df = case getColumn name df of Nothing -> throw $ ColumnsNotFoundException [name] "sum" (M.keys $ columnIndices df) - Just ((UnboxedColumn _ (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of - Just Refl -> VG.sum column + Just ((UnboxedColumn bm (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of + Just Refl -> VG.sum (dropNulls bm column) Nothing -> 0 - Just ((BoxedColumn _ (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of - Just Refl -> VG.sum column + Just ((BoxedColumn bm (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of + Just Refl -> VG.sum (dropNulls bm column) Nothing -> 0 Just (PackedText _ _) -> 0 Just (MergedColumn _ _) -> 0 -- matches the old eager These column (type never Num) @@ -277,7 +285,7 @@ sum expr df = case interpret df expr of Left e -> throw e Right (TColumn xs) -> case toVector @a @V.Vector xs of Left e -> throw e - Right xs' -> VG.sum xs' + Right xs' -> VG.sum (dropNulls (columnBitmap xs) xs') {- | /O(n)/ Impute missing values in a column using a derived scalar. diff --git a/tests/Operations/Statistics.hs b/tests/Operations/Statistics.hs index db907654..dc67124f 100644 --- a/tests/Operations/Statistics.hs +++ b/tests/Operations/Statistics.hs @@ -6,6 +6,7 @@ module Operations.Statistics where import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D +import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI import qualified DataFrame.Internal.Statistics as D @@ -254,6 +255,168 @@ correlationMissingColumn = (print $ D.correlation "x" "missingcol" correlationDf) ) +-- Nullable columns: statistics must skip null slots, not read the sentinel +-- stored there. + +nullableDf :: D.DataFrame +nullableDf = + D.fromNamedColumns [("x", DI.fromList [Just (10 :: Double), Nothing, Just 20])] + +nullableIntDf :: D.DataFrame +nullableIntDf = + D.fromNamedColumns [("n", DI.fromList [Just (10 :: Int), Nothing, Just 20])] + +nullableBoxedDf :: D.DataFrame +nullableBoxedDf = + D.fromNamedColumns [("b", DI.fromList [Just (10 :: Integer), Nothing, Just 20])] + +meanIgnoresNulls :: Test +meanIgnoresNulls = + TestCase + (assertEqual "mean skips nulls" 15.0 (D.mean (F.col @Double "x") nullableDf)) + +meanExprIgnoresNulls :: Test +meanExprIgnoresNulls = + TestCase + ( assertEqual + "mean over a derived nullable expression skips nulls" + 30.0 + (D.mean (F.lift (* 2) (F.col @Double "x")) nullableDf) + ) + +medianIgnoresNulls :: Test +medianIgnoresNulls = + TestCase + (assertEqual "median skips nulls" 15.0 (D.median (F.col @Double "x") nullableDf)) + +percentileIgnoresNulls :: Test +percentileIgnoresNulls = + TestCase + ( assertEqual + "percentile skips nulls" + 15.0 + (D.percentile 50 (F.col @Double "x") nullableDf) + ) + +stdDevIgnoresNulls :: Test +stdDevIgnoresNulls = + TestCase + ( assertBool + "standard deviation skips nulls" + ( abs (D.standardDeviation (F.col @Double "x") nullableDf - 7.0710678118654755) + < 1e-12 + ) + ) + +varianceIgnoresNulls :: Test +varianceIgnoresNulls = + TestCase + ( assertEqual + "variance skips nulls" + 50.0 + (D.variance (F.col @Double "x") nullableDf) + ) + +varianceExprIgnoresNulls :: Test +varianceExprIgnoresNulls = + TestCase + ( assertEqual + "variance over a derived nullable expression skips nulls" + 200.0 + (D.variance (F.lift (* 2) (F.col @Double "x")) nullableDf) + ) + +iqrIgnoresNulls :: Test +iqrIgnoresNulls = + TestCase + ( assertEqual + "inter-quartile range skips nulls" + 5.0 + (D.interQuartileRange (F.col @Double "x") nullableDf) + ) + +skewnessIgnoresNulls :: Test +skewnessIgnoresNulls = + TestCase + ( let skewDf = + D.fromNamedColumns + [("s", DI.fromList [Just (10 :: Double), Nothing, Just 20, Just 100, Just 11])] + in assertBool + "skewness skips nulls" + ( abs + ( D.skewness (F.col @Double "s") skewDf + - D.skewness' (VU.fromList [10 :: Double, 20, 100, 11]) + ) + < 1e-12 + ) + ) + +genericPercentileIgnoresNulls :: Test +genericPercentileIgnoresNulls = + TestCase + ( assertEqual + "genericPercentile skips the sentinel" + 10 + (D.genericPercentile 10 (F.col @Int "n") nullableIntDf) + ) + +genericPercentileBoxedNullableDoesNotThrow :: Test +genericPercentileBoxedNullableDoesNotThrow = + TestCase + ( assertEqual + "genericPercentile on a boxed nullable column skips the error thunk" + 20 + (D.genericPercentile 100 (F.col @Integer "b") nullableBoxedDf) + ) + +genericPercentileMaybeViewKeepsNothing :: Test +genericPercentileMaybeViewKeepsNothing = + TestCase + ( assertEqual + "a Maybe-typed view still sees its Nothings" + (Nothing :: Maybe Int) + (D.genericPercentile 0 (F.col @(Maybe Int) "n") nullableIntDf) + ) + +sumUnboxedNullable :: Test +sumUnboxedNullable = + TestCase + (assertEqual "sum skips null slots" 30.0 (D.sum (F.col @Double "x") nullableDf)) + +sumBoxedNullableDoesNotThrow :: Test +sumBoxedNullableDoesNotThrow = + TestCase + ( assertEqual + "sum on a boxed nullable column skips the error thunk" + (30 :: Integer) + (D.sum (F.col @Integer "b") nullableBoxedDf) + ) + +correlationIgnoresNullRows :: Test +correlationIgnoresNullRows = + TestCase + ( let dfc = + D.fromNamedColumns + [ ("a", DI.fromList [Just (1 :: Double), Nothing, Just 3]) + , ("c", DI.fromList [1 :: Double, 2, 3]) + ] + in case D.correlation "a" "c" dfc of + Nothing -> assertFailure "Expected Just 1.0, got Nothing" + Just r -> + assertBool + "null rows are dropped pairwise" + (abs (r - 1.0) < 1e-10) + ) + +frequenciesSkipsNulls :: Test +frequenciesSkipsNulls = + TestCase + ( assertEqual + "frequencies has no sentinel category" + 3 -- Statistic, 10 and 20 + (D.nColumns (D.frequencies (F.col @Int "n") nullableIntDf)) + ) + tests :: [Test] tests = [ TestLabel "medianOfOddLengthDataSet" medianOfOddLengthDataSet @@ -281,4 +444,24 @@ tests = , TestLabel "correlationPerfectNegative" correlationPerfectNegative , TestLabel "correlationSelfIdentity" correlationSelfIdentity , TestLabel "correlationMissingColumn" correlationMissingColumn + , TestLabel "meanIgnoresNulls" meanIgnoresNulls + , TestLabel "meanExprIgnoresNulls" meanExprIgnoresNulls + , TestLabel "medianIgnoresNulls" medianIgnoresNulls + , TestLabel "percentileIgnoresNulls" percentileIgnoresNulls + , TestLabel "stdDevIgnoresNulls" stdDevIgnoresNulls + , TestLabel "varianceIgnoresNulls" varianceIgnoresNulls + , TestLabel "varianceExprIgnoresNulls" varianceExprIgnoresNulls + , TestLabel "iqrIgnoresNulls" iqrIgnoresNulls + , TestLabel "skewnessIgnoresNulls" skewnessIgnoresNulls + , TestLabel "genericPercentileIgnoresNulls" genericPercentileIgnoresNulls + , TestLabel + "genericPercentileBoxedNullableDoesNotThrow" + genericPercentileBoxedNullableDoesNotThrow + , TestLabel + "genericPercentileMaybeViewKeepsNothing" + genericPercentileMaybeViewKeepsNothing + , TestLabel "sumUnboxedNullable" sumUnboxedNullable + , TestLabel "sumBoxedNullableDoesNotThrow" sumBoxedNullableDoesNotThrow + , TestLabel "correlationIgnoresNullRows" correlationIgnoresNullRows + , TestLabel "frequenciesSkipsNulls" frequenciesSkipsNulls ]