DataFrame

Why this library?

Install

cabal update
cabal install dataframe

build-depends: base >= 4, dataframe

$ dataframe
dataframe> df = D.fromColumns ["product" =: [1, 1, 2, 2, 3, 3 :: Int], "amount" =: [100, 120, 50, 20, 40, 30 :: Int]]
dataframe> df |> D.groupBy ["product"] |> ["total" .= F.countAll ]

Quick Start

-- cabal: build-depends: dataframe, text
-- cabal: default-extensions: OverloadedStrings, TypeApplications, TemplateHaskell, DataKinds, TypeFamilies, FlexibleInstances, FlexibleContexts, ScopedTypeVariables, DeriveGeneric, UndecidableInstances
import qualified DataFrame as D
import qualified DataFrame.Functions as F
import DataFrame.Expression.Operators

import DataFrame ((=:), as)

sales = D.fromColumns
    [ "product" =: [1, 1, 2, 2, 3, 3 :: Int]
    , "amount"  =: [100, 120, 50, 20, 40, 30 :: Int]
    ]

-- Group by product and compute totals
sales
    |> D.groupBy ["product"]
    |> D.aggregate [ F.sum (F.int "amount") `as` "total"
                   , F.count (F.int "amount") `as` "orders"
                   ]
    |> D.toMarkdown'

fileDf <- D.readCsv "./data/housing.csv"
fileDf <- D.readParquet "./data/mtcars.parquet"

-- Hugging Face datasets (needs network access, via the dataframe-huggingface package):
-- import qualified DataFrame.IO.HuggingFace as HF
-- fileDf <- HF.readParquet "hf://datasets/scikit-learn/iris/default/train/0000.parquet"

D.dimensions fileDf

Interactive REPL

df <- D.readCsv "./data/housing.csv"
D.dimensions df

D.describeColumns df |> D.toMarkdown'

$(D.declareColumns df)

df |> D.groupBy ["ocean_proximity"]
   |> D.aggregate [F.mean median_house_value `as` "avg_value"]
   |> D.toMarkdown'

df |> D.derive "rooms_per_household" (total_rooms / households) |> D.take 3 |> D.toMarkdown'

dataframe> df |> D.derive "nonsense" (latitude + ocean_proximity)

<interactive>:14:47: error: [GHC-83865]
    • Couldn't match type 'Text' with 'Double'
        Expected: Expr Double
          Actual: Expr Text
    • In the second argument of '(+)', namely 'ocean_proximity'
      In the second argument of 'derive', namely
        '(latitude + ocean_proximity)'

Generate column references from a CSV

-- Reads housing.csv at compile time and generates:
--   latitude :: Expr Double
--   total_rooms :: Expr Double
--   ocean_proximity :: Expr Text
--   ... one binding per column
$(D.declareColumnsFromCsvFile "./data/housing.csv")

df <- D.readCsv "./data/housing.csv"
df |> D.derive "rooms_per_household" (total_rooms / households)
   |> D.filter (median_income .>. 5)
   |> D.groupBy ["ocean_proximity"]
   |> D.aggregate [F.mean median_house_value `as` "avg_value"]
   |> D.toMarkdown'

-- Without TH — every column needs its name and type spelled out
df |> D.derive "rooms_per_household"
        (F.double "total_rooms" / F.double "households")
   |> D.filter (F.double "median_income" .>. F.lit 5)
   |> D.take 5
   |> D.toMarkdown'

Generate a schema type from a CSV

import qualified DataFrame.Typed as DT

-- Generates:
-- type HousingSchema = '[ '("longitude", Double)
--                       , '("latitude", Double)
--                       , '("total_rooms", Double)
--                       , ...
--                       ]
$(DT.deriveSchemaFromCsvFile "HousingSchema" "./data/housing.csv")

Generate a schema (and a row bridge) from a record ADT

import Data.Text (Text)
import Data.Int (Int64)

data Order = Order
    { orderId :: Int64
    , region  :: Text
    , amount  :: Double
    } deriving (Show, Eq)

$(DT.deriveSchemaFromType ''Order)
-- expands to:
--   type OrderSchema =
--     '[ '("order_id", Int64), '("region", Text), '("amount", Double)]
--   instance DT.HasSchema Order where
--     type Schema Order = OrderSchema
--     recordsToColumns   = ...
--     recordsFromFrame = ...

xs :: [Order]
xs = [Order 1 "us" 10.0, Order 2 "eu" 20.5]

-- Untyped: [Order] -> DataFrame
ordersDf :: D.DataFrame
ordersDf = D.fromRecords xs

ordersDf |> D.toMarkdown'

D.toRecords ordersDf :: Either Text [Order]

DT.thaw (DT.fromRecords xs :: DT.TypedDataFrame OrderSchema) |> D.toMarkdown'

$(D.deriveSchemaValues ''Order)
-- emits:
--   orderSchema     :: Schema
--   orderSchema     = makeSchema [("order_id", schemaType @Int64), ...]
--   orderOrderId    :: Expr Int64
--   orderOrderId    = col "order_id"
--   orderRegion     :: Expr Text
--   orderRegion     = col "region"
--   orderAmount     :: Expr Double
--   orderAmount     = col "amount"

orders :: IO D.DataFrame
orders = do
    raw <- D.readCsvWithSchema orderSchema "./data/orders.csv"
    pure (D.filterBy (> 100) orderAmount raw)

import GHC.Generics (Generic)
import DataFrame.Typed (Schema)

data OrderG = OrderG
    { orderGId :: Int64
    , regionG  :: Text
    , amountG  :: Double
    } deriving (Generic)

type OrderGSchema = DT.SchemaOf OrderG

instance DT.HasSchema OrderG where
    type Schema OrderG = OrderGSchema
    recordsToColumns   = DT.genericToColumns
    recordsFromFrame = DT.genericFromColumns

Typed API

type EmployeeSchema =
    '[ '("name", Text)
     , '("department", Text)
     , '("salary", Double)
     ]

employees <- D.readCsv "./data/employees.csv"
case DT.freeze @EmployeeSchema employees of
    Nothing  -> "Schema mismatch!"
    Just tdf -> tdf
        |> DT.derive @"bonus" (DT.col @"salary" * DT.lit 0.1)
        |> DT.filter (DT.col @"salary" DT..>. DT.lit 50000)
        |> DT.select @'["name", "bonus"]
        |> DT.thaw
        |> D.toMarkdown'

-- Typo in column name -> compile error
tdf |> DT.filter (DT.col @"slary" DT..>. DT.lit 50000)
-- error: Column 'slary' not found in schema

-- Wrong type -> compile error
tdf |> DT.filter (DT.col @"name" DT..>. DT.lit 50000)
-- error: Couldn't match type 'Text' with 'Double'

type ScoreSchema = '[ '("name", Text), '("score", Maybe Double)]

scoresDf = D.fromColumns
    [ "name"  =: ["a", "b", "c" :: Text]
    , "score" =: [Just 1.0, Nothing, Just 3.0 :: Maybe Double]
    ]

Just stdf = DT.freeze @ScoreSchema scoresDf

-- filterAllJust drops the null row and changes the column type from
-- (Maybe Double) to Double, so `scaled` can multiply it directly.
DT.thaw (DT.filterAllJust stdf |> DT.derive @"scaled" (DT.col @"score" * DT.lit 100)) |> D.toMarkdown'

Lazy Queries

import qualified DataFrame.Lazy as L
import DataFrame.Schema (schemaType, makeSchema)

housingSchema = makeSchema
    [ ("longitude",          schemaType @Double)
    , ("latitude",           schemaType @Double)
    , ("housing_median_age", schemaType @Double)
    , ("total_rooms",        schemaType @Double)
    , ("total_bedrooms",     schemaType @(Maybe Double))
    , ("population",         schemaType @Double)
    , ("households",         schemaType @Double)
    , ("median_income",      schemaType @Double)
    , ("median_house_value", schemaType @Double)
    , ("ocean_proximity",    schemaType @Text)
    ]

lazyResult <- L.run $
    L.scanCsv housingSchema "./data/housing.csv"
    |> L.filter  (F.double "median_income" .>. F.lit 5)
    |> L.derive  "value_per_income"
                 (F.double "median_house_value" / F.double "median_income")
    |> L.select  ["ocean_proximity", "median_house_value", "value_per_income"]
    |> L.take 1000

D.take 10 lazyResult |> D.toMarkdown'

Switching between APIs

Documentation