|
import logging |
|
import os |
|
from datetime import datetime |
|
from typing import Generator, List |
|
|
|
import bytewax.operators as op |
|
import pyarrow as pa |
|
import pyarrow.parquet as pq |
|
from bytewax.dataflow import Dataflow |
|
from bytewax.inputs import DynamicSource, StatelessSourcePartition |
|
from bytewax.outputs import DynamicSink, StatelessSinkPartition |
|
from bytewax.testing import run_main |
|
from feast import FeatureStore, FeatureView, RepoConfig |
|
from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping |
|
from pyarrow import RecordBatch |
|
|
|
logger = logging.getLogger(__name__) |
|
DEFAULT_BATCH_SIZE = 1000 |
|
|
|
|
|
class FeastPartition(StatelessSourcePartition[RecordBatch]): |
|
def __init__( |
|
self, |
|
path: str, |
|
mini_batch_size: int, |
|
): |
|
self._path = path |
|
self._mini_batch_size = mini_batch_size |
|
|
|
def next_batch(self, sched: datetime) -> Generator[list[RecordBatch], None, None]: |
|
logger.info(f"Processing path {self._path}") |
|
dataset = pq.ParquetDataset(self._path, use_legacy_dataset=False) |
|
for fragment in dataset.fragments: |
|
for batch in fragment.to_table().to_batches( |
|
max_chunksize=self._mini_batch_size |
|
): |
|
yield [batch] |
|
|
|
|
|
class FeastSource(DynamicSource): |
|
def __init__(self, paths: list[str], job_index: int, mini_batch_size: int): |
|
self._paths = paths |
|
self._job_index = job_index |
|
self._mini_batch_size = mini_batch_size |
|
|
|
def build( |
|
self, now: datetime, _worker_index: int, _worker_count: int |
|
) -> FeastPartition: |
|
return FeastPartition(self._paths[self._job_index], self._mini_batch_size) |
|
|
|
|
|
class FeastSinkPartition(StatelessSinkPartition): |
|
def __init__(self, config, feature_store: FeatureStore, feature_view: FeatureView): |
|
self._config = config |
|
self._feature_store = feature_store |
|
self._feature_view = feature_view |
|
|
|
def write_batch(self, mini_batch: list[RecordBatch]): |
|
table: pa.Table = pa.Table.from_batches(mini_batch) |
|
|
|
if self._feature_view.batch_source.field_mapping is not None: |
|
table = _run_pyarrow_field_mapping( |
|
table, self._feature_view.batch_source.field_mapping |
|
) |
|
join_key_to_value_type = { |
|
entity.name: entity.dtype.to_value_type() |
|
for entity in self._feature_view.entity_columns |
|
} |
|
rows_to_write = _convert_arrow_to_proto( |
|
table, self._feature_view, join_key_to_value_type |
|
) |
|
self._feature_store._get_provider().online_write_batch( |
|
config=self._config, |
|
table=self._feature_view, |
|
data=rows_to_write, |
|
progress=None, |
|
) |
|
|
|
|
|
class FeastSink(DynamicSink): |
|
def __init__(self, config, feature_store: FeatureStore, feature_view: FeatureView): |
|
self._config = config |
|
self._feature_store = feature_store |
|
self._feature_view = feature_view |
|
|
|
def build(self, _worker_index, _worker_count): |
|
return FeastSinkPartition(self._config, self._feature_store, self._feature_view) |
|
|
|
|
|
class BytewaxMaterializationDataflow: |
|
def __init__( |
|
self, |
|
config: RepoConfig, |
|
feature_view: FeatureView, |
|
paths: List[str], |
|
job_index: int, |
|
): |
|
feature_store = FeatureStore(config=config) |
|
mini_batch_size = int(os.getenv("BYTEWAX_MINI_BATCH_SIZE", DEFAULT_BATCH_SIZE)) |
|
flow = Dataflow("feast_materialization") |
|
feast_input = op.input( |
|
"inp", flow, FeastSource(paths, job_index, mini_batch_size) |
|
) |
|
flow.output("out", feast_input, FeastSink(config, feature_store, feature_view)) |
|
self._run_dataflow(flow) |
|
|
|
def _run_dataflow(self, flow: Dataflow): |
|
run_main(flow) |