GeezerVison is showing github.com — feat: Batch_engine config injection in feature_store.yaml through operator by aniketpalu · Pull Request #5938 · feast-dev/feast
feat: Batch_engine config injection in feature_store.yaml through operator
#5938
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement . We’ll occasionally send you account related emails.
Already on GitHub?
Sign in
to your account
Uh oh!
There was an error while loading. Please reload this page.
Show file tree
Hide file tree
Changes from all commits
Commits
File filter
Filter by extension
.go
(9)
.md
(1)
.yaml
(4)
All 3 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Hide whitespace
Diff view
Hide whitespace
There are no files selected for viewing
Original file line number
Diff line number
Diff line change
Expand Up
@@ -652,7 +652,7 @@ func getQualifiedFeatureName(viewName string, featureName string, fullFeatureNam
func validateJoinKeys(
joinKeyValues map[string]*prototypes.RepeatedValue,
expectedJoinKeysSet map[string]interface{}) error {
for joinKey, _ := range joinKeyValues {
for joinKey := range joinKeyValues {
if _, ok := expectedJoinKeysSet[joinKey]; !ok {
return fmt.Errorf("Invalid entity join key. key=%s", joinKey)
}
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -9,7 +9,6 @@ import (
prototypes "github.com/feast-dev/feast/go/protos/feast/types"
"github.com/feast-dev/feast/go/types"
"github.com/google/uuid"
)
const feastServerVersion = "0.0.1"
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -4,8 +4,8 @@ import (
"os"
"github.com/rs/zerolog"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
var tracer = otel.Tracer("github.com/feast-dev/feast/go/server")
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -75,6 +75,7 @@ type FeatureStoreSpec struct {
Services *FeatureStoreServices `json:"services,omitempty"`
AuthzConfig *AuthzConfig `json:"authz,omitempty"`
CronJob *FeastCronJob `json:"cronJob,omitempty"`
BatchEngine *BatchEngineConfig `json:"batchEngine,omitempty"`
}
// FeastProjectDir defines how to create the feast project directory.
Expand Down
Expand Up
@@ -155,6 +156,15 @@ type FeastCronJob struct {
FailedJobsHistoryLimit *int32 `json:"failedJobsHistoryLimit,omitempty"`
}
// BatchEngineConfig defines the batch compute engine configuration.
type BatchEngineConfig struct {
// Reference to a ConfigMap containing the batch engine configuration.
// The ConfigMap should contain YAML-formatted config with 'type' and engine-specific fields.
ConfigMapRef *corev1.LocalObjectReference `json:"configMapRef,omitempty"`
// Key name in the ConfigMap. Defaults to "config" if not specified.
ConfigMapKey string `json:"configMapKey,omitempty"`
}
// JobSpec describes how the job execution will look like.
type JobSpec struct {
// PodTemplateAnnotations are annotations to be applied to the CronJob's PodTemplate
Expand Down
Oops, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
Original file line number
Diff line number
Diff line change
Expand Up
@@ -50,7 +50,7 @@ metadata:
}
]
capabilities: Basic Install
createdAt: "2026-01-31T05:08:59Z "
createdAt: "2026-02-03T08:12:50Z "
operators.operatorframework.io/builder: operator-sdk-v1.38.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v4
name: feast-operator.v0.59.0
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -83,6 +83,27 @@ spec:
x-kubernetes-validations:
- message: One selection required between kubernetes or oidc.
rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, c)'
batchEngine:
description: BatchEngineConfig defines the batch compute engine configuration.
properties:
configMapKey:
description: Key name in the ConfigMap. Defaults to "config" if
not specified.
type: string
configMapRef:
description: Reference to a ConfigMap containing the batch engine
configuration.
properties:
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty.
type: string
type: object
x-kubernetes-map-type: atomic
type: object
cronJob:
description: FeastCronJob defines a CronJob to execute against a Feature
Store deployment.
Expand Down
Expand Up
@@ -4287,6 +4308,28 @@ spec:
- message: One selection required between kubernetes or oidc.
rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c,
c)'
batchEngine:
description: BatchEngineConfig defines the batch compute engine
configuration.
properties:
configMapKey:
description: Key name in the ConfigMap. Defaults to "config"
if not specified.
type: string
configMapRef:
description: Reference to a ConfigMap containing the batch
engine configuration.
properties:
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty.
type: string
type: object
x-kubernetes-map-type: atomic
type: object
cronJob:
description: FeastCronJob defines a CronJob to execute against
a Feature Store deployment.
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -83,6 +83,27 @@ spec:
x-kubernetes-validations:
- message: One selection required between kubernetes or oidc.
rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, c)'
batchEngine:
description: BatchEngineConfig defines the batch compute engine configuration.
properties:
configMapKey:
description: Key name in the ConfigMap. Defaults to "config" if
not specified.
type: string
configMapRef:
description: Reference to a ConfigMap containing the batch engine
configuration.
properties:
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty.
type: string
type: object
x-kubernetes-map-type: atomic
type: object
cronJob:
description: FeastCronJob defines a CronJob to execute against a Feature
Store deployment.
Expand Down
Expand Up
@@ -4287,6 +4308,28 @@ spec:
- message: One selection required between kubernetes or oidc.
rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c,
c)'
batchEngine:
description: BatchEngineConfig defines the batch compute engine
configuration.
properties:
configMapKey:
description: Key name in the ConfigMap. Defaults to "config"
if not specified.
type: string
configMapRef:
description: Reference to a ConfigMap containing the batch
engine configuration.
properties:
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty.
type: string
type: object
x-kubernetes-map-type: atomic
type: object
cronJob:
description: FeastCronJob defines a CronJob to execute against
a Feature Store deployment.
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -91,6 +91,27 @@ spec:
x-kubernetes-validations:
- message: One selection required between kubernetes or oidc.
rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, c)'
batchEngine:
description: BatchEngineConfig defines the batch compute engine configuration.
properties:
configMapKey:
description: Key name in the ConfigMap. Defaults to "config" if
not specified.
type: string
configMapRef:
description: Reference to a ConfigMap containing the batch engine
configuration.
properties:
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty.
type: string
type: object
x-kubernetes-map-type: atomic
type: object
cronJob:
description: FeastCronJob defines a CronJob to execute against a Feature
Store deployment.
Expand Down
Expand Up
@@ -4295,6 +4316,28 @@ spec:
- message: One selection required between kubernetes or oidc.
rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c,
c)'
batchEngine:
description: BatchEngineConfig defines the batch compute engine
configuration.
properties:
configMapKey:
description: Key name in the ConfigMap. Defaults to "config"
if not specified.
type: string
configMapRef:
description: Reference to a ConfigMap containing the batch
engine configuration.
properties:
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty.
type: string
type: object
x-kubernetes-map-type: atomic
type: object
cronJob:
description: FeastCronJob defines a CronJob to execute against
a Feature Store deployment.
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -28,6 +28,22 @@ _Appears in:_
| `oidc` _[OidcAuthz](#oidcauthz)_ | |
#### BatchEngineConfig
BatchEngineConfig defines the batch compute engine configuration.
_Appears in:_
- [FeatureStoreSpec](#featurestorespec)
| Field | Description |
| --- | --- |
| `configMapRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | Reference to a ConfigMap containing the batch engine configuration.
The ConfigMap should contain YAML-formatted config with 'type' and engine-specific fields. |
| `configMapKey` _string_ | Key name in the ConfigMap. Defaults to "config" if not specified. |
#### ContainerConfigs
Expand Down
Expand Up
@@ -226,6 +242,7 @@ _Appears in:_
| `services` _[FeatureStoreServices](#featurestoreservices)_ | |
| `authz` _[AuthzConfig](#authzconfig)_ | |
| `cronJob` _[FeastCronJob](#feastcronjob)_ | |
| `batchEngine` _[BatchEngineConfig](#batchengineconfig)_ | |
#### FeatureStoreStatus
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -44,12 +44,13 @@ func (feast *FeastServices) getServiceFeatureStoreYaml() ([]byte, error) {
}
func (feast *FeastServices) getServiceRepoConfig() (RepoConfig, error) {
return getServiceRepoConfig(feast.Handler.FeatureStore, feast.extractConfigFromSecret)
return getServiceRepoConfig(feast.Handler.FeatureStore, feast.extractConfigFromSecret, feast.extractConfigFromConfigMap )
}
func getServiceRepoConfig(
featureStore *feastdevv1.FeatureStore,
secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) {
secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error),
configMapExtractionFunc func(configMapRef string, configMapKey string) (map[string]interface{}, error)) (RepoConfig, error) {
repoConfig, err := getBaseServiceRepoConfig(featureStore, secretExtractionFunc)
if err != nil {
return repoConfig, err
Expand Down
Expand Up
@@ -78,6 +79,13 @@ func getServiceRepoConfig(
}
}
if appliedSpec.BatchEngine != nil {
err := setRepoConfigBatchEngine(appliedSpec.BatchEngine, configMapExtractionFunc, &repoConfig)
if err != nil {
return repoConfig, err
}
}
return repoConfig, nil
}
Expand Down
Expand Up
@@ -248,6 +256,34 @@ func setRepoConfigOffline(services *feastdevv1.FeatureStoreServices, secretExtra
return nil
}
func setRepoConfigBatchEngine(
batchEngineConfig *feastdevv1.BatchEngineConfig,
configMapExtractionFunc func(configMapRef string, configMapKey string) (map[string]interface{}, error),
repoConfig *RepoConfig) error {
if batchEngineConfig.ConfigMapRef == nil {
return nil
}
configMapKey := batchEngineConfig.ConfigMapKey
if configMapKey == "" {
configMapKey = "config"
}
config, err := configMapExtractionFunc(batchEngineConfig.ConfigMapRef.Name, configMapKey)
if err != nil {
return err
}
// Extract type from config
engineType, ok := config["type"].(string)
if !ok {
return fmt.Errorf("batch engine config must contain 'type' field")
}
delete(config, "type")
repoConfig.BatchEngine = &ComputeEngineConfig{
Type: engineType,
Parameters: config,
}
return nil
}
func (feast *FeastServices) getClientFeatureStoreYaml(secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) ([]byte, error) {
clientRepo, err := getClientRepoConfig(feast.Handler.FeatureStore, secretExtractionFunc, feast)
if err != nil {
Expand Down
Expand Up
@@ -388,6 +424,26 @@ func (feast *FeastServices) extractConfigFromSecret(storeType string, secretRef
return parameters, nil
}
func (feast *FeastServices) extractConfigFromConfigMap(configMapRef string, configMapKey string) (map[string]interface{}, error) {
configMap, err := feast.getConfigMap(configMapRef)
if err != nil {
return nil, err
}
if configMapKey == "" {
configMapKey = "config"
}
val, exists := configMap.Data[configMapKey]
if !exists {
return nil, fmt.Errorf("configmap key %s doesn't exist in configmap %s", configMapKey, configMapRef)
}
var config map[string]interface{}
err = yaml.Unmarshal([]byte(val), &config)
if err != nil {
return nil, fmt.Errorf("configmap %s contains invalid YAML in key %s", configMapRef, configMapKey)
}
return config, nil
}
func mergeStructWithDBParametersMap(parametersMap *map[string]interface{}, s interface{}) error {
for key, val := range *parametersMap {
hasAttribute, err := hasAttrib(s, key, val)
Expand Down
Loading
Oops, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
Loading
Oops, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.