GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — Avoid error thrown when no spec storage for warehouse/serving is regi… · feast-dev/feast@7032b50

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • ingestion/src
    • main/java/feast/ingestion/transform
      • SplitOutputByStore.java
    • test/java/feast/ingestion/transform
      • SplitOutputByStoreTest.java
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java‎

Copy file name to clipboardExpand all lines: ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java
-1Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ public class SplitOutputByStore extends PTransform<PFeatureRows, PFeatureRows> {
5454
public PFeatureRows expand(PFeatureRows input) {
5555
Map<String, Write> transforms = getFeatureStoreTransforms();
5656
Set<String> keys = transforms.keySet();
57-
Preconditions.checkArgument(transforms.size() > 0, "no write transforms found");
5857

5958
log.info(String.format("Splitting on keys = [%s]", String.join(",", keys)));
6059
MultiOutputSplit<String> splitter = new MultiOutputSplit<>(selector, keys, specs);
Collapse file

‎ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java‎

Copy file name to clipboardExpand all lines: ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java
+121Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import feast.storage.MockTransforms;
2626
import feast.types.FeatureRowExtendedProto.FeatureRowExtended;
2727
import feast.types.FeatureRowProto.FeatureRow;
28+
import java.util.Collections;
2829
import java.util.List;
2930
import java.util.Map;
3031
import org.apache.beam.sdk.testing.PAssert;
@@ -240,6 +241,126 @@ public void testSplitWhereFeature2HasNoStoreId() {
240241
}
241242

242243

244+
@Test
245+
public void testSplitWhereNoStorageSpec() {
246+
// There is no storage spec registered and feature 1 has no storage set
247+
// feature1 should get thrown harmlessly
248+
249+
SerializableFunction<FeatureSpec, String> selector = (fs) -> fs.getDataStores().getServing()
250+
.getId();
251+
MockSpecService specService = new MockSpecService();
252+
specService.entitySpecs.put("e1", EntitySpec.getDefaultInstance());
253+
specService.featureSpecs.put(
254+
"f1", FeatureSpec.newBuilder().setEntity("e1")
255+
.build());
256+
257+
Specs specs =
258+
Specs.of(
259+
"jobname",
260+
ImportSpec.newBuilder()
261+
.addEntities("e1")
262+
.setSchema(
263+
Schema.newBuilder()
264+
.addAllFields(
265+
Collections.singletonList(
266+
Field.newBuilder().setFeatureId("f1").build())))
267+
.build(),
268+
specService);
269+
assertNull(specs.getError());
270+
List<FeatureStore> stores = Collections.emptyList();
271+
SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs);
272+
273+
PCollection<FeatureRowExtended> input =
274+
pipeline
275+
.apply(
276+
Create.of(
277+
FeatureRow.newBuilder()
278+
.addFeatures(Features.of("f1", Values.ofInt32(1)))
279+
.build()))
280+
.apply(new ToFeatureRowExtended());
281+
PFeatureRows pfrows = PFeatureRows.of(input);
282+
pfrows = pfrows.apply("do split", split);
283+
284+
PAssert.that(
285+
pfrows
286+
.getErrors()).empty();
287+
PAssert.that(
288+
pfrows
289+
.getMain()
290+
.apply(
291+
MapElements.into(TypeDescriptor.of(FeatureRow.class))
292+
.via(FeatureRowExtended::getRow)))
293+
.containsInAnyOrder(
294+
FeatureRow.newBuilder()
295+
.addFeatures(Features.of("f1", Values.ofInt32(1)))
296+
.setEventTimestamp(Timestamp.getDefaultInstance())
297+
.build());
298+
299+
pipeline.run();
300+
}
301+
302+
303+
@Test
304+
public void testSplitWhereNoStorageSpecForAFeature() {
305+
// There is no storage spec registered but feature 1 has storage set
306+
// feature1 should get thrown harmlessly
307+
308+
SerializableFunction<FeatureSpec, String> selector = (fs) -> fs.getDataStores().getServing()
309+
.getId();
310+
MockSpecService specService = new MockSpecService();
311+
specService.entitySpecs.put("e1", EntitySpec.getDefaultInstance());
312+
specService.featureSpecs.put(
313+
"f1", FeatureSpec.newBuilder().setEntity("e1")
314+
.setDataStores(
315+
DataStores.newBuilder().setServing(DataStore.newBuilder().setId("store1")
316+
.build()))
317+
.build());
318+
319+
Specs specs =
320+
Specs.of(
321+
"jobname",
322+
ImportSpec.newBuilder()
323+
.addEntities("e1")
324+
.setSchema(
325+
Schema.newBuilder()
326+
.addAllFields(
327+
Collections.singletonList(
328+
Field.newBuilder().setFeatureId("f1").build())))
329+
.build(),
330+
specService);
331+
assertNull(specs.getError());
332+
List<FeatureStore> stores = Collections.emptyList();
333+
SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs);
334+
335+
PCollection<FeatureRowExtended> input =
336+
pipeline
337+
.apply(
338+
Create.of(
339+
FeatureRow.newBuilder()
340+
.addFeatures(Features.of("f1", Values.ofInt32(1)))
341+
.build()))
342+
.apply(new ToFeatureRowExtended());
343+
PFeatureRows pfrows = PFeatureRows.of(input);
344+
pfrows = pfrows.apply("do split", split);
345+
346+
PAssert.that(
347+
pfrows
348+
.getErrors()).empty();
349+
PAssert.that(
350+
pfrows
351+
.getMain()
352+
.apply(
353+
MapElements.into(TypeDescriptor.of(FeatureRow.class))
354+
.via(FeatureRowExtended::getRow)))
355+
.containsInAnyOrder(
356+
FeatureRow.newBuilder()
357+
.addFeatures(Features.of("f1", Values.ofInt32(1)))
358+
.setEventTimestamp(Timestamp.getDefaultInstance())
359+
.build());
360+
361+
pipeline.run();
362+
}
363+
243364
@Test
244365
public void testWriteTags() {
245366
TupleTag<FeatureRowExtended> tag1 = new TupleTag<>("TAG1");

0 commit comments

Comments
0 (0)
You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.