Skip to content

Commit

Permalink
Test map type with lopsided stats
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sunjeet
Sunjeet committed Feb 3, 2025
1 parent 6c8e157 commit 593931e
Show file tree
Hide file tree
Showing 6 changed files with 255 additions and 38 deletions.
6 changes: 2 additions & 4 deletions ...netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,15 @@

import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.read.engine.set.HollowSetTypeReadState;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
import com.netflix.hollow.core.schema.HollowListSchema;
import com.netflix.hollow.core.write.HollowListTypeWriteState;
import com.netflix.hollow.core.write.HollowListWriteRecord;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.Before;
import org.mockito.Mock;
Expand Down Expand Up @@ -105,7 +103,7 @@ protected void assertDataUnchanged(HollowListTypeReadState typeState, int[][] li
}
}
if (!actual.equals(expected)) {
System.out.println("Here");
System.out.println("// SNAP: TODO: Here");
}

assertTrue(matched);
Expand Down
51 changes: 44 additions & 7 deletions ...t/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsJoinerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,17 @@
import com.netflix.hollow.api.consumer.HollowConsumer;
import com.netflix.hollow.api.producer.HollowProducer;
import com.netflix.hollow.api.producer.fs.HollowInMemoryBlobStager;
import com.netflix.hollow.core.read.engine.PopulatedOrdinalListener;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
import com.netflix.hollow.core.write.HollowSetWriteRecord;
import com.netflix.hollow.test.InMemoryBlobStore;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.junit.Assert;
import org.junit.Test;

Expand Down Expand Up @@ -146,27 +153,27 @@ public int maxDeltasBeforeDoubleSnapshot() {
assertEquals(8, dataElements1.totalNumberOfElements);
assertEquals(7, dataElements1.maxOrdinal);

long v3 = oneRunCycle(p, new int[][] {{0}, {1}});
long v3 = oneRunCycle(p, new int[][] {{0}, {16}});
c.triggerRefreshTo(v3);
assertEquals(2, c.getStateEngine().getTypeState("TestList").numShards());

long v4 = oneRunCycle(p, new int[][] {{0}});
c.triggerRefreshTo(v4);
assertEquals(1, c.getStateEngine().getTypeState("TestList").numShards());
assertDataUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}});
assertValuesUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}});

long v5 = oneRunCycle(p, new int[][] {{0}, {1}, {2}});
long v5 = oneRunCycle(p, new int[][] {{0}, {1}, {16}});
c.triggerRefreshTo(v5);
assertDataUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}, {1}, {2}});
assertValuesUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}, {1}, {16}});

c.triggerRefreshTo(v1);
assertDataUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, {15}});
assertValuesUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, {15}});

c.triggerRefreshTo(v2);
assertDataUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, {15}, {16}});
assertValuesUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, {15}, {16}});

c.triggerRefreshTo(v5);
assertDataUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}, {1}, {2}});
assertValuesUnchanged((HollowListTypeReadState) c.getStateEngine().getTypeState("TestList"), new int[][] {{0}, {1}, {16}});
}

private long oneRunCycle(HollowProducer p, int listContents[][]) {
Expand Down Expand Up @@ -197,4 +204,34 @@ private long oneRunCycle(HollowProducer p, int listContents[][]) {
}
});
}

protected void assertValuesUnchanged(HollowListTypeReadState typeState, int[][] listContents) {
int numListRecords = listContents.length;
if (typeState.getListener(PopulatedOrdinalListener.class) != null) {
assertEquals(listContents.length, typeState.getPopulatedOrdinals().cardinality());
}
for(int i=0;i<numListRecords;i++) {
List<Integer> expected = Arrays.stream(listContents[i]).boxed().collect(Collectors.toList());
boolean matched = false;
List<Integer> actual = null;
for (int listRecordOridnal=0; listRecordOridnal<=typeState.maxOrdinal(); listRecordOridnal++) {
HollowOrdinalIterator iter = typeState.ordinalIterator(listRecordOridnal);
actual = new ArrayList<>();
int o = iter.next();
while (o != HollowOrdinalIterator.NO_MORE_ORDINALS) {
actual.add(((HollowObjectTypeReadState) typeState.getStateEngine().getTypeState("TestObject")).readInt(o, 2));
o = iter.next();
}
if (actual.equals(expected)) {
matched = true;
break;
}
}
if (!actual.equals(expected)) {
System.out.println("// SNAP: TODO: Remove this and similar");
}

assertTrue(matched);
}
}
}
53 changes: 34 additions & 19 deletions ...m/netflix/hollow/core/read/engine/map/AbstractHollowMapTypeDataElementsSplitJoinTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
package com.netflix.hollow.core.read.engine.map;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;

import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest;
import com.netflix.hollow.core.read.iterator.HollowMapEntryOrdinalIterator;
import com.netflix.hollow.core.schema.HollowMapSchema;
import com.netflix.hollow.core.write.HollowMapTypeWriteState;
import com.netflix.hollow.core.write.HollowMapWriteRecord;
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
Expand Down Expand Up @@ -40,7 +43,7 @@ protected void initializeTypeStates() {
writeStateEngine.setTargetMaxTypeShardSize(4 * 100 * 1000 * 1024);
}

private void populateWriteStateEngine(int[][][] maps) {
protected void populateWriteStateEngine(HollowWriteStateEngine writeStateEngine, int[][][] maps) {
// populate state so that there is 1:1 correspondence in key/value ordinal to value in int type
// find max value across all maps
if (maps.length > 0) {
Expand All @@ -49,8 +52,18 @@ private void populateWriteStateEngine(int[][][] maps) {
.flatMapToInt(Arrays::stream)
.max()
.orElseThrow(() -> new IllegalArgumentException("Array is empty"));

// populate write state with that many ordinals
super.populateWriteStateEngine(numKeyValueOrdinals);
HollowObjectWriteRecord rec = new HollowObjectWriteRecord(schema);
for(int i=0;i<numKeyValueOrdinals;i++) {
rec.reset();
rec.setLong("longField", i);
rec.setString("stringField", "Value" + i);
rec.setInt("intField", i);
rec.setDouble("doubleField", i);

writeStateEngine.add("TestObject", rec);
}
}
for(int[][] map : maps) {
HollowMapWriteRecord rec = new HollowMapWriteRecord();
Expand All @@ -63,12 +76,12 @@ private void populateWriteStateEngine(int[][][] maps) {
}

protected HollowMapTypeReadState populateTypeStateWith(int[][][] maps) throws IOException {
populateWriteStateEngine(maps);
populateWriteStateEngine(writeStateEngine, maps);
roundTripSnapshot();
return (HollowMapTypeReadState) readStateEngine.getTypeState("TestMap");
}

protected int[][][] generateListContents(int numRecords) {
protected int[][][] generateMapContents(int numRecords) {
int[][][] maps = new int[numRecords][][];
Random random = new Random();
int maxEntries = 10;
Expand All @@ -87,22 +100,24 @@ protected void assertDataUnchanged(HollowMapTypeReadState typeState, int[][][] m
int numMapRecords = maps.length;
for(int i=0;i<numMapRecords;i++) {
Map<Integer, Integer> expected = convertToMap(maps[i]);
Map<Integer, Integer> actual = readMap(typeState, i);
assertEquals(expected, actual);
}
}

public static Map<Integer, Integer> readMap(HollowMapTypeReadState typeState, int ordinal) {
Map<Integer, Integer> result = new HashMap<>();
HollowMapEntryOrdinalIterator iter = typeState.ordinalIterator(ordinal);
boolean hasMore = iter.next();
while (hasMore) {
int key = iter.getKey();
int value = iter.getValue();
result.put(key, value);
hasMore = iter.next();
boolean matched = false;
for (int mapRecordOridnal=0; mapRecordOridnal<=typeState.maxOrdinal(); mapRecordOridnal++) {
Map<Integer, Integer> actual = new HashMap<>();
HollowMapEntryOrdinalIterator iter = typeState.ordinalIterator(mapRecordOridnal);
boolean hasMore = iter.next();
while (hasMore) {
int key = iter.getKey();
int value = iter.getValue();
actual.put(key, value);
hasMore = iter.next();
}
if (actual.equals(expected)) {
matched = true;
break;
}
}
assertTrue(matched);
}
return result;
}

public static Map<Integer, Integer> convertToMap(int[][] array) {
Expand Down
Loading

0 comments on commit 593931e

Please sign in to comment.