[da-vinci] Add delayed ingestion in dvc for target region push #4658
Annotations
4 errors
|
Publish Test Report:
clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/SITWithPWiseWithoutBufferAfterLeaderTest.java#L1
Wanted but not invoked:
hostLevelIngestionStats.recordTotalBytesConsumed(
<any long>
);
-> at com.linkedin.davinci.stats.HostLevelIngestionStats.recordTotalBytesConsumed(HostLevelIngestionStats.java:499)
However, there were exactly 33 interactions with this mock:
hostLevelIngestionStats.recordProcessConsumerActionLatency(
12.0d
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerActions(StoreIngestionTask.java:1971)
hostLevelIngestionStats.recordCheckLongRunningTasksLatency(
0.00502d
);
-> at com.linkedin.davinci.kafka.consumer.LeaderFollowerStoreIngestionTask.checkLongRunningTaskState(LeaderFollowerStoreIngestionTask.java:773)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.recordQuotaMetrics(StoreIngestionTask.java:1637)
hostLevelIngestionStats.recordProcessConsumerActionLatency(
0.0d
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerActions(StoreIngestionTask.java:1971)
hostLevelIngestionStats.recordCheckLongRunningTasksLatency(
0.002805d
);
-> at com.linkedin.davinci.kafka.consumer.LeaderFollowerStoreIngestionTask.checkLongRunningTaskState(LeaderFollowerStoreIngestionTask.java:773)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.recordQuotaMetrics(StoreIngestionTask.java:1637)
hostLevelIngestionStats.recordConsumerRecordsQueuePutLatency(
0.101388d,
1740706442036L
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1304)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1309)
hostLevelIngestionStats.recordConsumerRecordsQueuePutLatency(
0.06621500000000001d,
1740706442037L
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1304)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1309)
hostLevelIngestionStats.recordTotalRecordsConsumed();
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerRecord(StoreIngestionTask.java:2633)
hostLevelIngestionStats.recordProcessConsumerActionLatency(
0.0d
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerActions(StoreIngestionTask.java:1971)
hostLevelIngestionStats.recordCheckLongRunningTasksLatency(
0.001933d
);
-> at com.linkedin.davinci.kafka.consumer.LeaderFollowerStoreIngestionTask.checkLongRunningTaskState(LeaderFollowerStoreIngestionTask.java:773)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.recordQuotaMetrics(StoreIngestionTask.java:1637)
hostLevelIngestionStats.recordProcessConsumerActionLatency(
0.0d
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerActions(StoreIngestionTask.java:1971)
hostLevelIngestionStats.recordCheckLongRunningTasksLatency(
0.002485d
);
-> at com.linkedin.davinci.kafka.consumer.LeaderFollowerStoreIngestionTask.checkLongRunningTaskState(LeaderFollowerStoreIngestionTask.java:773)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.recordQuotaMetrics(StoreIngestionTask.java:1637)
hostLevelIngestionStats.recordConsumerRecordsQueuePutLatency(
0.068298d,
1740706442053L
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1304)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1309)
hostLevelIngestionStats.recordTotalRecordsConsumed();
-> at com.linkedin.davinci.kafka.co
|
|
Publish Test Report:
clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/SITWithTWiseAndBufferAfterLeaderTest.java#L1
org.mockito.exceptions.verification.ArgumentsAreDifferent: Argument(s) are different! Wanted:
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
2,
OffsetRecord{localVersionTopicOffset=3, upstreamOffset=-1, leaderTopic=null, offsetLag=0, eventTimeEpochMs=-1, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=true, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTaskTest.lambda$testNotifier$35(StoreIngestionTaskTest.java:1623)
Actual invocations have different arguments:
storageMetadataService.getLastOffset(
"TestTopic_5d8e3a0877_230dfcc2_v1",
1
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.getLastOffset(DeepCopyOffsetManager.java:49)
storageMetadataService.getLastOffset(
"TestTopic_5d8e3a0877_230dfcc2_v1",
2
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.getLastOffset(DeepCopyOffsetManager.java:49)
storageMetadataService.computeStoreVersionState(
"TestTopic_5d8e3a0877_230dfcc2_v1",
com.linkedin.venice.offsets.DeepCopyStorageMetadataService$$Lambda$704/606988466@48db6378
);
-> at com.linkedin.venice.offsets.DeepCopyStorageMetadataService.computeStoreVersionState(DeepCopyStorageMetadataService.java:34)
storageMetadataService.computeStoreVersionState(
"TestTopic_5d8e3a0877_230dfcc2_v1",
com.linkedin.venice.offsets.DeepCopyStorageMetadataService$$Lambda$704/606988466@1a05fc52
);
-> at com.linkedin.venice.offsets.DeepCopyStorageMetadataService.computeStoreVersionState(DeepCopyStorageMetadataService.java:34)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
1,
OffsetRecord{localVersionTopicOffset=1, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=false, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.put(DeepCopyOffsetManager.java:38)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
2,
OffsetRecord{localVersionTopicOffset=1, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=false, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.put(DeepCopyOffsetManager.java:38)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
1,
OffsetRecord{localVersionTopicOffset=2, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=false, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.put(DeepCopyOffsetManager.java:38)
storageMetadataService.computeStoreVersionState(
"TestTopic_5d8e3a0877_230dfcc2_v1",
com.linkedin.venice.offsets.DeepCopyStorageMetadataService$$Lambda$704/606988466@28a98a17
);
-> at com.linkedin.venice.offsets.DeepCopyStorageMetadataService.computeStoreVersionState(DeepCopyStorageMetadataService.java:34)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
1,
OffsetRecord{localVersionTopicOffset=3, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=true, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.put(DeepCopyOffsetManager.java:38)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
2,
OffsetRecord{localVersionTopicOffset=2, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=false, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.li
|
|
Publish Test Report:
clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/SITWithPWiseWithoutBufferAfterLeaderTest.java#L1
Wanted but not invoked:
hostLevelIngestionStats.recordTotalBytesConsumed(
<any long>
);
-> at com.linkedin.davinci.stats.HostLevelIngestionStats.recordTotalBytesConsumed(HostLevelIngestionStats.java:499)
However, there were exactly 33 interactions with this mock:
hostLevelIngestionStats.recordProcessConsumerActionLatency(
12.0d
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerActions(StoreIngestionTask.java:1971)
hostLevelIngestionStats.recordCheckLongRunningTasksLatency(
0.00502d
);
-> at com.linkedin.davinci.kafka.consumer.LeaderFollowerStoreIngestionTask.checkLongRunningTaskState(LeaderFollowerStoreIngestionTask.java:773)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.recordQuotaMetrics(StoreIngestionTask.java:1637)
hostLevelIngestionStats.recordProcessConsumerActionLatency(
0.0d
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerActions(StoreIngestionTask.java:1971)
hostLevelIngestionStats.recordCheckLongRunningTasksLatency(
0.002805d
);
-> at com.linkedin.davinci.kafka.consumer.LeaderFollowerStoreIngestionTask.checkLongRunningTaskState(LeaderFollowerStoreIngestionTask.java:773)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.recordQuotaMetrics(StoreIngestionTask.java:1637)
hostLevelIngestionStats.recordConsumerRecordsQueuePutLatency(
0.101388d,
1740706442036L
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1304)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1309)
hostLevelIngestionStats.recordConsumerRecordsQueuePutLatency(
0.06621500000000001d,
1740706442037L
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1304)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1309)
hostLevelIngestionStats.recordTotalRecordsConsumed();
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerRecord(StoreIngestionTask.java:2633)
hostLevelIngestionStats.recordProcessConsumerActionLatency(
0.0d
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerActions(StoreIngestionTask.java:1971)
hostLevelIngestionStats.recordCheckLongRunningTasksLatency(
0.001933d
);
-> at com.linkedin.davinci.kafka.consumer.LeaderFollowerStoreIngestionTask.checkLongRunningTaskState(LeaderFollowerStoreIngestionTask.java:773)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.recordQuotaMetrics(StoreIngestionTask.java:1637)
hostLevelIngestionStats.recordProcessConsumerActionLatency(
0.0d
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.processConsumerActions(StoreIngestionTask.java:1971)
hostLevelIngestionStats.recordCheckLongRunningTasksLatency(
0.002485d
);
-> at com.linkedin.davinci.kafka.consumer.LeaderFollowerStoreIngestionTask.checkLongRunningTaskState(LeaderFollowerStoreIngestionTask.java:773)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.recordQuotaMetrics(StoreIngestionTask.java:1637)
hostLevelIngestionStats.recordConsumerRecordsQueuePutLatency(
0.068298d,
1740706442053L
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1304)
hostLevelIngestionStats.recordStorageQuotaUsed(
NaNd
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTask.produceToStoreBufferServiceOrKafka(StoreIngestionTask.java:1309)
hostLevelIngestionStats.recordTotalRecordsConsumed();
-> at com.linkedin.davinci.kafka.co
|
|
Publish Test Report:
clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/SITWithTWiseAndBufferAfterLeaderTest.java#L1
org.mockito.exceptions.verification.ArgumentsAreDifferent: Argument(s) are different! Wanted:
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
2,
OffsetRecord{localVersionTopicOffset=3, upstreamOffset=-1, leaderTopic=null, offsetLag=0, eventTimeEpochMs=-1, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=true, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.davinci.kafka.consumer.StoreIngestionTaskTest.lambda$testNotifier$35(StoreIngestionTaskTest.java:1623)
Actual invocations have different arguments:
storageMetadataService.getLastOffset(
"TestTopic_5d8e3a0877_230dfcc2_v1",
1
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.getLastOffset(DeepCopyOffsetManager.java:49)
storageMetadataService.getLastOffset(
"TestTopic_5d8e3a0877_230dfcc2_v1",
2
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.getLastOffset(DeepCopyOffsetManager.java:49)
storageMetadataService.computeStoreVersionState(
"TestTopic_5d8e3a0877_230dfcc2_v1",
com.linkedin.venice.offsets.DeepCopyStorageMetadataService$$Lambda$704/606988466@48db6378
);
-> at com.linkedin.venice.offsets.DeepCopyStorageMetadataService.computeStoreVersionState(DeepCopyStorageMetadataService.java:34)
storageMetadataService.computeStoreVersionState(
"TestTopic_5d8e3a0877_230dfcc2_v1",
com.linkedin.venice.offsets.DeepCopyStorageMetadataService$$Lambda$704/606988466@1a05fc52
);
-> at com.linkedin.venice.offsets.DeepCopyStorageMetadataService.computeStoreVersionState(DeepCopyStorageMetadataService.java:34)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
1,
OffsetRecord{localVersionTopicOffset=1, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=false, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.put(DeepCopyOffsetManager.java:38)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
2,
OffsetRecord{localVersionTopicOffset=1, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=false, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.put(DeepCopyOffsetManager.java:38)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
1,
OffsetRecord{localVersionTopicOffset=2, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=false, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.put(DeepCopyOffsetManager.java:38)
storageMetadataService.computeStoreVersionState(
"TestTopic_5d8e3a0877_230dfcc2_v1",
com.linkedin.venice.offsets.DeepCopyStorageMetadataService$$Lambda$704/606988466@28a98a17
);
-> at com.linkedin.venice.offsets.DeepCopyStorageMetadataService.computeStoreVersionState(DeepCopyStorageMetadataService.java:34)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
1,
OffsetRecord{localVersionTopicOffset=3, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=true, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.linkedin.venice.offsets.DeepCopyOffsetManager.put(DeepCopyOffsetManager.java:38)
storageMetadataService.put(
"TestTopic_5d8e3a0877_230dfcc2_v1",
2,
OffsetRecord{localVersionTopicOffset=2, upstreamOffset=-1, leaderTopic=null, offsetLag=9223372036854775807, eventTimeEpochMs=1740706393354, latestProducerProcessingTimeInMs=0, isEndOfPushReceived=false, databaseInfo={}, realTimeProducerState={}, recordTransformerClassHash=null}
);
-> at com.li
|
Loading