We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 82d6697 commit e5ec1ccCopy full SHA for e5ec1cc
src/cpp/src/continuous_batching_impl.cpp
@@ -258,7 +258,7 @@ ContinuousBatchingPipeline::ContinuousBatchingImpl::generate(const std::vector<o
258
bool continue_generation = true;
259
while (has_non_finished_requests() && continue_generation) {
260
step();
261
- if (streamer_ptr) {
+ if (streamer_ptr && generations.at(0)->can_read()) {
262
std::unordered_map<uint64_t, GenerationOutput> token = generations.at(0).get()->back();
263
OPENVINO_ASSERT(1 == token.size());
264
OPENVINO_ASSERT(1 == token.begin()->second.generated_ids.size());
0 commit comments