test fixes for latest transformers and review fixes

Author: dtrawins

optimum/intel/openvino/modeling_decoder.py

@@ -338,7 +338,6 @@ def compile(self):
         if self.compiled_model is None:
             super().compile()
             self.compiled_model = self.request
-            # self.request = self.request.create_infer_request()
 
     def _make_stateful(self):
         patch_stateful(self.config, self.model)
@@ -358,16 +357,11 @@ class OVModelForCausalLM(OVBaseDecoderModel, GenerationMixin):
 
     def generate(self, *args, **kwargs):
         self.compile()
-        infer_context = [self.compiled_model.create_infer_request()]
-        kwargs["infer_context"] = infer_context
+        if kwargs.get("infer_request") is None:
+            infer_context = [self.compiled_model.create_infer_request()]
+            kwargs["infer_context"] = infer_context
         return super().generate(*args, **kwargs)
 
-    def __call__(self, *args, **kwargs):
-        self.compile()
-        infer_context = [self.compiled_model.create_infer_request()]
-        kwargs["infer_context"] = infer_context
-        return super().__call__(*args, **kwargs)
-
     @add_start_docstrings_to_model_forward(
         INPUTS_DOCSTRING.format("batch_size, sequence_length")
         + TEXT_GENERATION_EXAMPLE.format(
@@ -482,7 +476,7 @@ def forward(
             # for stateful models, infer request is created in generate and __call_ methods and passed in the cycle via past_key_values param
             infer_request = past_key_values[1]
         else:
-            if infer_context[0] is not None:
+            if infer_context is not None:
                 infer_request = infer_context[
                     0
                 ]  # Use passed inference request if provided in kwargs, create new one overwise
@@ -501,7 +495,7 @@ def forward(
         if not self.stateful:
             if self.use_cache:
                 # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 corresponds to the self-attention layer)
-                past_key_values = tuple(infer_context[0].get_tensor(key).data for key in self.key_value_output_names)
+                past_key_values = tuple(infer_request.get_tensor(key).data for key in self.key_value_output_names)
                 if self.config.model_type not in MULTI_QUERY_ATTN_MODELS:
                     # Tuple of tuple of length `n_layers`, with each tuple of length equal to 2 (k/v of self-attention)
                     past_key_values = tuple(
@@ -690,9 +684,6 @@ def _reorder_cache(
             batch_size = beam_idx.shape[0]
             indices = np.array(range(batch_size * self.config.num_attention_heads))
             indices = indices.reshape([batch_size, self.config.num_attention_heads])
-            # self.next_beam_idx = np.take(indices, beam_idx, 0).flatten()
-            # return past_key_values
-            # print("_reorder_cache output",np.take(indices, beam_idx, 0).flatten())
             return ((np.take(indices, beam_idx, 0).flatten()), past_key_values[1])
         else:
             standardized_past = self._convert_to_standard_cache(past_key_values, batch_size=len(beam_idx))

tests/openvino/test_modeling.py

@@ -516,11 +516,9 @@ def test_compare_to_transformers(self, model_arch):
             input_shape = tokens["input_ids"].shape
             position_ids = torch.arange(0, input_shape[-1], dtype=torch.long).unsqueeze(0).view(-1, input_shape[-1])
         ov_outputs = ov_model(**tokens, position_ids=position_ids)
-
-        self.assertTrue("logits" in ov_outputs)
         self.assertIsInstance(ov_outputs.logits, torch.Tensor)
 
-        is_stateful = self.IS_SUPPORT_STATEFUL
+        is_stateful = ov_model.config.model_type not in {"gpt_bigcode", "llama"} and self.IS_SUPPORT_STATEFUL
         self.assertEqual(ov_model.stateful, is_stateful)
 
         with torch.no_grad():
@@ -541,7 +539,8 @@ def test_compare_to_transformers_multithreading(self, model_arch):
         ov_model = OVModelForCausalLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
         self.assertIsInstance(ov_model.config, PretrainedConfig)
         self.assertTrue(ov_model.use_cache)
-        self.assertEqual(ov_model.stateful, self.IS_SUPPORT_STATEFUL)
+        is_stateful = ov_model.config.model_type not in {"gpt_bigcode", "llama"} and self.IS_SUPPORT_STATEFUL
+        self.assertEqual(ov_model.stateful, is_stateful)
         transformers_model = AutoModelForCausalLM.from_pretrained(model_id)
         tokenizer = AutoTokenizer.from_pretrained(model_id)
         inputs_list = ["This is a sample", "Here is another sample", "That's the thrid one", "This is the last sample"]
@@ -607,7 +606,7 @@ def run_ov_model(input_text, model):
             # Tokenizer is not supposed to be shared by multiple threads
             tokenizer = AutoTokenizer.from_pretrained(model_id)
             pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
-            outputs = pipe(input_text, max_length=10)
+            outputs = pipe(input_text, max_length=30)
             self.assertEqual(pipe.device, model.device)
             for i in range(len(outputs)):
                 self.assertTrue(all(input_text[i] in item["generated_text"] for item in outputs[i]))