[CPU] Optimize FullyConnected op in dynamic quantization mode

Author: dmitry-gorokhov

src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp

@@ -177,37 +177,27 @@ static bool useDynamicQuantizationImpl(size_t dqGroupSize,
         return false;
     }
 
-    // TODO: heuristic: disable avx2 asymmetric
-    bool is_asymmetric_weights = one_of(weightsDesc->getPrecision(), ov::element::u8, ov::element::u4);
-    if (is_asymmetric_weights && !dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core_vnni)) {
-        return false;
-    }
-
     const size_t simdWidth = 16;
     if (dqGroupSize % simdWidth) {
         return false;
     }
 
-    if (weightsDesc->getPrecision() == ov::element::u4) {
-        int ic = weightsDesc->getShape().getStaticDims()[1];
-        int minGroupSize = INT_MAX;
-
-        MemoryCPtr scalesPtr = memory.count(ARG_WEI | ARG_ATTR_SCALES) ? memory.at(ARG_WEI | ARG_ATTR_SCALES) : nullptr;
-
-        if (scalesPtr && scalesPtr->getShape().getRank() == 3) {
-            auto scalesDims = scalesPtr->getShape().getStaticDims();
-            auto groupsNum = needTranspose ? scalesDims[1] : scalesDims[0];
-            minGroupSize = ic / groupsNum;
-        }
-
-        if (zpPtr && zpPtr->getShape().getRank() == 3) {
-            auto zpDims = zpPtr->getShape().getStaticDims();
-            int groupsNum = needTranspose ? zpDims[1] : zpDims[0];
-            minGroupSize = std::min(minGroupSize, ic / groupsNum);
+    MemoryCPtr scalesPtr = memory.count(ARG_WEI | ARG_ATTR_SCALES) ? memory.at(ARG_WEI | ARG_ATTR_SCALES) : nullptr;
+    int ic = weightsDesc->getShape().getStaticDims()[1];
+    if (scalesPtr && scalesPtr->getShape().getRank() != 1) {
+        auto scalesDims = scalesPtr->getShape().getStaticDims();
+        auto groupsNum = scalesDims[1];
+        size_t groupSize = ic / groupsNum;
+        if (groupsNum != 1 && groupSize % std::min(dqGroupSize, groupSize)) {
+            return false;
         }
+    }
 
-        const size_t minLoopSize = 8;
-        if (minGroupSize != INT_MAX && minGroupSize % minLoopSize) {
+    if (zpPtr && zpPtr->getShape().getRank() != 1) {
+        auto zpDims = zpPtr->getShape().getStaticDims();
+        int groupsNum = zpDims[1];
+        size_t groupSize = ic / groupsNum;
+        if (groupsNum != 1 && groupSize % std::min(dqGroupSize, groupSize)) {
             return false;
         }
     }

src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/x64/matmul_weights_decompression.cpp

@@ -208,6 +208,8 @@ const std::vector<MatMulDecompressionShapeParams> input_shapes_basic_dyn_quant =
     {{{}, {{1, 1, 128}}}, {128, 32}},
     {{{}, {{1, 3, 144}}}, {144, 64}, 16lu},
     {{{}, {{1, 1, 1728}}}, {1728, 128}, 64lu},
+    // jit_brgemm_kernel corner cases: ic iters > 1 && has oc tail
+    {{{}, {{1, 1, 640}}}, {640, 90}},
 };
 
 const std::vector<ov::test::ElementType> weights_precisions_dyn_quant = {ov::element::u8, ov::element::u4};
@@ -280,8 +282,6 @@ const std::vector<MatMulDecompressionShapeParams> input_shapes_scalar_scale = {
     {{{}, {{1, 10, 128}}}, {128, 32}},
 };
 
-const std::vector<ov::test::ElementType> weights_precisions_scalar_scale = {ov::element::u8};
-
 std::vector<ov::AnyMap> filter_additional_config_scalar_scale() {
     std::vector<ov::AnyMap> additional_config = {
         {{ov::hint::dynamic_quantization_group_size(0)}},
@@ -293,7 +293,7 @@ std::vector<ov::AnyMap> filter_additional_config_scalar_scale() {
 INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_scalar_scale,
                          MatmulWeightsDecompression,
                          ::testing::Combine(::testing::ValuesIn(input_shapes_scalar_scale),
-                                            ::testing::ValuesIn(weights_precisions_scalar_scale),
+                                            ::testing::Values(ov::element::u8),
                                             ::testing::ValuesIn(decompression_precisions),
                                             ::testing::Values(ov::element::undefined),
                                             ::testing::Values(false),
@@ -305,6 +305,35 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_scalar_scale,
                                             ::testing::Values(true)),
                          MatmulWeightsDecompression::getTestCaseName);
 
+
+const std::vector<MatMulDecompressionShapeParams> input_shapes_non_multiples_groups = {
+    {{{}, {{1, 3, 192}}}, {192, 128}, 96lu},
+};
+
+std::vector<ov::AnyMap> filter_additional_config_non_multiples_groups() {
+    std::vector<ov::AnyMap> additional_config = {
+        {{ov::hint::dynamic_quantization_group_size(64)}}
+    };
+    return additional_config;
+}
+
+// Dynamic quantization requires weights compression group size to be divisible on dq group size
+// The test is intended to chech such case is correctly handled via non dq path
+INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_non_multiples_groups,
+                         MatmulWeightsDecompression,
+                         ::testing::Combine(::testing::ValuesIn(input_shapes_non_multiples_groups),
+                                            ::testing::Values(ov::element::u8),
+                                            ::testing::ValuesIn(decompression_precisions),
+                                            ::testing::Values(ov::element::undefined),
+                                            ::testing::ValuesIn(transpose_weights),
+                                            ::testing::Values(DecompressionType::full),
+                                            ::testing::Values(DecompressionType::full),
+                                            ::testing::Values(false),
+                                            ::testing::ValuesIn(filter_additional_config_non_multiples_groups()),
+                                            ::testing::Values(emptyFusingSpec),
+                                            ::testing::Values(true)),
+                         MatmulWeightsDecompression::getTestCaseName);
+
 }  // namespace
 }  // namespace test
 }  // namespace ov