This backport contains 1 patch from gcc main stream tree. The commit id of these patchs list as following in the order of time. 0001-Apply-maximum-nunits-for-BB-SLP.patch 9b75f56d4b7951c60a656396dddd4a65787b95bc diff -Nurp a/gcc/testsuite/gcc.dg/vect/bb-slp-4.c b/gcc/testsuite/gcc.dg/vect/bb-slp-4.c --- a/gcc/testsuite/gcc.dg/vect/bb-slp-4.c 2020-12-20 18:46:19.539633230 +0800 +++ b/gcc/testsuite/gcc.dg/vect/bb-slp-4.c 2020-12-20 18:48:12.799633230 +0800 @@ -38,5 +38,4 @@ int main (void) return 0; } -/* { dg-final { scan-tree-dump-times "basic block vectorized" 0 "slp2" } } */ - +/* { dg-final { scan-tree-dump-times "basic block vectorized" 1 "slp2" } } */ diff -Nurp a/gcc/testsuite/gcc.dg/vect/bb-slp-bool-1.c b/gcc/testsuite/gcc.dg/vect/bb-slp-bool-1.c --- a/gcc/testsuite/gcc.dg/vect/bb-slp-bool-1.c 1970-01-01 08:00:00.000000000 +0800 +++ b/gcc/testsuite/gcc.dg/vect/bb-slp-bool-1.c 2020-12-20 18:48:12.799633230 +0800 @@ -0,0 +1,44 @@ +#include "tree-vect.h" + +void __attribute__ ((noipa)) +f1 (_Bool *x, unsigned short *y) +{ + x[0] = (y[0] == 1); + x[1] = (y[1] == 1); +} + +void __attribute__ ((noipa)) +f2 (_Bool *x, unsigned short *y) +{ + x[0] = (y[0] == 1); + x[1] = (y[1] == 1); + x[2] = (y[2] == 1); + x[3] = (y[3] == 1); + x[4] = (y[4] == 1); + x[5] = (y[5] == 1); + x[6] = (y[6] == 1); + x[7] = (y[7] == 1); +} + +_Bool x[8]; +unsigned short y[8] = { 11, 1, 9, 5, 1, 44, 1, 1 }; + +int +main (void) +{ + check_vect (); + + f1 (x, y); + + if (x[0] || !x[1]) + __builtin_abort (); + + x[1] = 0; + + f2 (x, y); + + if (x[0] || !x[1] || x[2] | x[3] || !x[4] || x[5] || !x[6] || !x[7]) + __builtin_abort (); + + return 0; +} diff -Nurp a/gcc/testsuite/gcc.target/aarch64/vect_mixed_sizes_14.c b/gcc/testsuite/gcc.target/aarch64/vect_mixed_sizes_14.c --- a/gcc/testsuite/gcc.target/aarch64/vect_mixed_sizes_14.c 1970-01-01 08:00:00.000000000 +0800 +++ b/gcc/testsuite/gcc.target/aarch64/vect_mixed_sizes_14.c 2020-12-20 18:48:11.811633230 +0800 @@ -0,0 +1,26 @@ +/* { dg-options "-O2 -ftree-vectorize" } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +/* +** foo: +** ( +** ldr d([0-9]+), \[x1\] +** ldr q([0-9]+), \[x0\] +** saddw v([0-9]+)\.4s, v\2\.4s, v\1\.4h +** str q\3, \[x0\] +** | +** ldr q([0-9]+), \[x0\] +** ldr d([0-9]+), \[x1\] +** saddw v([0-9]+)\.4s, v\4\.4s, v\5\.4h +** str q\6, \[x0\] +** ) +** ret +*/ +void +foo (int *x, short *y) +{ + x[0] += y[0]; + x[1] += y[1]; + x[2] += y[2]; + x[3] += y[3]; +} diff -Nurp a/gcc/testsuite/gcc.target/i386/pr84101.c b/gcc/testsuite/gcc.target/i386/pr84101.c --- a/gcc/testsuite/gcc.target/i386/pr84101.c 2020-12-20 18:46:18.383633230 +0800 +++ b/gcc/testsuite/gcc.target/i386/pr84101.c 2020-12-20 18:48:11.611633230 +0800 @@ -18,4 +18,5 @@ uint64_pair_t pair(int num) return p ; } -/* { dg-final { scan-tree-dump-not "basic block vectorized" "slp2" } } */ +/* See PR92266 for the XFAIL. */ +/* { dg-final { scan-tree-dump-not "basic block vectorized" "slp2" { xfail ilp32 } } } */ diff -Nurp a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c --- a/gcc/tree-vect-data-refs.c 2020-12-20 18:46:19.911633230 +0800 +++ b/gcc/tree-vect-data-refs.c 2020-12-20 18:48:11.047633230 +0800 @@ -4312,9 +4312,8 @@ vect_analyze_data_refs (vec_info *vinfo, /* Set vectype for STMT. */ scalar_type = TREE_TYPE (DR_REF (dr)); - STMT_VINFO_VECTYPE (stmt_info) - = get_vectype_for_scalar_type (vinfo, scalar_type); - if (!STMT_VINFO_VECTYPE (stmt_info)) + tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type); + if (!vectype) { if (dump_enabled_p ()) { @@ -4345,14 +4344,19 @@ vect_analyze_data_refs (vec_info *vinfo, if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "got vectype for stmt: %G%T\n", - stmt_info->stmt, STMT_VINFO_VECTYPE (stmt_info)); + stmt_info->stmt, vectype); } /* Adjust the minimal vectorization factor according to the vector type. */ - vf = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info)); + vf = TYPE_VECTOR_SUBPARTS (vectype); *min_vf = upper_bound (*min_vf, vf); + /* Leave the BB vectorizer to pick the vector type later, based on + the final dataref group size and SLP node size. */ + if (is_a (vinfo)) + STMT_VINFO_VECTYPE (stmt_info) = vectype; + if (gatherscatter != SG_NONE) { gather_scatter_info gs_info; diff -Nurp a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c --- a/gcc/tree-vect-patterns.c 2020-12-20 18:46:19.979633230 +0800 +++ b/gcc/tree-vect-patterns.c 2020-12-20 18:48:11.227633230 +0800 @@ -4142,9 +4142,10 @@ vect_recog_bool_pattern (stmt_vec_info s && STMT_VINFO_DATA_REF (stmt_vinfo)) { stmt_vec_info pattern_stmt_info; - vectype = STMT_VINFO_VECTYPE (stmt_vinfo); - gcc_assert (vectype != NULL_TREE); - if (!VECTOR_MODE_P (TYPE_MODE (vectype))) + tree nunits_vectype; + if (!vect_get_vector_types_for_stmt (stmt_vinfo, &vectype, + &nunits_vectype) + || !VECTOR_MODE_P (TYPE_MODE (vectype))) return NULL; if (check_bool_pattern (var, vinfo, bool_stmts)) diff -Nurp a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c --- a/gcc/tree-vect-slp.c 2020-12-20 18:46:17.763633230 +0800 +++ b/gcc/tree-vect-slp.c 2020-12-20 18:48:11.227633230 +0800 @@ -606,6 +606,77 @@ again: return 0; } +/* Try to assign vector type VECTYPE to STMT_INFO for BB vectorization. + Return true if we can, meaning that this choice doesn't conflict with + existing SLP nodes that use STMT_INFO. */ + +static bool +vect_update_shared_vectype (stmt_vec_info stmt_info, tree vectype) +{ + tree old_vectype = STMT_VINFO_VECTYPE (stmt_info); + if (old_vectype && useless_type_conversion_p (vectype, old_vectype)) + return true; + + if (STMT_VINFO_GROUPED_ACCESS (stmt_info) + && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))) + { + /* We maintain the invariant that if any statement in the group is + used, all other members of the group have the same vector type. */ + stmt_vec_info first_info = DR_GROUP_FIRST_ELEMENT (stmt_info); + stmt_vec_info member_info = first_info; + for (; member_info; member_info = DR_GROUP_NEXT_ELEMENT (member_info)) + if (STMT_VINFO_NUM_SLP_USES (member_info) > 0 + || is_pattern_stmt_p (member_info)) + break; + + if (!member_info) + { + for (member_info = first_info; member_info; + member_info = DR_GROUP_NEXT_ELEMENT (member_info)) + STMT_VINFO_VECTYPE (member_info) = vectype; + return true; + } + } + else if (STMT_VINFO_NUM_SLP_USES (stmt_info) == 0 + && !is_pattern_stmt_p (stmt_info)) + { + STMT_VINFO_VECTYPE (stmt_info) = vectype; + return true; + } + + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "Build SLP failed: incompatible vector" + " types for: %G", stmt_info->stmt); + dump_printf_loc (MSG_NOTE, vect_location, + " old vector type: %T\n", old_vectype); + dump_printf_loc (MSG_NOTE, vect_location, + " new vector type: %T\n", vectype); + } + return false; +} + +/* Try to infer and assign a vector type to all the statements in STMTS. + Used only for BB vectorization. */ + +static bool +vect_update_all_shared_vectypes (vec stmts) +{ + tree vectype, nunits_vectype; + if (!vect_get_vector_types_for_stmt (stmts[0], &vectype, + &nunits_vectype, stmts.length ())) + return false; + + stmt_vec_info stmt_info; + unsigned int i; + FOR_EACH_VEC_ELT (stmts, i, stmt_info) + if (!vect_update_shared_vectype (stmt_info, vectype)) + return false; + + return true; +} + /* Return true if call statements CALL1 and CALL2 are similar enough to be combined into the same SLP group. */ @@ -751,6 +822,7 @@ vect_build_slp_tree_1 (unsigned char *sw stmt_vec_info stmt_info; FOR_EACH_VEC_ELT (stmts, i, stmt_info) { + vec_info *vinfo = stmt_info->vinfo; gimple *stmt = stmt_info->stmt; swap[i] = 0; matches[i] = false; @@ -784,7 +856,7 @@ vect_build_slp_tree_1 (unsigned char *sw tree nunits_vectype; if (!vect_get_vector_types_for_stmt (stmt_info, &vectype, - &nunits_vectype) + &nunits_vectype, group_size) || (nunits_vectype && !vect_record_max_nunits (stmt_info, group_size, nunits_vectype, max_nunits))) @@ -796,6 +868,10 @@ vect_build_slp_tree_1 (unsigned char *sw gcc_assert (vectype); + if (is_a (vinfo) + && !vect_update_shared_vectype (stmt_info, vectype)) + continue; + if (gcall *call_stmt = dyn_cast (stmt)) { rhs_code = CALL_EXPR; @@ -1328,7 +1404,8 @@ vect_build_slp_tree_2 (vec_info *vinfo, FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild) if (SLP_TREE_DEF_TYPE (grandchild) != vect_external_def) break; - if (!grandchild) + if (!grandchild + && vect_update_all_shared_vectypes (oprnd_info->def_stmts)) { /* Roll back. */ this_tree_size = old_tree_size; @@ -1369,7 +1446,8 @@ vect_build_slp_tree_2 (vec_info *vinfo, do extra work to cancel the pattern so the uses see the scalar version. */ && !is_pattern_stmt_p (stmt_info) - && !oprnd_info->any_pattern) + && !oprnd_info->any_pattern + && vect_update_all_shared_vectypes (oprnd_info->def_stmts)) { if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1488,7 +1566,9 @@ vect_build_slp_tree_2 (vec_info *vinfo, FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild) if (SLP_TREE_DEF_TYPE (grandchild) != vect_external_def) break; - if (!grandchild) + if (!grandchild + && (vect_update_all_shared_vectypes + (oprnd_info->def_stmts))) { /* Roll back. */ this_tree_size = old_tree_size; @@ -2026,8 +2106,8 @@ vect_analyze_slp_instance (vec_info *vin if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) { scalar_type = TREE_TYPE (DR_REF (dr)); - vectype = get_vectype_for_scalar_type (vinfo, scalar_type); group_size = DR_GROUP_SIZE (stmt_info); + vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size); } else if (!dr && REDUC_GROUP_FIRST_ELEMENT (stmt_info)) { @@ -2669,22 +2749,13 @@ vect_slp_analyze_node_operations_1 (vec_ Memory accesses already got their vector type assigned in vect_analyze_data_refs. */ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); - if (bb_vinfo - && ! STMT_VINFO_DATA_REF (stmt_info)) + if (bb_vinfo && STMT_VINFO_VECTYPE (stmt_info) == boolean_type_node) { - tree vectype, nunits_vectype; - if (!vect_get_vector_types_for_stmt (stmt_info, &vectype, - &nunits_vectype)) - /* We checked this when building the node. */ - gcc_unreachable (); - if (vectype == boolean_type_node) - { - vectype = vect_get_mask_type_for_stmt (stmt_info); - if (!vectype) - /* vect_get_mask_type_for_stmt has already explained the - failure. */ - return false; - } + tree vectype = vect_get_mask_type_for_stmt (stmt_info, node); + if (!vectype) + /* vect_get_mask_type_for_stmt has already explained the + failure. */ + return false; stmt_vec_info sstmt_info; unsigned int i; @@ -3585,7 +3656,7 @@ vect_get_constant_vectors (slp_tree op_n && vect_mask_constant_operand_p (stmt_vinfo)) vector_type = truth_type_for (stmt_vectype); else - vector_type = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op)); + vector_type = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), op_node); unsigned int number_of_vectors = vect_get_num_vectors (SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) diff -Nurp a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c --- a/gcc/tree-vect-stmts.c 2020-12-20 18:46:17.707633230 +0800 +++ b/gcc/tree-vect-stmts.c 2020-12-20 18:48:11.227633230 +0800 @@ -798,7 +798,7 @@ vect_prologue_cost_for_slp_op (slp_tree /* Without looking at the actual initializer a vector of constants can be implemented as load from the constant pool. When all elements are the same we can use a splat. */ - tree vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op)); + tree vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op), node); unsigned group_size = SLP_TREE_SCALAR_STMTS (node).length (); unsigned num_vects_to_check; unsigned HOST_WIDE_INT const_nunits; @@ -3308,7 +3308,7 @@ vectorizable_call (stmt_vec_info stmt_in /* If all arguments are external or constant defs, infer the vector type from the scalar type. */ if (!vectype_in) - vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type); + vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node); if (vec_stmt) gcc_assert (vectype_in); if (!vectype_in) @@ -4106,7 +4106,8 @@ vectorizable_simd_clone_call (stmt_vec_i && bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR) { tree arg_type = TREE_TYPE (gimple_call_arg (stmt, i)); - arginfo[i].vectype = get_vectype_for_scalar_type (vinfo, arg_type); + arginfo[i].vectype = get_vectype_for_scalar_type (vinfo, arg_type, + slp_node); if (arginfo[i].vectype == NULL || (simd_clone_subparts (arginfo[i].vectype) > bestn->simdclone->simdlen)) @@ -4805,7 +4806,7 @@ vectorizable_conversion (stmt_vec_info s /* If op0 is an external or constant def, infer the vector type from the scalar type. */ if (!vectype_in) - vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type); + vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node); if (vec_stmt) gcc_assert (vectype_in); if (!vectype_in) @@ -5558,7 +5559,7 @@ vectorizable_shift (stmt_vec_info stmt_i /* If op0 is an external or constant def, infer the vector type from the scalar type. */ if (!vectype) - vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0)); + vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node); if (vec_stmt) gcc_assert (vectype); if (!vectype) @@ -5656,7 +5657,8 @@ vectorizable_shift (stmt_vec_info stmt_i "vector/vector shift/rotate found.\n"); if (!op1_vectype) - op1_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op1)); + op1_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op1), + slp_node); incompatible_op1_vectype_p = (op1_vectype == NULL_TREE || maybe_ne (TYPE_VECTOR_SUBPARTS (op1_vectype), @@ -6000,7 +6002,8 @@ vectorizable_operation (stmt_vec_info st vectype = vectype_out; } else - vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0)); + vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), + slp_node); } if (vec_stmt) gcc_assert (vectype); @@ -8903,7 +8906,7 @@ vectorizable_load (stmt_vec_info stmt_in condition operands are supportable using vec_is_simple_use. */ static bool -vect_is_simple_cond (tree cond, vec_info *vinfo, +vect_is_simple_cond (tree cond, vec_info *vinfo, slp_tree slp_node, tree *comp_vectype, enum vect_def_type *dts, tree vectype) { @@ -8966,7 +8969,8 @@ vect_is_simple_cond (tree cond, vec_info scalar_type = build_nonstandard_integer_type (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype))), TYPE_UNSIGNED (scalar_type)); - *comp_vectype = get_vectype_for_scalar_type (vinfo, scalar_type); + *comp_vectype = get_vectype_for_scalar_type (vinfo, scalar_type, + slp_node); } return true; @@ -9073,7 +9077,7 @@ vectorizable_condition (stmt_vec_info st then_clause = gimple_assign_rhs2 (stmt); else_clause = gimple_assign_rhs3 (stmt); - if (!vect_is_simple_cond (cond_expr, stmt_info->vinfo, + if (!vect_is_simple_cond (cond_expr, stmt_info->vinfo, slp_node, &comp_vectype, &dts[0], slp_node ? NULL : vectype) || !comp_vectype) return false; @@ -9564,7 +9568,8 @@ vectorizable_comparison (stmt_vec_info s /* Invariant comparison. */ if (!vectype) { - vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1)); + vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1), + slp_node); if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), nunits)) return false; } @@ -10322,31 +10327,93 @@ get_related_vectype_for_scalar_type (mac /* Function get_vectype_for_scalar_type. Returns the vector type corresponding to SCALAR_TYPE as supported - by the target. */ + by the target. If GROUP_SIZE is nonzero and we're performing BB + vectorization, make sure that the number of elements in the vector + is no bigger than GROUP_SIZE. */ tree -get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type) +get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type, + unsigned int group_size) { + /* For BB vectorization, we should always have a group size once we've + constructed the SLP tree; the only valid uses of zero GROUP_SIZEs + are tentative requests during things like early data reference + analysis and pattern recognition. */ + if (is_a (vinfo)) + gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0); + else + group_size = 0; + tree vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode, scalar_type); if (vectype && vinfo->vector_mode == VOIDmode) vinfo->vector_mode = TYPE_MODE (vectype); + /* Register the natural choice of vector type, before the group size + has been applied. */ if (vectype) vinfo->used_vector_modes.add (TYPE_MODE (vectype)); + /* If the natural choice of vector type doesn't satisfy GROUP_SIZE, + try again with an explicit number of elements. */ + if (vectype + && group_size + && maybe_ge (TYPE_VECTOR_SUBPARTS (vectype), group_size)) + { + /* Start with the biggest number of units that fits within + GROUP_SIZE and halve it until we find a valid vector type. + Usually either the first attempt will succeed or all will + fail (in the latter case because GROUP_SIZE is too small + for the target), but it's possible that a target could have + a hole between supported vector types. + + If GROUP_SIZE is not a power of 2, this has the effect of + trying the largest power of 2 that fits within the group, + even though the group is not a multiple of that vector size. + The BB vectorizer will then try to carve up the group into + smaller pieces. */ + unsigned int nunits = 1 << floor_log2 (group_size); + do + { + vectype = get_related_vectype_for_scalar_type (vinfo->vector_mode, + scalar_type, nunits); + nunits /= 2; + } + while (nunits > 1 && !vectype); + } + return vectype; } +/* Return the vector type corresponding to SCALAR_TYPE as supported + by the target. NODE, if nonnull, is the SLP tree node that will + use the returned vector type. */ + +tree +get_vectype_for_scalar_type (vec_info *vinfo, tree scalar_type, slp_tree node) +{ + unsigned int group_size = 0; + if (node) + { + group_size = SLP_TREE_SCALAR_OPS (node).length (); + if (group_size == 0) + group_size = SLP_TREE_SCALAR_STMTS (node).length (); + } + return get_vectype_for_scalar_type (vinfo, scalar_type, group_size); +} + /* Function get_mask_type_for_scalar_type. Returns the mask type corresponding to a result of comparison - of vectors of specified SCALAR_TYPE as supported by target. */ + of vectors of specified SCALAR_TYPE as supported by target. + NODE, if nonnull, is the SLP tree node that will use the returned + vector type. */ tree -get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type) +get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type, + slp_tree node) { - tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type); + tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, node); if (!vectype) return NULL; @@ -11033,6 +11100,9 @@ vect_gen_while_not (gimple_seq *seq, tre /* Try to compute the vector types required to vectorize STMT_INFO, returning true on success and false if vectorization isn't possible. + If GROUP_SIZE is nonzero and we're performing BB vectorization, + take sure that the number of elements in the vectors is no bigger + than GROUP_SIZE. On success: @@ -11050,11 +11120,21 @@ vect_gen_while_not (gimple_seq *seq, tre opt_result vect_get_vector_types_for_stmt (stmt_vec_info stmt_info, tree *stmt_vectype_out, - tree *nunits_vectype_out) + tree *nunits_vectype_out, + unsigned int group_size) { vec_info *vinfo = stmt_info->vinfo; gimple *stmt = stmt_info->stmt; + /* For BB vectorization, we should always have a group size once we've + constructed the SLP tree; the only valid uses of zero GROUP_SIZEs + are tentative requests during things like early data reference + analysis and pattern recognition. */ + if (is_a (vinfo)) + gcc_assert (vinfo->slp_instances.is_empty () || group_size != 0); + else + group_size = 0; + *stmt_vectype_out = NULL_TREE; *nunits_vectype_out = NULL_TREE; @@ -11085,7 +11165,7 @@ vect_get_vector_types_for_stmt (stmt_vec tree vectype; tree scalar_type = NULL_TREE; - if (STMT_VINFO_VECTYPE (stmt_info)) + if (group_size == 0 && STMT_VINFO_VECTYPE (stmt_info)) { *stmt_vectype_out = vectype = STMT_VINFO_VECTYPE (stmt_info); if (dump_enabled_p ()) @@ -11094,15 +11174,17 @@ vect_get_vector_types_for_stmt (stmt_vec } else { - gcc_assert (!STMT_VINFO_DATA_REF (stmt_info)); - if (gimple_call_internal_p (stmt, IFN_MASK_STORE)) + if (data_reference *dr = STMT_VINFO_DATA_REF (stmt_info)) + scalar_type = TREE_TYPE (DR_REF (dr)); + else if (gimple_call_internal_p (stmt, IFN_MASK_STORE)) scalar_type = TREE_TYPE (gimple_call_arg (stmt, 3)); else scalar_type = TREE_TYPE (gimple_get_lhs (stmt)); /* Pure bool ops don't participate in number-of-units computation. For comparisons use the types being compared. */ - if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type) + if (!STMT_VINFO_DATA_REF (stmt_info) + && VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type) && is_gimple_assign (stmt) && gimple_assign_rhs_code (stmt) != COND_EXPR) { @@ -11122,9 +11204,16 @@ vect_get_vector_types_for_stmt (stmt_vec } if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "get vectype for scalar type: %T\n", scalar_type); - vectype = get_vectype_for_scalar_type (vinfo, scalar_type); + { + if (group_size) + dump_printf_loc (MSG_NOTE, vect_location, + "get vectype for scalar type (group size %d):" + " %T\n", group_size, scalar_type); + else + dump_printf_loc (MSG_NOTE, vect_location, + "get vectype for scalar type: %T\n", scalar_type); + } + vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size); if (!vectype) return opt_result::failure_at (stmt, "not vectorized:" @@ -11155,7 +11244,8 @@ vect_get_vector_types_for_stmt (stmt_vec dump_printf_loc (MSG_NOTE, vect_location, "get vectype for smallest scalar type: %T\n", scalar_type); - nunits_vectype = get_vectype_for_scalar_type (vinfo, scalar_type); + nunits_vectype = get_vectype_for_scalar_type (vinfo, scalar_type, + group_size); if (!nunits_vectype) return opt_result::failure_at (stmt, "not vectorized: unsupported data-type %T\n", @@ -11183,10 +11273,11 @@ vect_get_vector_types_for_stmt (stmt_vec /* Try to determine the correct vector type for STMT_INFO, which is a statement that produces a scalar boolean result. Return the vector - type on success, otherwise return NULL_TREE. */ + type on success, otherwise return NULL_TREE. NODE, if nonnull, + is the SLP tree node that will use the returned vector type. */ opt_tree -vect_get_mask_type_for_stmt (stmt_vec_info stmt_info) +vect_get_mask_type_for_stmt (stmt_vec_info stmt_info, slp_tree node) { vec_info *vinfo = stmt_info->vinfo; gimple *stmt = stmt_info->stmt; @@ -11198,7 +11289,7 @@ vect_get_mask_type_for_stmt (stmt_vec_in && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (stmt)))) { scalar_type = TREE_TYPE (gimple_assign_rhs1 (stmt)); - mask_type = get_mask_type_for_scalar_type (vinfo, scalar_type); + mask_type = get_mask_type_for_scalar_type (vinfo, scalar_type, node); if (!mask_type) return opt_tree::failure_at (stmt, diff -Nurp a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h --- a/gcc/tree-vectorizer.h 2020-12-20 18:46:17.851633230 +0800 +++ b/gcc/tree-vectorizer.h 2020-12-20 18:48:11.227633230 +0800 @@ -1618,8 +1618,9 @@ extern void vect_update_inits_of_drs (lo /* In tree-vect-stmts.c. */ extern tree get_related_vectype_for_scalar_type (machine_mode, tree, poly_uint64 = 0); -extern tree get_vectype_for_scalar_type (vec_info *, tree); -extern tree get_mask_type_for_scalar_type (vec_info *, tree); +extern tree get_vectype_for_scalar_type (vec_info *, tree, unsigned int = 0); +extern tree get_vectype_for_scalar_type (vec_info *, tree, slp_tree); +extern tree get_mask_type_for_scalar_type (vec_info *, tree, slp_tree = 0); extern tree get_same_sized_vectype (tree, tree); extern bool vect_chooses_same_modes_p (vec_info *, machine_mode); extern bool vect_get_loop_mask_type (loop_vec_info); @@ -1671,8 +1672,8 @@ extern void optimize_mask_stores (struct extern gcall *vect_gen_while (tree, tree, tree); extern tree vect_gen_while_not (gimple_seq *, tree, tree, tree); extern opt_result vect_get_vector_types_for_stmt (stmt_vec_info, tree *, - tree *); -extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info); + tree *, unsigned int = 0); +extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info, slp_tree = 0); /* In tree-vect-data-refs.c. */ extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);