packages/gcc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
gcc/fix-ICE-in-vect.patch
xiezhiheng ae8eb7c877 [Upload patch] Upload patches to add new features 
and fix bugs

- add-fp-model-options.patch: New file
- enable-simd-math.patch: Enable simd math library in C and Fortran
- fix-CTOR-vectorization.patch: New file
- fix-range-set-by-vectorization-on-niter-IVs.patch: New file
- medium-code-mode.patch: Fix bugs when used with fpic
- optabs-Dont-use-scalar-conversions-for-vectors.patch: New file
- PR92429-do-not-fold-when-updating.patch: New file
- redundant-loop-elimination.patch: Fix some programming specifications
- fix-ICE-in-vect.patch: New file
- Fix-type-mismatch-in-SLPed-constructors.patch: New file
- add-check-for-pressure-in-sche1.patch: New file
- revert-moutline-atomics.patch: New file
- fix-ICE-in-eliminate-stmt.patch: New file
- revise-type-before-build-MULT.patch: New file
- Simplify-X-C1-C2.patch: New file
- gcc.spec: Add new patches
2021-04-29 10:30:11 +08:00

1038 lines
37 KiB
Diff
Raw Blame History

This backport contains 5 patch from gcc main stream tree.
The commit id of these patchs list as following in the order of time.
0001-Improve-tree-vect-patterns.c-handling-of-boolean-com.patch
ce19a4822794992097deab96bf15bf78ff481ea1
0002-Make-vectorizable_operation-punt-early-on-codes-it-d.patch
4177e933b309408e69eb5561fee7a3cc5e6f8899
0003-Make-vect_get_mask_type_for_stmt-take-a-group-size.patch
1c5d68a677b076262c5508e6d4fbdb765cba2d2f
0004-Record-the-vector-mask-precision-in-stmt_vec_info.patch
0c3ea6b3424ee4d32d97ca5d7453891b587b3132
0005-Don-t-defer-choice-of-vector-type-for-bools-PR-92596.patch
02d895504cc59be06fc3f7ec0cfd4eb160561211
diff -Nurp a/gcc/testsuite/gcc.dg/vect/bb-slp-43.c b/gcc/testsuite/gcc.dg/vect/bb-slp-43.c
--- a/gcc/testsuite/gcc.dg/vect/bb-slp-43.c 1970-01-01 08:00:00.000000000 +0800
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-43.c 2021-02-08 09:21:04.487633230 +0800
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+
+void
+f (int *restrict x, short *restrict y)
+{
+ x[0] = x[0] == 1 & y[0] == 2;
+ x[1] = x[1] == 1 & y[1] == 2;
+ x[2] = x[2] == 1 & y[2] == 2;
+ x[3] = x[3] == 1 & y[3] == 2;
+ x[4] = x[4] == 1 & y[4] == 2;
+ x[5] = x[5] == 1 & y[5] == 2;
+ x[6] = x[6] == 1 & y[6] == 2;
+ x[7] = x[7] == 1 & y[7] == 2;
+}
+
+/* { dg-final { scan-tree-dump-not "mixed mask and nonmask" "slp2" } } */
+/* { dg-final { scan-tree-dump-not "vector operands from scalars" "slp2" { target { { vect_int && vect_bool_cmp } && { vect_unpack && vect_hw_misalign } } xfail vect_variable_length } } } */
diff -Nurp a/gcc/testsuite/gcc.dg/vect/bb-slp-pr92596.c b/gcc/testsuite/gcc.dg/vect/bb-slp-pr92596.c
--- a/gcc/testsuite/gcc.dg/vect/bb-slp-pr92596.c 1970-01-01 08:00:00.000000000 +0800
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-pr92596.c 2021-02-08 09:21:04.487633230 +0800
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3" } */
+
+typedef struct {
+ long n[5];
+} secp256k1_fe;
+
+secp256k1_fe a;
+
+void fn1(int p1) { a.n[0] = a.n[1] = a.n[2] = p1; }
+void fn2() {
+ int b;
+ fn1(!b);
+}
diff -Nurp a/gcc/testsuite/gcc.dg/vect/vect-bool-cmp-2.c b/gcc/testsuite/gcc.dg/vect/vect-bool-cmp-2.c
--- a/gcc/testsuite/gcc.dg/vect/vect-bool-cmp-2.c 1970-01-01 08:00:00.000000000 +0800
+++ b/gcc/testsuite/gcc.dg/vect/vect-bool-cmp-2.c 2021-02-08 09:21:04.487633230 +0800
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+
+void
+f (_Bool *restrict x, _Bool *restrict y)
+{
+ for (int i = 0; i < 128; ++i)
+ x[i] = x[i] == y[i];
+}
+
+/* { dg-final { scan-tree-dump "loop vectorized" "vect" { target vect_bool_cmp } } } */
diff -Nurp a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
--- a/gcc/testsuite/lib/target-supports.exp 2021-02-08 09:24:27.611633230 +0800
+++ b/gcc/testsuite/lib/target-supports.exp 2021-02-08 09:21:03.859633230 +0800
@@ -5668,6 +5668,16 @@ proc check_effective_target_vect_bswap {
|| [istarget amdgcn-*-*] }}]
}
+# Return 1 if the target supports comparison of bool vectors for at
+# least one vector length.
+
+proc check_effective_target_vect_bool_cmp { } {
+ return [check_cached_effective_target_indexed vect_bool_cmp {
+ expr { [istarget i?86-*-*] || [istarget x86_64-*-*]
+ || [istarget aarch64*-*-*]
+ || [is-effective-target arm_neon] }}]
+}
+
# Return 1 if the target supports hardware vector shift operation for char.
proc check_effective_target_vect_shift_char { } {
diff -Nurp a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
--- a/gcc/tree-vect-loop.c 2021-02-08 09:24:26.471633230 +0800
+++ b/gcc/tree-vect-loop.c 2021-02-08 09:21:02.719633230 +0800
@@ -164,8 +164,7 @@ static stmt_vec_info vect_is_simple_redu
static opt_result
vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info,
bool vectype_maybe_set_p,
- poly_uint64 *vf,
- vec<stmt_vec_info > *mask_producers)
+ poly_uint64 *vf)
{
gimple *stmt = stmt_info->stmt;
@@ -193,8 +192,6 @@ vect_determine_vf_for_stmt_1 (stmt_vec_i
gcc_assert ((STMT_VINFO_DATA_REF (stmt_info)
|| vectype_maybe_set_p)
&& STMT_VINFO_VECTYPE (stmt_info) == stmt_vectype);
- else if (stmt_vectype == boolean_type_node)
- mask_producers->safe_push (stmt_info);
else
STMT_VINFO_VECTYPE (stmt_info) = stmt_vectype;
}
@@ -207,21 +204,17 @@ vect_determine_vf_for_stmt_1 (stmt_vec_i
/* Subroutine of vect_determine_vectorization_factor. Set the vector
types of STMT_INFO and all attached pattern statements and update
- the vectorization factor VF accordingly. If some of the statements
- produce a mask result whose vector type can only be calculated later,
- add them to MASK_PRODUCERS. Return true on success or false if
- something prevented vectorization. */
+ the vectorization factor VF accordingly. Return true on success
+ or false if something prevented vectorization. */
static opt_result
-vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf,
- vec<stmt_vec_info > *mask_producers)
+vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf)
{
vec_info *vinfo = stmt_info->vinfo;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
stmt_info->stmt);
- opt_result res
- = vect_determine_vf_for_stmt_1 (stmt_info, false, vf, mask_producers);
+ opt_result res = vect_determine_vf_for_stmt_1 (stmt_info, false, vf);
if (!res)
return res;
@@ -240,10 +233,7 @@ vect_determine_vf_for_stmt (stmt_vec_inf
dump_printf_loc (MSG_NOTE, vect_location,
"==> examining pattern def stmt: %G",
def_stmt_info->stmt);
- if (!vect_determine_vf_for_stmt_1 (def_stmt_info, true,
- vf, mask_producers))
- res = vect_determine_vf_for_stmt_1 (def_stmt_info, true,
- vf, mask_producers);
+ res = vect_determine_vf_for_stmt_1 (def_stmt_info, true, vf);
if (!res)
return res;
}
@@ -252,7 +242,7 @@ vect_determine_vf_for_stmt (stmt_vec_inf
dump_printf_loc (MSG_NOTE, vect_location,
"==> examining pattern statement: %G",
stmt_info->stmt);
- res = vect_determine_vf_for_stmt_1 (stmt_info, true, vf, mask_producers);
+ res = vect_determine_vf_for_stmt_1 (stmt_info, true, vf);
if (!res)
return res;
}
@@ -297,7 +287,6 @@ vect_determine_vectorization_factor (loo
tree vectype;
stmt_vec_info stmt_info;
unsigned i;
- auto_vec<stmt_vec_info> mask_producers;
DUMP_VECT_SCOPE ("vect_determine_vectorization_factor");
@@ -355,8 +344,7 @@ vect_determine_vectorization_factor (loo
{
stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
opt_result res
- = vect_determine_vf_for_stmt (stmt_info, &vectorization_factor,
- &mask_producers);
+ = vect_determine_vf_for_stmt (stmt_info, &vectorization_factor);
if (!res)
return res;
}
@@ -374,16 +362,6 @@ vect_determine_vectorization_factor (loo
return opt_result::failure_at (vect_location,
"not vectorized: unsupported data-type\n");
LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
-
- for (i = 0; i < mask_producers.length (); i++)
- {
- stmt_info = mask_producers[i];
- opt_tree mask_type = vect_get_mask_type_for_stmt (stmt_info);
- if (!mask_type)
- return opt_result::propagate_failure (mask_type);
- STMT_VINFO_VECTYPE (stmt_info) = mask_type;
- }
-
return opt_result::success ();
}
diff -Nurp a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
--- a/gcc/tree-vectorizer.h 2021-02-08 09:24:26.463633230 +0800
+++ b/gcc/tree-vectorizer.h 2021-02-08 09:21:02.619633230 +0800
@@ -1080,6 +1080,23 @@ struct _stmt_vec_info {
unsigned int operation_precision;
signop operation_sign;
+ /* If the statement produces a boolean result, this value describes
+ how we should choose the associated vector type. The possible
+ values are:
+
+ - an integer precision N if we should use the vector mask type
+ associated with N-bit integers. This is only used if all relevant
+ input booleans also want the vector mask type for N-bit integers,
+ or if we can convert them into that form by pattern-matching.
+
+ - ~0U if we considered choosing a vector mask type but decided
+ to treat the boolean as a normal integer type instead.
+
+ - 0 otherwise. This means either that the operation isn't one that
+ could have a vector mask type (and so should have a normal vector
+ type instead) or that we simply haven't made a choice either way. */
+ unsigned int mask_precision;
+
/* True if this is only suitable for SLP vectorization. */
bool slp_vect_only_p;
};
@@ -1236,6 +1253,15 @@ nested_in_vect_loop_p (struct loop *loop
&& (loop->inner == (gimple_bb (stmt_info->stmt))->loop_father));
}
+/* Return true if STMT_INFO should produce a vector mask type rather than
+ a normal nonmask type. */
+
+static inline bool
+vect_use_mask_type_p (stmt_vec_info stmt_info)
+{
+ return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
+}
+
/* Return TRUE if a statement represented by STMT_INFO is a part of a
pattern. */
@@ -1620,7 +1646,7 @@ extern tree get_related_vectype_for_scal
poly_uint64 = 0);
extern tree get_vectype_for_scalar_type (vec_info *, tree, unsigned int = 0);
extern tree get_vectype_for_scalar_type (vec_info *, tree, slp_tree);
-extern tree get_mask_type_for_scalar_type (vec_info *, tree, slp_tree = 0);
+extern tree get_mask_type_for_scalar_type (vec_info *, tree, unsigned int = 0);
extern tree get_same_sized_vectype (tree, tree);
extern bool vect_chooses_same_modes_p (vec_info *, machine_mode);
extern bool vect_get_loop_mask_type (loop_vec_info);
@@ -1673,7 +1699,7 @@ extern gcall *vect_gen_while (tree, tree
extern tree vect_gen_while_not (gimple_seq *, tree, tree, tree);
extern opt_result vect_get_vector_types_for_stmt (stmt_vec_info, tree *,
tree *, unsigned int = 0);
-extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info, slp_tree = 0);
+extern opt_tree vect_get_mask_type_for_stmt (stmt_vec_info, unsigned int = 0);
/* In tree-vect-data-refs.c. */
extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);
diff -Nurp a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
--- a/gcc/tree-vect-patterns.c 2021-02-08 09:24:26.467633230 +0800
+++ b/gcc/tree-vect-patterns.c 2021-02-08 09:21:02.543633230 +0800
@@ -112,7 +112,12 @@ vect_init_pattern_stmt (gimple *pattern_
STMT_VINFO_DEF_TYPE (pattern_stmt_info)
= STMT_VINFO_DEF_TYPE (orig_stmt_info);
if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
- STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype;
+ {
+ gcc_assert (VECTOR_BOOLEAN_TYPE_P (vectype)
+ == vect_use_mask_type_p (orig_stmt_info));
+ STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype;
+ pattern_stmt_info->mask_precision = orig_stmt_info->mask_precision;
+ }
return pattern_stmt_info;
}
@@ -131,17 +136,25 @@ vect_set_pattern_stmt (gimple *pattern_s
/* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE
is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
- be different from the vector type of the final pattern statement. */
+ be different from the vector type of the final pattern statement.
+ If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
+ from which it was derived. */
static inline void
append_pattern_def_seq (stmt_vec_info stmt_info, gimple *new_stmt,
- tree vectype = NULL_TREE)
+ tree vectype = NULL_TREE,
+ tree scalar_type_for_mask = NULL_TREE)
{
+ gcc_assert (!scalar_type_for_mask
+ == (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype)));
vec_info *vinfo = stmt_info->vinfo;
if (vectype)
{
stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
+ if (scalar_type_for_mask)
+ new_stmt_info->mask_precision
+ = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (scalar_type_for_mask));
}
gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
new_stmt);
@@ -3875,107 +3888,22 @@ adjust_bool_stmts (hash_set <gimple *> &
return gimple_assign_lhs (pattern_stmt);
}
-/* Helper for search_type_for_mask. */
+/* Return the proper type for converting bool VAR into
+ an integer value or NULL_TREE if no such type exists.
+ The type is chosen so that the converted value has the
+ same number of elements as VAR's vector type. */
static tree
-search_type_for_mask_1 (tree var, vec_info *vinfo,
- hash_map<gimple *, tree> &cache)
+integer_type_for_mask (tree var, vec_info *vinfo)
{
- tree rhs1;
- enum tree_code rhs_code;
- tree res = NULL_TREE, res2;
-
if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
return NULL_TREE;
stmt_vec_info def_stmt_info = vect_get_internal_def (vinfo, var);
- if (!def_stmt_info)
+ if (!def_stmt_info || !vect_use_mask_type_p (def_stmt_info))
return NULL_TREE;
- gassign *def_stmt = dyn_cast <gassign *> (def_stmt_info->stmt);
- if (!def_stmt)
- return NULL_TREE;
-
- tree *c = cache.get (def_stmt);
- if (c)
- return *c;
-
- rhs_code = gimple_assign_rhs_code (def_stmt);
- rhs1 = gimple_assign_rhs1 (def_stmt);
-
- switch (rhs_code)
- {
- case SSA_NAME:
- case BIT_NOT_EXPR:
- CASE_CONVERT:
- res = search_type_for_mask_1 (rhs1, vinfo, cache);
- break;
-
- case BIT_AND_EXPR:
- case BIT_IOR_EXPR:
- case BIT_XOR_EXPR:
- res = search_type_for_mask_1 (rhs1, vinfo, cache);
- res2 = search_type_for_mask_1 (gimple_assign_rhs2 (def_stmt), vinfo,
- cache);
- if (!res || (res2 && TYPE_PRECISION (res) > TYPE_PRECISION (res2)))
- res = res2;
- break;
-
- default:
- if (TREE_CODE_CLASS (rhs_code) == tcc_comparison)
- {
- tree comp_vectype, mask_type;
-
- if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1)))
- {
- res = search_type_for_mask_1 (rhs1, vinfo, cache);
- res2 = search_type_for_mask_1 (gimple_assign_rhs2 (def_stmt),
- vinfo, cache);
- if (!res || (res2 && TYPE_PRECISION (res) > TYPE_PRECISION (res2)))
- res = res2;
- break;
- }
-
- comp_vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs1));
- if (comp_vectype == NULL_TREE)
- {
- res = NULL_TREE;
- break;
- }
-
- mask_type = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (rhs1));
- if (!mask_type
- || !expand_vec_cmp_expr_p (comp_vectype, mask_type, rhs_code))
- {
- res = NULL_TREE;
- break;
- }
-
- if (TREE_CODE (TREE_TYPE (rhs1)) != INTEGER_TYPE
- || !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
- {
- scalar_mode mode = SCALAR_TYPE_MODE (TREE_TYPE (rhs1));
- res = build_nonstandard_integer_type (GET_MODE_BITSIZE (mode), 1);
- }
- else
- res = TREE_TYPE (rhs1);
- }
- }
-
- cache.put (def_stmt, res);
- return res;
-}
-
-/* Return the proper type for converting bool VAR into
- an integer value or NULL_TREE if no such type exists.
- The type is chosen so that converted value has the
- same number of elements as VAR's vector type. */
-
-static tree
-search_type_for_mask (tree var, vec_info *vinfo)
-{
- hash_map<gimple *, tree> cache;
- return search_type_for_mask_1 (var, vinfo, cache);
+ return build_nonstandard_integer_type (def_stmt_info->mask_precision, 1);
}
/* Function vect_recog_bool_pattern
@@ -4067,7 +3995,7 @@ vect_recog_bool_pattern (stmt_vec_info s
}
else
{
- tree type = search_type_for_mask (var, vinfo);
+ tree type = integer_type_for_mask (var, vinfo);
tree cst0, cst1, tmp;
if (!type)
@@ -4152,7 +4080,7 @@ vect_recog_bool_pattern (stmt_vec_info s
rhs = adjust_bool_stmts (bool_stmts, TREE_TYPE (vectype), stmt_vinfo);
else
{
- tree type = search_type_for_mask (var, vinfo);
+ tree type = integer_type_for_mask (var, vinfo);
tree cst0, cst1, new_vectype;
if (!type)
@@ -4207,7 +4135,7 @@ build_mask_conversion (tree mask, tree v
masktype = truth_type_for (vectype);
tmp = vect_recog_temp_ssa_var (TREE_TYPE (masktype), NULL);
stmt = gimple_build_assign (tmp, CONVERT_EXPR, mask);
- append_pattern_def_seq (stmt_vinfo, stmt, masktype);
+ append_pattern_def_seq (stmt_vinfo, stmt, masktype, TREE_TYPE (vectype));
return tmp;
}
@@ -4275,7 +4203,7 @@ vect_recog_mask_conversion_pattern (stmt
}
tree mask_arg = gimple_call_arg (last_stmt, mask_argno);
- tree mask_arg_type = search_type_for_mask (mask_arg, vinfo);
+ tree mask_arg_type = integer_type_for_mask (mask_arg, vinfo);
if (!mask_arg_type)
return NULL;
vectype2 = get_mask_type_for_scalar_type (vinfo, mask_arg_type);
@@ -4328,7 +4256,7 @@ vect_recog_mask_conversion_pattern (stmt
if (TREE_CODE (rhs1) == SSA_NAME)
{
- rhs1_type = search_type_for_mask (rhs1, vinfo);
+ rhs1_type = integer_type_for_mask (rhs1, vinfo);
if (!rhs1_type)
return NULL;
}
@@ -4352,8 +4280,8 @@ vect_recog_mask_conversion_pattern (stmt
rhs1_op1 = TREE_OPERAND (rhs1, 1);
if (!rhs1_op0 || !rhs1_op1)
return NULL;
- rhs1_op0_type = search_type_for_mask (rhs1_op0, vinfo);
- rhs1_op1_type = search_type_for_mask (rhs1_op1, vinfo);
+ rhs1_op0_type = integer_type_for_mask (rhs1_op0, vinfo);
+ rhs1_op1_type = integer_type_for_mask (rhs1_op1, vinfo);
if (!rhs1_op0_type)
rhs1_type = TREE_TYPE (rhs1_op0);
@@ -4441,7 +4369,8 @@ vect_recog_mask_conversion_pattern (stmt
pattern_stmt = gimple_build_assign (tmp, TREE_CODE (rhs1),
rhs1_op0, rhs1_op1);
rhs1 = tmp;
- append_pattern_def_seq (stmt_vinfo, pattern_stmt, vectype2);
+ append_pattern_def_seq (stmt_vinfo, pattern_stmt, vectype2,
+ rhs1_type);
}
if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
@@ -4474,8 +4403,8 @@ vect_recog_mask_conversion_pattern (stmt
rhs2 = gimple_assign_rhs2 (last_stmt);
- rhs1_type = search_type_for_mask (rhs1, vinfo);
- rhs2_type = search_type_for_mask (rhs2, vinfo);
+ rhs1_type = integer_type_for_mask (rhs1, vinfo);
+ rhs2_type = integer_type_for_mask (rhs2, vinfo);
if (!rhs1_type || !rhs2_type
|| TYPE_PRECISION (rhs1_type) == TYPE_PRECISION (rhs2_type))
@@ -4558,7 +4487,7 @@ static tree
vect_convert_mask_for_vectype (tree mask, tree vectype,
stmt_vec_info stmt_info, vec_info *vinfo)
{
- tree mask_type = search_type_for_mask (mask, vinfo);
+ tree mask_type = integer_type_for_mask (mask, vinfo);
if (mask_type)
{
tree mask_vectype = get_mask_type_for_scalar_type (vinfo, mask_type);
@@ -4997,6 +4926,148 @@ vect_determine_precisions_from_users (st
vect_set_min_input_precision (stmt_info, type, min_input_precision);
}
+/* Return true if the statement described by STMT_INFO sets a boolean
+ SSA_NAME and if we know how to vectorize this kind of statement using
+ vector mask types. */
+
+static bool
+possible_vector_mask_operation_p (stmt_vec_info stmt_info)
+{
+ tree lhs = gimple_get_lhs (stmt_info->stmt);
+ if (!lhs
+ || TREE_CODE (lhs) != SSA_NAME
+ || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
+ return false;
+
+ if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
+ {
+ tree_code rhs_code = gimple_assign_rhs_code (assign);
+ switch (rhs_code)
+ {
+ CASE_CONVERT:
+ case SSA_NAME:
+ case BIT_NOT_EXPR:
+ case BIT_IOR_EXPR:
+ case BIT_XOR_EXPR:
+ case BIT_AND_EXPR:
+ return true;
+
+ default:
+ return TREE_CODE_CLASS (rhs_code) == tcc_comparison;
+ }
+ }
+ return false;
+}
+
+/* If STMT_INFO sets a boolean SSA_NAME, see whether we should use
+ a vector mask type instead of a normal vector type. Record the
+ result in STMT_INFO->mask_precision. */
+
+static void
+vect_determine_mask_precision (stmt_vec_info stmt_info)
+{
+ vec_info *vinfo = stmt_info->vinfo;
+
+ if (!possible_vector_mask_operation_p (stmt_info)
+ || stmt_info->mask_precision)
+ return;
+
+ auto_vec<stmt_vec_info, 32> worklist;
+ worklist.quick_push (stmt_info);
+ while (!worklist.is_empty ())
+ {
+ stmt_info = worklist.last ();
+ unsigned int orig_length = worklist.length ();
+
+ /* If at least one boolean input uses a vector mask type,
+ pick the mask type with the narrowest elements.
+
+ ??? This is the traditional behavior. It should always produce
+ the smallest number of operations, but isn't necessarily the
+ optimal choice. For example, if we have:
+
+ a = b & c
+
+ where:
+
+ - the user of a wants it to have a mask type for 16-bit elements (M16)
+ - b also uses M16
+ - c uses a mask type for 8-bit elements (M8)
+
+ then picking M8 gives:
+
+ - 1 M16->M8 pack for b
+ - 1 M8 AND for a
+ - 2 M8->M16 unpacks for the user of a
+
+ whereas picking M16 would have given:
+
+ - 2 M8->M16 unpacks for c
+ - 2 M16 ANDs for a
+
+ The number of operations are equal, but M16 would have given
+ a shorter dependency chain and allowed more ILP. */
+ unsigned int precision = ~0U;
+ gassign *assign = as_a <gassign *> (stmt_info->stmt);
+ unsigned int nops = gimple_num_ops (assign);
+ for (unsigned int i = 1; i < nops; ++i)
+ {
+ tree rhs = gimple_op (assign, i);
+ if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs)))
+ continue;
+
+ stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
+ if (!def_stmt_info)
+ /* Don't let external or constant operands influence the choice.
+ We can convert them to whichever vector type we pick. */
+ continue;
+
+ if (def_stmt_info->mask_precision)
+ {
+ if (precision > def_stmt_info->mask_precision)
+ precision = def_stmt_info->mask_precision;
+ }
+ else if (possible_vector_mask_operation_p (def_stmt_info))
+ worklist.safe_push (def_stmt_info);
+ }
+
+ /* Defer the choice if we need to visit operands first. */
+ if (orig_length != worklist.length ())
+ continue;
+
+ /* If the statement compares two values that shouldn't use vector masks,
+ try comparing the values as normal scalars instead. */
+ tree_code rhs_code = gimple_assign_rhs_code (assign);
+ if (precision == ~0U
+ && TREE_CODE_CLASS (rhs_code) == tcc_comparison)
+ {
+ tree rhs1_type = TREE_TYPE (gimple_assign_rhs1 (assign));
+ scalar_mode mode;
+ tree vectype, mask_type;
+ if (is_a <scalar_mode> (TYPE_MODE (rhs1_type), &mode)
+ && (vectype = get_vectype_for_scalar_type (vinfo, rhs1_type))
+ && (mask_type = get_mask_type_for_scalar_type (vinfo, rhs1_type))
+ && expand_vec_cmp_expr_p (vectype, mask_type, rhs_code))
+ precision = GET_MODE_BITSIZE (mode);
+ }
+
+ if (dump_enabled_p ())
+ {
+ if (precision == ~0U)
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "using normal nonmask vectors for %G",
+ stmt_info->stmt);
+ else
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "using boolean precision %d for %G",
+ precision, stmt_info->stmt);
+ }
+
+ stmt_info->mask_precision = precision;
+ worklist.pop ();
+ }
+}
+
/* Handle vect_determine_precisions for STMT_INFO, given that we
have already done so for the users of its result. */
@@ -5009,6 +5080,7 @@ vect_determine_stmt_precisions (stmt_vec
vect_determine_precisions_from_range (stmt_info, stmt);
vect_determine_precisions_from_users (stmt_info, stmt);
}
+ vect_determine_mask_precision (stmt_info);
}
/* Walk backwards through the vectorizable region to determine the
+-search_type_for_mask (tree var, vec_info *vinfo)
+-{
+- hash_map<gimple *, tree> cache;
+- return search_type_for_mask_1 (var, vinfo, cache);
++ return build_nonstandard_integer_type (def_stmt_info->mask_precision, 1);
+ }
+
+ /* Function vect_recog_bool_pattern
+@@ -4371,7 +4298,7 @@ vect_recog_mask_conversion_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+
+ it is better for b1 and b2 to use the mask type associated
+ with int elements rather bool (byte) elements. */
+- rhs1_type = search_type_for_mask (TREE_OPERAND (rhs1, 0), vinfo);
++ rhs1_type = integer_type_for_mask (TREE_OPERAND (rhs1, 0), vinfo);
+ if (!rhs1_type)
+ rhs1_type = TREE_TYPE (TREE_OPERAND (rhs1, 0));
+ }
+@@ -4427,7 +4354,8 @@ vect_recog_mask_conversion_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
+ tmp = vect_recog_temp_ssa_var (TREE_TYPE (rhs1), NULL);
+ pattern_stmt = gimple_build_assign (tmp, rhs1);
+ rhs1 = tmp;
+- append_pattern_def_seq (stmt_vinfo, pattern_stmt, vectype2);
++ append_pattern_def_seq (stmt_vinfo, pattern_stmt, vectype2,
++ rhs1_type);
+ }
+
+ if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
diff -Nurp a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
--- a/gcc/tree-vect-slp.c 2021-02-08 09:24:26.471633230 +0800
+++ b/gcc/tree-vect-slp.c 2021-02-08 09:21:02.719633230 +0800
@@ -906,17 +906,6 @@ vect_build_slp_tree_1 (unsigned char *sw
|| rhs_code == LROTATE_EXPR
|| rhs_code == RROTATE_EXPR)
{
- if (vectype == boolean_type_node)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Build SLP failed: shift of a"
- " boolean.\n");
- /* Fatal mismatch. */
- matches[0] = false;
- return false;
- }
-
vec_mode = TYPE_MODE (vectype);
/* First see if we have a vector/vector shift. */
@@ -1137,9 +1126,8 @@ vect_build_slp_tree_1 (unsigned char *sw
if (alt_stmt_code != ERROR_MARK
&& TREE_CODE_CLASS (alt_stmt_code) != tcc_reference)
{
- if (vectype == boolean_type_node
- || !vect_two_operations_perm_ok_p (stmts, group_size,
- vectype, alt_stmt_code))
+ if (!vect_two_operations_perm_ok_p (stmts, group_size,
+ vectype, alt_stmt_code))
{
for (i = 0; i < group_size; ++i)
if (gimple_assign_rhs_code (stmts[i]->stmt) == alt_stmt_code)
@@ -2746,24 +2734,6 @@ vect_slp_analyze_node_operations_1 (vec_
stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
gcc_assert (STMT_SLP_TYPE (stmt_info) != loop_vect);
- /* For BB vectorization vector types are assigned here.
- Memory accesses already got their vector type assigned
- in vect_analyze_data_refs. */
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- if (bb_vinfo && STMT_VINFO_VECTYPE (stmt_info) == boolean_type_node)
- {
- tree vectype = vect_get_mask_type_for_stmt (stmt_info, node);
- if (!vectype)
- /* vect_get_mask_type_for_stmt has already explained the
- failure. */
- return false;
-
- stmt_vec_info sstmt_info;
- unsigned int i;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, sstmt_info)
- STMT_VINFO_VECTYPE (sstmt_info) = vectype;
- }
-
/* Calculate the number of vector statements to be created for the
scalar stmts in this node. For SLP reductions it is equal to the
number of vector statements in the children (which has already been
diff -Nurp a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
--- a/gcc/tree-vect-stmts.c 2021-02-08 09:24:26.371633230 +0800
+++ b/gcc/tree-vect-stmts.c 2021-02-08 09:21:02.543633230 +0800
@@ -3334,6 +3334,15 @@ vectorizable_call (stmt_vec_info stmt_in
return false;
}
+ if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
+ != VECTOR_BOOLEAN_TYPE_P (vectype_in))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "mixed mask and nonmask vector types\n");
+ return false;
+ }
+
/* FORNOW */
nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
@@ -5938,6 +5947,21 @@ vectorizable_operation (stmt_vec_info st
orig_code = code = gimple_assign_rhs_code (stmt);
+ /* Shifts are handled in vectorizable_shift. */
+ if (code == LSHIFT_EXPR
+ || code == RSHIFT_EXPR
+ || code == LROTATE_EXPR
+ || code == RROTATE_EXPR)
+ return false;
+
+ /* Comparisons are handled in vectorizable_comparison. */
+ if (TREE_CODE_CLASS (code) == tcc_comparison)
+ return false;
+
+ /* Conditions are handled in vectorizable_condition. */
+ if (code == COND_EXPR)
+ return false;
+
/* For pointer addition and subtraction, we should use the normal
plus and minus for the vector operation. */
if (code == POINTER_PLUS_EXPR)
@@ -5961,7 +5985,8 @@ vectorizable_operation (stmt_vec_info st
/* Most operations cannot handle bit-precision types without extra
truncations. */
- if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
+ bool mask_op_p = VECTOR_BOOLEAN_TYPE_P (vectype_out);
+ if (!mask_op_p
&& !type_has_mode_precision_p (TREE_TYPE (scalar_dest))
/* Exception are bitwise binary operations. */
&& code != BIT_IOR_EXPR
@@ -6023,10 +6048,11 @@ vectorizable_operation (stmt_vec_info st
if (maybe_ne (nunits_out, nunits_in))
return false;
+ tree vectype2 = NULL_TREE, vectype3 = NULL_TREE;
if (op_type == binary_op || op_type == ternary_op)
{
op1 = gimple_assign_rhs2 (stmt);
- if (!vect_is_simple_use (op1, vinfo, &dt[1]))
+ if (!vect_is_simple_use (op1, vinfo, &dt[1], &vectype2))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -6037,7 +6063,7 @@ vectorizable_operation (stmt_vec_info st
if (op_type == ternary_op)
{
op2 = gimple_assign_rhs3 (stmt);
- if (!vect_is_simple_use (op2, vinfo, &dt[2]))
+ if (!vect_is_simple_use (op2, vinfo, &dt[2], &vectype3))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -6062,10 +6088,20 @@ vectorizable_operation (stmt_vec_info st
gcc_assert (ncopies >= 1);
- /* Shifts are handled in vectorizable_shift (). */
- if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
- || code == RROTATE_EXPR)
- return false;
+ /* Reject attempts to combine mask types with nonmask types, e.g. if
+ we have an AND between a (nonmask) boolean loaded from memory and
+ a (mask) boolean result of a comparison.
+
+ TODO: We could easily fix these cases up using pattern statements. */
+ if (VECTOR_BOOLEAN_TYPE_P (vectype) != mask_op_p
+ || (vectype2 && VECTOR_BOOLEAN_TYPE_P (vectype2) != mask_op_p)
+ || (vectype3 && VECTOR_BOOLEAN_TYPE_P (vectype3) != mask_op_p))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "mixed mask and nonmask vector types\n");
+ return false;
+ }
/* Supportable by target? */
@@ -10410,14 +10446,15 @@ get_vectype_for_scalar_type (vec_info *v
Returns the mask type corresponding to a result of comparison
of vectors of specified SCALAR_TYPE as supported by target.
- NODE, if nonnull, is the SLP tree node that will use the returned
- vector type. */
+ If GROUP_SIZE is nonzero and we're performing BB vectorization,
+ make sure that the number of elements in the vector is no bigger
+ than GROUP_SIZE. */
tree
get_mask_type_for_scalar_type (vec_info *vinfo, tree scalar_type,
- slp_tree node)
+ unsigned int group_size)
{
- tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, node);
+ tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type, group_size);
if (!vectype)
return NULL;
@@ -11112,9 +11149,6 @@ vect_gen_while_not (gimple_seq *seq, tre
- Set *STMT_VECTYPE_OUT to:
- NULL_TREE if the statement doesn't need to be vectorized;
- - boolean_type_node if the statement is a boolean operation whose
- vector type can only be determined once all the other vector types
- are known; and
- the equivalent of STMT_VINFO_VECTYPE otherwise.
- Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
@@ -11171,11 +11205,22 @@ vect_get_vector_types_for_stmt (stmt_vec
tree scalar_type = NULL_TREE;
if (group_size == 0 && STMT_VINFO_VECTYPE (stmt_info))
{
- *stmt_vectype_out = vectype = STMT_VINFO_VECTYPE (stmt_info);
+ vectype = STMT_VINFO_VECTYPE (stmt_info);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"precomputed vectype: %T\n", vectype);
}
+ else if (vect_use_mask_type_p (stmt_info))
+ {
+ unsigned int precision = stmt_info->mask_precision;
+ scalar_type = build_nonstandard_integer_type (precision, 1);
+ vectype = get_mask_type_for_scalar_type (vinfo, scalar_type, group_size);
+ if (!vectype)
+ return opt_result::failure_at (stmt, "not vectorized: unsupported"
+ " data-type %T\n", scalar_type);
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
+ }
else
{
if (data_reference *dr = STMT_VINFO_DATA_REF (stmt_info))
@@ -11185,28 +11230,6 @@ vect_get_vector_types_for_stmt (stmt_vec
else
scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
- /* Pure bool ops don't participate in number-of-units computation.
- For comparisons use the types being compared. */
- if (!STMT_VINFO_DATA_REF (stmt_info)
- && VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type)
- && is_gimple_assign (stmt)
- && gimple_assign_rhs_code (stmt) != COND_EXPR)
- {
- *stmt_vectype_out = boolean_type_node;
-
- tree rhs1 = gimple_assign_rhs1 (stmt);
- if (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt)) == tcc_comparison
- && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1)))
- scalar_type = TREE_TYPE (rhs1);
- else
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "pure bool operation.\n");
- return opt_result::success ();
- }
- }
-
if (dump_enabled_p ())
{
if (group_size)
@@ -11224,18 +11247,15 @@ vect_get_vector_types_for_stmt (stmt_vec
" unsupported data-type %T\n",
scalar_type);
- if (!*stmt_vectype_out)
- *stmt_vectype_out = vectype;
-
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n", vectype);
}
+ *stmt_vectype_out = vectype;
/* Don't try to compute scalar types if the stmt produces a boolean
vector; use the existing vector type instead. */
tree nunits_vectype = vectype;
- if (!VECTOR_BOOLEAN_TYPE_P (vectype)
- && *stmt_vectype_out != boolean_type_node)
+ if (!VECTOR_BOOLEAN_TYPE_P (vectype))
{
/* The number of units is set according to the smallest scalar
type (or the largest vector size, but we only support one
@@ -11260,9 +11280,8 @@ vect_get_vector_types_for_stmt (stmt_vec
}
}
- gcc_assert (*stmt_vectype_out == boolean_type_node
- || multiple_p (TYPE_VECTOR_SUBPARTS (nunits_vectype),
- TYPE_VECTOR_SUBPARTS (*stmt_vectype_out)));
+ gcc_assert (multiple_p (TYPE_VECTOR_SUBPARTS (nunits_vectype),
+ TYPE_VECTOR_SUBPARTS (*stmt_vectype_out)));
if (dump_enabled_p ())
{
@@ -11274,82 +11293,3 @@ vect_get_vector_types_for_stmt (stmt_vec
*nunits_vectype_out = nunits_vectype;
return opt_result::success ();
}
-
-/* Try to determine the correct vector type for STMT_INFO, which is a
- statement that produces a scalar boolean result. Return the vector
- type on success, otherwise return NULL_TREE. NODE, if nonnull,
- is the SLP tree node that will use the returned vector type. */
-
-opt_tree
-vect_get_mask_type_for_stmt (stmt_vec_info stmt_info, slp_tree node)
-{
- vec_info *vinfo = stmt_info->vinfo;
- gimple *stmt = stmt_info->stmt;
- tree mask_type = NULL;
- tree vectype, scalar_type;
-
- if (is_gimple_assign (stmt)
- && TREE_CODE_CLASS (gimple_assign_rhs_code (stmt)) == tcc_comparison
- && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (stmt))))
- {
- scalar_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
- mask_type = get_mask_type_for_scalar_type (vinfo, scalar_type, node);
-
- if (!mask_type)
- return opt_tree::failure_at (stmt,
- "not vectorized: unsupported mask\n");
- }
- else
- {
- tree rhs;
- ssa_op_iter iter;
- enum vect_def_type dt;
-
- FOR_EACH_SSA_TREE_OPERAND (rhs, stmt, iter, SSA_OP_USE)
- {
- if (!vect_is_simple_use (rhs, stmt_info->vinfo, &dt, &vectype))
- return opt_tree::failure_at (stmt,
- "not vectorized:can't compute mask"
- " type for statement, %G", stmt);
-
- /* No vectype probably means external definition.
- Allow it in case there is another operand which
- allows to determine mask type. */
- if (!vectype)
- continue;
-
- if (!mask_type)
- mask_type = vectype;
- else if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
- TYPE_VECTOR_SUBPARTS (vectype)))
- return opt_tree::failure_at (stmt,
- "not vectorized: different sized mask"
- " types in statement, %T and %T\n",
- mask_type, vectype);
- else if (VECTOR_BOOLEAN_TYPE_P (mask_type)
- != VECTOR_BOOLEAN_TYPE_P (vectype))
- return opt_tree::failure_at (stmt,
- "not vectorized: mixed mask and "
- "nonmask vector types in statement, "
- "%T and %T\n",
- mask_type, vectype);
- }
-
- /* We may compare boolean value loaded as vector of integers.
- Fix mask_type in such case. */
- if (mask_type
- && !VECTOR_BOOLEAN_TYPE_P (mask_type)
- && gimple_code (stmt) == GIMPLE_ASSIGN
- && TREE_CODE_CLASS (gimple_assign_rhs_code (stmt)) == tcc_comparison)
- mask_type = truth_type_for (mask_type);
- }
-
- /* No mask_type should mean loop invariant predicate.
- This is probably a subject for optimization in if-conversion. */
- if (!mask_type)
- return opt_tree::failure_at (stmt,
- "not vectorized: can't compute mask type "
- "for statement: %G", stmt);
-
- return opt_tree::success (mask_type);
-}
Reference in New Issue View Git Blame Copy Permalink