This backport contains 5 patchs from gcc main stream tree. The commit id of these patchs list as following in the order of time. 8801ca5c28c3a9e9f36fa39a6a4455b48c8221fa 9ac1403ca2c65ba4f28cf051b5326617fa9298d1 7e99af4816cfad578094fcf08e2377f3ed76e201 ef8777c14ce8694f53eab7a88d24513cbf541ba4 dccbf1e2a6e544f71b4a5795f0c79015db019fc3 diff -Nurp a/gcc/testsuite/gcc.dg/vect/pr92677.c b/gcc/testsuite/gcc.dg/vect/pr92677.c --- a/gcc/testsuite/gcc.dg/vect/pr92677.c 1970-01-01 08:00:00.000000000 +0800 +++ b/gcc/testsuite/gcc.dg/vect/pr92677.c 2020-10-26 18:31:50.980000000 +0800 @@ -0,0 +1,26 @@ +/* { dg-do compile } */ +/* { dg-additional-options "-O3" } */ + +int a, c; +int *b; +long d; +double *e; + +void fn1() { + long f; + double g, h; + while (c) { + if (d) { + g = *e; + *(b + 4) = g; + } + if (f) { + h = *(e + 2); + *(b + 6) = h; + } + e += a; + b += 8; + c--; + d += 2; + } +} diff -Nurp a/gcc/testsuite/gcc.dg/vect/slp-46.c b/gcc/testsuite/gcc.dg/vect/slp-46.c --- a/gcc/testsuite/gcc.dg/vect/slp-46.c 1970-01-01 08:00:00.000000000 +0800 +++ b/gcc/testsuite/gcc.dg/vect/slp-46.c 2020-10-26 18:31:56.512000000 +0800 @@ -0,0 +1,96 @@ +/* { dg-require-effective-target vect_double } */ + +#include "tree-vect.h" + +double x[1024], y[1024]; + +void __attribute__((noipa)) foo() +{ + for (int i = 0; i < 512; ++i) + { + x[2*i] = y[i]; + x[2*i+1] = y[i]; + } +} + +void __attribute__((noipa)) bar() +{ + for (int i = 0; i < 512; ++i) + { + x[2*i] = y[2*i]; + x[2*i+1] = y[2*i]; + } +} + +void __attribute__((noipa)) baz() +{ + for (int i = 0; i < 512; ++i) + { + x[2*i] = y[511-i]; + x[2*i+1] = y[511-i]; + } +} + +void __attribute__((noipa)) boo() +{ + for (int i = 0; i < 512; ++i) + { + x[2*i] = y[2*(511-i)]; + x[2*i+1] = y[2*(511-i)]; + } +} + +int +main () +{ + check_vect (); + + for (int i = 0; i < 1024; ++i) + { + x[i] = 0; + y[i] = i; + __asm__ volatile (""); + } + + foo (); + for (int i = 0; i < 1024; ++i) + if (x[i] != y[i/2]) + abort (); + + for (int i = 0; i < 1024; ++i) + { + x[i] = 0; + __asm__ volatile (""); + } + + bar (); + for (int i = 0; i < 1024; ++i) + if (x[i] != y[2*(i/2)]) + abort (); + + for (int i = 0; i < 1024; ++i) + { + x[i] = 0; + __asm__ volatile (""); + } + + baz (); + for (int i = 0; i < 1024; ++i) + if (x[i] != y[511 - i/2]) + abort (); + + for (int i = 0; i < 1024; ++i) + { + x[i] = 0; + __asm__ volatile (""); + } + + boo (); + for (int i = 0; i < 1024; ++i) + if (x[i] != y[2*(511 - i/2)]) + abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" } } */ diff -Nurp a/gcc/testsuite/gcc.dg/vect/vect-cond-reduc-5.c b/gcc/testsuite/gcc.dg/vect/vect-cond-reduc-5.c --- a/gcc/testsuite/gcc.dg/vect/vect-cond-reduc-5.c 1970-01-01 08:00:00.000000000 +0800 +++ b/gcc/testsuite/gcc.dg/vect/vect-cond-reduc-5.c 2020-10-26 18:31:53.584000000 +0800 @@ -0,0 +1,36 @@ +#include "tree-vect.h" + +#define N 512 + +int a[N], b[N]; + +int __attribute__((noipa)) +foo (int aval, int bval) +{ + int i, res = 0; + for (i=0; inum, chrec_convert (type, evol, at_stmt), code, rhs1, at_stmt); - res = follow_ssa_edge - (loop, SSA_NAME_DEF_STMT (rhs0), halting_phi, &evol, limit); + res = follow_ssa_edge_expr + (loop, at_stmt, rhs0, halting_phi, &evol, limit); if (res == t_true) *evolution_of_loop = evol; else if (res == t_false) @@ -979,8 +979,8 @@ follow_ssa_edge_binary (struct loop *loo (loop->num, chrec_convert (type, *evolution_of_loop, at_stmt), code, rhs0, at_stmt); - res = follow_ssa_edge - (loop, SSA_NAME_DEF_STMT (rhs1), halting_phi, + res = follow_ssa_edge_expr + (loop, at_stmt, rhs1, halting_phi, evolution_of_loop, limit); if (res == t_true) ; @@ -1000,8 +1000,8 @@ follow_ssa_edge_binary (struct loop *loo (loop->num, chrec_convert (type, *evolution_of_loop, at_stmt), code, rhs1, at_stmt); - res = follow_ssa_edge - (loop, SSA_NAME_DEF_STMT (rhs0), halting_phi, + res = follow_ssa_edge_expr + (loop, at_stmt, rhs0, halting_phi, evolution_of_loop, limit); if (res == t_true) ; @@ -1018,8 +1018,8 @@ follow_ssa_edge_binary (struct loop *loo (loop->num, chrec_convert (type, *evolution_of_loop, at_stmt), code, rhs0, at_stmt); - res = follow_ssa_edge - (loop, SSA_NAME_DEF_STMT (rhs1), halting_phi, + res = follow_ssa_edge_expr + (loop, at_stmt, rhs1, halting_phi, evolution_of_loop, limit); if (res == t_true) ; @@ -1050,8 +1050,8 @@ follow_ssa_edge_binary (struct loop *loo *evolution_of_loop = add_to_evolution (loop->num, chrec_convert (type, *evolution_of_loop, at_stmt), MINUS_EXPR, rhs1, at_stmt); - res = follow_ssa_edge (loop, SSA_NAME_DEF_STMT (rhs0), halting_phi, - evolution_of_loop, limit); + res = follow_ssa_edge_expr (loop, at_stmt, rhs0, halting_phi, + evolution_of_loop, limit); if (res == t_true) ; else if (res == t_dont_know) @@ -1071,140 +1071,6 @@ follow_ssa_edge_binary (struct loop *loo return res; } -/* Follow the ssa edge into the expression EXPR. - Return true if the strongly connected component has been found. */ - -static t_bool -follow_ssa_edge_expr (struct loop *loop, gimple *at_stmt, tree expr, - gphi *halting_phi, tree *evolution_of_loop, - int limit) -{ - enum tree_code code = TREE_CODE (expr); - tree type = TREE_TYPE (expr), rhs0, rhs1; - t_bool res; - - /* The EXPR is one of the following cases: - - an SSA_NAME, - - an INTEGER_CST, - - a PLUS_EXPR, - - a POINTER_PLUS_EXPR, - - a MINUS_EXPR, - - an ASSERT_EXPR, - - other cases are not yet handled. */ - - switch (code) - { - CASE_CONVERT: - /* This assignment is under the form "a_1 = (cast) rhs. */ - res = follow_ssa_edge_expr (loop, at_stmt, TREE_OPERAND (expr, 0), - halting_phi, evolution_of_loop, limit); - *evolution_of_loop = chrec_convert (type, *evolution_of_loop, at_stmt); - break; - - case INTEGER_CST: - /* This assignment is under the form "a_1 = 7". */ - res = t_false; - break; - - case SSA_NAME: - /* This assignment is under the form: "a_1 = b_2". */ - res = follow_ssa_edge - (loop, SSA_NAME_DEF_STMT (expr), halting_phi, evolution_of_loop, limit); - break; - - case POINTER_PLUS_EXPR: - case PLUS_EXPR: - case MINUS_EXPR: - /* This case is under the form "rhs0 +- rhs1". */ - rhs0 = TREE_OPERAND (expr, 0); - rhs1 = TREE_OPERAND (expr, 1); - type = TREE_TYPE (rhs0); - STRIP_USELESS_TYPE_CONVERSION (rhs0); - STRIP_USELESS_TYPE_CONVERSION (rhs1); - res = follow_ssa_edge_binary (loop, at_stmt, type, rhs0, code, rhs1, - halting_phi, evolution_of_loop, limit); - break; - - case ADDR_EXPR: - /* Handle &MEM[ptr + CST] which is equivalent to POINTER_PLUS_EXPR. */ - if (TREE_CODE (TREE_OPERAND (expr, 0)) == MEM_REF) - { - expr = TREE_OPERAND (expr, 0); - rhs0 = TREE_OPERAND (expr, 0); - rhs1 = TREE_OPERAND (expr, 1); - type = TREE_TYPE (rhs0); - STRIP_USELESS_TYPE_CONVERSION (rhs0); - STRIP_USELESS_TYPE_CONVERSION (rhs1); - res = follow_ssa_edge_binary (loop, at_stmt, type, - rhs0, POINTER_PLUS_EXPR, rhs1, - halting_phi, evolution_of_loop, limit); - } - else - res = t_false; - break; - - case ASSERT_EXPR: - /* This assignment is of the form: "a_1 = ASSERT_EXPR " - It must be handled as a copy assignment of the form a_1 = a_2. */ - rhs0 = ASSERT_EXPR_VAR (expr); - if (TREE_CODE (rhs0) == SSA_NAME) - res = follow_ssa_edge (loop, SSA_NAME_DEF_STMT (rhs0), - halting_phi, evolution_of_loop, limit); - else - res = t_false; - break; - - default: - res = t_false; - break; - } - - return res; -} - -/* Follow the ssa edge into the right hand side of an assignment STMT. - Return true if the strongly connected component has been found. */ - -static t_bool -follow_ssa_edge_in_rhs (struct loop *loop, gimple *stmt, - gphi *halting_phi, tree *evolution_of_loop, - int limit) -{ - enum tree_code code = gimple_assign_rhs_code (stmt); - tree type = gimple_expr_type (stmt), rhs1, rhs2; - t_bool res; - - switch (code) - { - CASE_CONVERT: - /* This assignment is under the form "a_1 = (cast) rhs. */ - res = follow_ssa_edge_expr (loop, stmt, gimple_assign_rhs1 (stmt), - halting_phi, evolution_of_loop, limit); - *evolution_of_loop = chrec_convert (type, *evolution_of_loop, stmt); - break; - - case POINTER_PLUS_EXPR: - case PLUS_EXPR: - case MINUS_EXPR: - rhs1 = gimple_assign_rhs1 (stmt); - rhs2 = gimple_assign_rhs2 (stmt); - type = TREE_TYPE (rhs1); - res = follow_ssa_edge_binary (loop, stmt, type, rhs1, code, rhs2, - halting_phi, evolution_of_loop, limit); - break; - - default: - if (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS) - res = follow_ssa_edge_expr (loop, stmt, gimple_assign_rhs1 (stmt), - halting_phi, evolution_of_loop, limit); - else - res = t_false; - break; - } - - return res; -} - /* Checks whether the I-th argument of a PHI comes from a backedge. */ static bool @@ -1244,8 +1110,8 @@ follow_ssa_edge_in_condition_phi_branch if (TREE_CODE (branch) == SSA_NAME) { *evolution_of_branch = init_cond; - return follow_ssa_edge (loop, SSA_NAME_DEF_STMT (branch), halting_phi, - evolution_of_branch, limit); + return follow_ssa_edge_expr (loop, condition_phi, branch, halting_phi, + evolution_of_branch, limit); } /* This case occurs when one of the condition branches sets @@ -1352,65 +1218,158 @@ follow_ssa_edge_inner_loop_phi (struct l evolution_of_loop, limit); } -/* Follow an SSA edge from a loop-phi-node to itself, constructing a - path that is analyzed on the return walk. */ +/* Follow the ssa edge into the expression EXPR. + Return true if the strongly connected component has been found. */ static t_bool -follow_ssa_edge (struct loop *loop, gimple *def, gphi *halting_phi, - tree *evolution_of_loop, int limit) +follow_ssa_edge_expr (struct loop *loop, gimple *at_stmt, tree expr, + gphi *halting_phi, tree *evolution_of_loop, + int limit) { - struct loop *def_loop; + enum tree_code code; + tree type, rhs0, rhs1 = NULL_TREE; - if (gimple_nop_p (def)) - return t_false; + /* The EXPR is one of the following cases: + - an SSA_NAME, + - an INTEGER_CST, + - a PLUS_EXPR, + - a POINTER_PLUS_EXPR, + - a MINUS_EXPR, + - an ASSERT_EXPR, + - other cases are not yet handled. */ - /* Give up if the path is longer than the MAX that we allow. */ - if (limit > PARAM_VALUE (PARAM_SCEV_MAX_EXPR_COMPLEXITY)) - return t_dont_know; - - def_loop = loop_containing_stmt (def); - - switch (gimple_code (def)) - { - case GIMPLE_PHI: - if (!loop_phi_node_p (def)) - /* DEF is a condition-phi-node. Follow the branches, and - record their evolutions. Finally, merge the collected - information and set the approximation to the main - variable. */ - return follow_ssa_edge_in_condition_phi - (loop, as_a (def), halting_phi, evolution_of_loop, - limit); - - /* When the analyzed phi is the halting_phi, the - depth-first search is over: we have found a path from - the halting_phi to itself in the loop. */ - if (def == halting_phi) - return t_true; + /* For SSA_NAME look at the definition statement, handling + PHI nodes and otherwise expand appropriately for the expression + handling below. */ + if (TREE_CODE (expr) == SSA_NAME) + { + gimple *def = SSA_NAME_DEF_STMT (expr); - /* Otherwise, the evolution of the HALTING_PHI depends - on the evolution of another loop-phi-node, i.e. the - evolution function is a higher degree polynomial. */ - if (def_loop == loop) + if (gimple_nop_p (def)) return t_false; - /* Inner loop. */ - if (flow_loop_nested_p (loop, def_loop)) - return follow_ssa_edge_inner_loop_phi - (loop, as_a (def), halting_phi, evolution_of_loop, - limit + 1); + /* Give up if the path is longer than the MAX that we allow. */ + if (limit > PARAM_VALUE (PARAM_SCEV_MAX_EXPR_COMPLEXITY)) + return t_dont_know; - /* Outer loop. */ - return t_false; + if (gphi *phi = dyn_cast (def)) + { + if (!loop_phi_node_p (phi)) + /* DEF is a condition-phi-node. Follow the branches, and + record their evolutions. Finally, merge the collected + information and set the approximation to the main + variable. */ + return follow_ssa_edge_in_condition_phi + (loop, phi, halting_phi, evolution_of_loop, limit); + + /* When the analyzed phi is the halting_phi, the + depth-first search is over: we have found a path from + the halting_phi to itself in the loop. */ + if (phi == halting_phi) + return t_true; + + /* Otherwise, the evolution of the HALTING_PHI depends + on the evolution of another loop-phi-node, i.e. the + evolution function is a higher degree polynomial. */ + class loop *def_loop = loop_containing_stmt (def); + if (def_loop == loop) + return t_false; + + /* Inner loop. */ + if (flow_loop_nested_p (loop, def_loop)) + return follow_ssa_edge_inner_loop_phi + (loop, phi, halting_phi, evolution_of_loop, + limit + 1); - case GIMPLE_ASSIGN: - return follow_ssa_edge_in_rhs (loop, def, halting_phi, - evolution_of_loop, limit); + /* Outer loop. */ + return t_false; + } - default: /* At this level of abstraction, the program is just a set of GIMPLE_ASSIGNs and PHI_NODEs. In principle there is no - other node to be handled. */ + other def to be handled. */ + if (!is_gimple_assign (def)) + return t_false; + + code = gimple_assign_rhs_code (def); + switch (get_gimple_rhs_class (code)) + { + case GIMPLE_BINARY_RHS: + rhs0 = gimple_assign_rhs1 (def); + rhs1 = gimple_assign_rhs2 (def); + break; + case GIMPLE_UNARY_RHS: + case GIMPLE_SINGLE_RHS: + rhs0 = gimple_assign_rhs1 (def); + break; + default: + return t_false; + } + type = TREE_TYPE (gimple_assign_lhs (def)); + at_stmt = def; + } + else + { + code = TREE_CODE (expr); + type = TREE_TYPE (expr); + switch (code) + { + CASE_CONVERT: + rhs0 = TREE_OPERAND (expr, 0); + break; + case POINTER_PLUS_EXPR: + case PLUS_EXPR: + case MINUS_EXPR: + rhs0 = TREE_OPERAND (expr, 0); + rhs1 = TREE_OPERAND (expr, 1); + break; + default: + rhs0 = expr; + } + } + + switch (code) + { + CASE_CONVERT: + { + /* This assignment is under the form "a_1 = (cast) rhs. */ + t_bool res = follow_ssa_edge_expr (loop, at_stmt, rhs0, halting_phi, + evolution_of_loop, limit); + *evolution_of_loop = chrec_convert (type, *evolution_of_loop, at_stmt); + return res; + } + + case INTEGER_CST: + /* This assignment is under the form "a_1 = 7". */ + return t_false; + + case ADDR_EXPR: + { + /* Handle &MEM[ptr + CST] which is equivalent to POINTER_PLUS_EXPR. */ + if (TREE_CODE (TREE_OPERAND (rhs0, 0)) != MEM_REF) + return t_false; + tree mem = TREE_OPERAND (rhs0, 0); + rhs0 = TREE_OPERAND (mem, 0); + rhs1 = TREE_OPERAND (mem, 1); + code = POINTER_PLUS_EXPR; + } + /* Fallthru. */ + case POINTER_PLUS_EXPR: + case PLUS_EXPR: + case MINUS_EXPR: + /* This case is under the form "rhs0 +- rhs1". */ + STRIP_USELESS_TYPE_CONVERSION (rhs0); + STRIP_USELESS_TYPE_CONVERSION (rhs1); + return follow_ssa_edge_binary (loop, at_stmt, type, rhs0, code, rhs1, + halting_phi, evolution_of_loop, limit); + + case ASSERT_EXPR: + /* This assignment is of the form: "a_1 = ASSERT_EXPR " + It must be handled as a copy assignment of the form a_1 = a_2. */ + return follow_ssa_edge_expr (loop, at_stmt, ASSERT_EXPR_VAR (rhs0), + halting_phi, evolution_of_loop, limit); + + default: return t_false; } } @@ -1504,7 +1463,6 @@ analyze_evolution_in_loop (gphi *loop_ph for (i = 0; i < n; i++) { tree arg = PHI_ARG_DEF (loop_phi_node, i); - gimple *ssa_chain; tree ev_fn; t_bool res; @@ -1517,11 +1475,10 @@ analyze_evolution_in_loop (gphi *loop_ph { bool val = false; - ssa_chain = SSA_NAME_DEF_STMT (arg); - /* Pass in the initial condition to the follow edge function. */ ev_fn = init_cond; - res = follow_ssa_edge (loop, ssa_chain, loop_phi_node, &ev_fn, 0); + res = follow_ssa_edge_expr (loop, loop_phi_node, arg, + loop_phi_node, &ev_fn, 0); /* If ev_fn has no evolution in the inner loop, and the init_cond is not equal to ev_fn, then we have an diff -Nurp a/gcc/tree-ssa-sccvn.c b/gcc/tree-ssa-sccvn.c --- a/gcc/tree-ssa-sccvn.c 2020-10-26 18:28:58.736000000 +0800 +++ b/gcc/tree-ssa-sccvn.c 2020-10-26 18:31:45.768000000 +0800 @@ -2456,7 +2456,8 @@ vn_reference_lookup_3 (ao_ref *ref, tree (vuse, vr->set, vr->type, vr->operands, val); } /* For now handle clearing memory with partial defs. */ - else if (integer_zerop (gimple_call_arg (def_stmt, 1)) + else if (known_eq (ref->size, maxsize) + && integer_zerop (gimple_call_arg (def_stmt, 1)) && tree_to_poly_int64 (len).is_constant (&leni) && offset.is_constant (&offseti) && offset2.is_constant (&offset2i) @@ -2494,7 +2495,8 @@ vn_reference_lookup_3 (ao_ref *ref, tree return vn_reference_lookup_or_insert_for_pieces (vuse, vr->set, vr->type, vr->operands, val); } - else if (maxsize.is_constant (&maxsizei) + else if (known_eq (ref->size, maxsize) + && maxsize.is_constant (&maxsizei) && maxsizei % BITS_PER_UNIT == 0 && offset.is_constant (&offseti) && offseti % BITS_PER_UNIT == 0 diff -Nurp a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c --- a/gcc/tree-vect-data-refs.c 2020-10-26 18:28:58.792000000 +0800 +++ b/gcc/tree-vect-data-refs.c 2020-10-26 18:31:56.512000000 +0800 @@ -1045,7 +1045,7 @@ vect_compute_data_ref_alignment (dr_vec_ if (tree_int_cst_sgn (drb->step) < 0) /* PLUS because STEP is negative. */ misalignment += ((TYPE_VECTOR_SUBPARTS (vectype) - 1) - * TREE_INT_CST_LOW (drb->step)); + * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)))); unsigned int const_misalignment; if (!known_misalignment (misalignment, vect_align_c, &const_misalignment)) diff -Nurp a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c --- a/gcc/tree-vect-loop.c 2020-10-26 18:28:58.728000000 +0800 +++ b/gcc/tree-vect-loop.c 2020-10-26 18:31:53.584000000 +0800 @@ -1850,7 +1850,10 @@ vect_dissolve_slp_only_groups (loop_vec_ DR_GROUP_FIRST_ELEMENT (vinfo) = vinfo; DR_GROUP_NEXT_ELEMENT (vinfo) = NULL; DR_GROUP_SIZE (vinfo) = 1; - DR_GROUP_GAP (vinfo) = group_size - 1; + if (STMT_VINFO_STRIDED_P (first_element)) + DR_GROUP_GAP (vinfo) = 0; + else + DR_GROUP_GAP (vinfo) = group_size - 1; vinfo = next; } } @@ -4516,18 +4519,26 @@ vect_create_epilog_for_reduction (stmt_v zeroes. */ if (STMT_VINFO_REDUC_TYPE (reduc_info) == COND_REDUCTION) { + auto_vec, 2> ccompares; stmt_vec_info cond_info = STMT_VINFO_REDUC_DEF (reduc_info); cond_info = vect_stmt_to_vectorize (cond_info); - while (gimple_assign_rhs_code (cond_info->stmt) != COND_EXPR) + while (cond_info != reduc_info) { + if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR) + { + gimple *vec_stmt = STMT_VINFO_VEC_STMT (cond_info)->stmt; + gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR); + ccompares.safe_push + (std::make_pair (unshare_expr (gimple_assign_rhs1 (vec_stmt)), + STMT_VINFO_REDUC_IDX (cond_info) == 2)); + } cond_info = loop_vinfo->lookup_def (gimple_op (cond_info->stmt, 1 + STMT_VINFO_REDUC_IDX (cond_info))); cond_info = vect_stmt_to_vectorize (cond_info); } - gimple *vec_stmt = STMT_VINFO_VEC_STMT (cond_info)->stmt; - gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR); + gcc_assert (ccompares.length () != 0); tree indx_before_incr, indx_after_incr; poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype); @@ -4569,37 +4580,35 @@ vect_create_epilog_for_reduction (stmt_v add_phi_arg (as_a (new_phi), vec_zero, loop_preheader_edge (loop), UNKNOWN_LOCATION); - /* Now take the condition from the loops original cond_expr - (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for + /* Now take the condition from the loops original cond_exprs + and produce a new cond_exprs (INDEX_COND_EXPR) which for every match uses values from the induction variable (INDEX_BEFORE_INCR) otherwise uses values from the phi node (NEW_PHI_TREE). Finally, we update the phi (NEW_PHI_TREE) to take the value of the new cond_expr (INDEX_COND_EXPR). */ - - /* Duplicate the condition from vec_stmt. */ - tree ccompare = unshare_expr (gimple_assign_rhs1 (vec_stmt)); - - /* Create a conditional, where the condition is taken from vec_stmt - (CCOMPARE). The then and else values mirror the main VEC_COND_EXPR: - the reduction phi corresponds to NEW_PHI_TREE and the new values - correspond to INDEX_BEFORE_INCR. */ - gcc_assert (STMT_VINFO_REDUC_IDX (cond_info) >= 1); - tree index_cond_expr; - if (STMT_VINFO_REDUC_IDX (cond_info) == 2) - index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type, - ccompare, indx_before_incr, new_phi_tree); - else - index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type, - ccompare, new_phi_tree, indx_before_incr); - induction_index = make_ssa_name (cr_index_vector_type); - gimple *index_condition = gimple_build_assign (induction_index, - index_cond_expr); - gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT); - stmt_vec_info index_vec_info = loop_vinfo->add_stmt (index_condition); + gimple_seq stmts = NULL; + for (int i = ccompares.length () - 1; i != -1; --i) + { + tree ccompare = ccompares[i].first; + if (ccompares[i].second) + new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR, + cr_index_vector_type, + ccompare, + indx_before_incr, new_phi_tree); + else + new_phi_tree = gimple_build (&stmts, VEC_COND_EXPR, + cr_index_vector_type, + ccompare, + new_phi_tree, indx_before_incr); + } + gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT); + stmt_vec_info index_vec_info + = loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (new_phi_tree)); STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type; /* Update the phi with the vec cond. */ + induction_index = new_phi_tree; add_phi_arg (as_a (new_phi), induction_index, loop_latch_edge (loop), UNKNOWN_LOCATION); }