packages/gcc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
gcc/0050-Port-IPA-prefetch-to-GCC-12.patch
郑晨卉 3ee8545fc2 [Sync] Sync patch from openeuler/gcc
2024-04-11 19:37:37 +08:00

2072 lines
62 KiB
Diff
Raw Blame History

From 7ee50ce44c652e21ca8ad33dc4e175f02b51b072 Mon Sep 17 00:00:00 2001
From: Diachkov Ilia <diachkov.ilia1@huawei-partners.com>
Date: Fri, 8 Mar 2024 06:50:39 +0800
Subject: [PATCH 18/18] Port IPA prefetch to GCC 12
---
gcc/Makefile.in | 1 +
gcc/cgraph.cc | 1 +
gcc/cgraph.h | 2 +
gcc/common.opt | 8 +
gcc/ipa-devirt.cc | 54 +-
gcc/ipa-prefetch.cc | 1819 +++++++++++++++++++++++++++++++++++++++++++
gcc/ipa-sra.cc | 8 +
gcc/params.opt | 8 +
gcc/passes.def | 1 +
gcc/timevar.def | 1 +
gcc/tree-pass.h | 1 +
11 files changed, 1902 insertions(+), 2 deletions(-)
create mode 100644 gcc/ipa-prefetch.cc
diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index 876000bda..10544e4a9 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1468,6 +1468,7 @@ OBJS = \
ipa-modref.o \
ipa-modref-tree.o \
ipa-predicate.o \
+ ipa-prefetch.o \
ipa-profile.o \
ipa-prop.o \
ipa-param-manipulation.o \
diff --git a/gcc/cgraph.cc b/gcc/cgraph.cc
index 3734c85db..7d738b891 100644
--- a/gcc/cgraph.cc
+++ b/gcc/cgraph.cc
@@ -998,6 +998,7 @@ cgraph_node::create_indirect_edge (gcall *call_stmt, int ecf_flags,
edge->indirect_info = cgraph_allocate_init_indirect_info ();
edge->indirect_info->ecf_flags = ecf_flags;
edge->indirect_info->vptr_changed = true;
+ edge->indirect_info->targets = NULL;
/* Record polymorphic call info. */
if (!cloning_p
diff --git a/gcc/cgraph.h b/gcc/cgraph.h
index d96690326..b84ff2f98 100644
--- a/gcc/cgraph.h
+++ b/gcc/cgraph.h
@@ -1659,6 +1659,8 @@ public:
int param_index;
/* ECF flags determined from the caller. */
int ecf_flags;
+ /* Vector of potential call targets determined by analysis. */
+ vec<cgraph_node *, va_gc_atomic> *targets;
/* Number of speculative call targets, it's less than GCOV_TOPN_VALUES. */
unsigned num_speculative_call_targets : 16;
diff --git a/gcc/common.opt b/gcc/common.opt
index 1eb62ada5..e65a06af9 100644
--- a/gcc/common.opt
+++ b/gcc/common.opt
@@ -1328,6 +1328,10 @@ fdevirtualize
Common Var(flag_devirtualize) Optimization
Try to convert virtual calls to direct ones.
+fipa-ic
+Common Var(flag_ipa_ic) Optimization Init(0)
+Perform interprocedural analysis of indirect calls.
+
ficp
Common Var(flag_icp) Optimization Init(0)
Try to promote indirect calls to direct ones.
@@ -2367,6 +2371,10 @@ fprefetch-loop-arrays
Common Var(flag_prefetch_loop_arrays) Init(-1) Optimization
Generate prefetch instructions, if available, for arrays in loops.
+fipa-prefetch
+Common Var(flag_ipa_prefetch) Init(0) Optimization
+Generate prefetch instructions, if available, using IPA info.
+
fprofile
Common Var(profile_flag)
Enable basic program profiling code.
diff --git a/gcc/ipa-devirt.cc b/gcc/ipa-devirt.cc
index 318535d06..dd3562d56 100644
--- a/gcc/ipa-devirt.cc
+++ b/gcc/ipa-devirt.cc
@@ -5758,6 +5758,54 @@ merge_fs_map_for_ftype_aliases ()
}
}
+/* Save results of indirect call analysis for the next passes. */
+
+static void
+save_analysis_results ()
+{
+ if (dump_file)
+ fprintf (dump_file, "\n\nSave results of indirect call analysis.\n");
+
+ struct cgraph_node *n;
+ FOR_EACH_FUNCTION (n)
+ {
+ cgraph_edge *e, *next;
+ for (e = n->indirect_calls; e; e = next)
+ {
+ next = e->next_callee;
+ if (e->indirect_info->polymorphic)
+ continue;
+ gcall *stmt = e->call_stmt;
+ gcc_assert (stmt != NULL);
+ tree call_fn = gimple_call_fn (stmt);
+ tree call_fn_ty = TREE_TYPE (call_fn);
+ if (!POINTER_TYPE_P (call_fn_ty))
+ continue;
+
+ tree ctype = TYPE_CANONICAL (TREE_TYPE (call_fn_ty));
+ unsigned ctype_uid = ctype ? TYPE_UID (ctype) : 0;
+ if (!ctype_uid || unsafe_types->count (ctype_uid)
+ || !fs_map->count (ctype_uid))
+ continue;
+ /* TODO: cleanup noninterposable aliases. */
+ decl_set *decls = (*fs_map)[ctype_uid];
+ if (dump_file)
+ {
+ fprintf (dump_file, "For call ");
+ print_gimple_stmt (dump_file, stmt, 0);
+ }
+ vec_alloc (e->indirect_info->targets, decls->size ());
+ for (decl_set::const_iterator it = decls->begin ();
+ it != decls->end (); it++)
+ {
+ struct cgraph_node *target = cgraph_node::get (*it);
+ /* TODO: maybe discard some targets. */
+ e->indirect_info->targets->quick_push (target);
+ }
+ }
+ }
+}
+
/* Dump function types with set of functions corresponding to it. */
static void
@@ -5822,6 +5870,8 @@ collect_function_signatures ()
}
}
merge_fs_map_for_ftype_aliases ();
+ if (flag_ipa_ic)
+ save_analysis_results ();
if (dump_file)
dump_function_signature_sets ();
}
@@ -6217,7 +6267,7 @@ ipa_icp (void)
optimize indirect calls. */
collect_function_type_aliases ();
collect_function_signatures ();
- bool optimized = optimize_indirect_calls ();
+ bool optimized = flag_icp ? optimize_indirect_calls () : false;
remove_type_alias_map (ta_map);
remove_type_alias_map (fta_map);
@@ -6264,7 +6314,7 @@ public:
/* opt_pass methods: */
virtual bool gate (function *)
{
- return (optimize && flag_icp && !seen_error ()
+ return (optimize && (flag_icp || flag_ipa_ic) && !seen_error ()
&& (in_lto_p || flag_whole_program));
}
diff --git a/gcc/ipa-prefetch.cc b/gcc/ipa-prefetch.cc
new file mode 100644
index 000000000..aeea51105
--- /dev/null
+++ b/gcc/ipa-prefetch.cc
@@ -0,0 +1,1819 @@
+/* IPA prefetch optimizations.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ Contributed by Ilia Diachkov.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+/* IPA prefetch is an interprocedural pass that detects cases of indirect
+ memory access potentially in loops and inserts prefetch instructions
+ to optimize cache usage during these indirect memory accesses. */
+
+#include "config.h"
+#define INCLUDE_SET
+#define INCLUDE_MAP
+#include "system.h"
+#include "coretypes.h"
+#include "target.h"
+#include "tm.h"
+#include "tree.h"
+#include "tree-pass.h"
+#include "cgraph.h"
+#include "diagnostic-core.h"
+#include "function.h"
+#include "basic-block.h"
+#include "gimple.h"
+#include "vec.h"
+#include "tree-pretty-print.h"
+#include "gimple-pretty-print.h"
+#include "gimple-iterator.h"
+#include "gimple-walk.h"
+#include "cfg.h"
+#include "cfghooks.h"
+#include "ssa.h"
+#include "tree-dfa.h"
+#include "fold-const.h"
+#include "tree-inline.h"
+#include "stor-layout.h"
+#include "tree-into-ssa.h"
+#include "tree-cfg.h"
+#include "alloc-pool.h"
+#include "symbol-summary.h"
+#include "ipa-prop.h"
+#include "tree-eh.h"
+#include "bitmap.h"
+#include "cfgloop.h"
+#include "langhooks.h"
+#include "ipa-param-manipulation.h"
+#include "ipa-fnsummary.h"
+#include "tree-ssa-loop.h"
+#include "tree-ssa-loop-ivopts.h"
+#include "gimple-fold.h"
+#include "gimplify.h"
+
+namespace {
+
+/* Call graph analysis. */
+
+typedef std::set<cgraph_edge *> edge_set;
+typedef std::set<cgraph_node *> node_set;
+typedef std::map<cgraph_node *, edge_set *> node_to_iedge_map;
+typedef std::map<cgraph_node *, node_set *> node_to_node_map;
+typedef std::map<cgraph_edge *, double> edge_in_loop;
+typedef std::map<cgraph_node *, double> node_in_loop;
+
+static edge_in_loop *el_map = NULL;
+static node_in_loop *nl_map = NULL;
+static node_to_iedge_map *icn_map = NULL;
+/* Contains nodes which reachable from a given node. */
+static node_to_node_map *nn_map = NULL;
+
+static bool
+can_be_optimized (cgraph_node *n)
+{
+ /* TODO: maybe check also inlined_to. */
+ return opt_for_fn (n->decl, flag_ipa_prefetch) && n->has_gimple_body_p ();
+}
+
+static void
+analyze_cgraph_edge (cgraph_edge *e)
+{
+ gcall *stmt = e->call_stmt;
+ gcc_checking_assert (e && stmt);
+ basic_block bb = gimple_bb (stmt);
+ gcc_checking_assert (bb);
+ /* TODO: add the same check for indirect calls. */
+ if (e->callee && !can_be_optimized (e->callee))
+ return;
+
+ if (dump_file)
+ {
+ if (e->callee)
+ fprintf (dump_file, "\t%*s%s %s%*s ", 1, "",
+ e->callee->dump_name (), !e->inline_failed ? "inlined" :
+ cgraph_inline_failed_string (e->inline_failed), 1, "");
+ else
+ fprintf (dump_file, "\t%*s%s %s%*s ", 1, "", "(indirect)",
+ "n/a", 1, "");
+ fprintf (dump_file, "freq:%4.2f", e->sreal_frequency ().to_double ());
+
+ if (e->callee && cross_module_call_p (e))
+ fprintf (dump_file, " cross module");
+
+ class ipa_call_summary *es = ipa_call_summaries->get (e);
+ if (es)
+ fprintf (dump_file, " loop depth:%2i size:%2i time: %2i",
+ es->loop_depth, es->call_stmt_size, es->call_stmt_time);
+
+ fprintf (dump_file, "\n");
+ }
+ if (e->indirect_info && dump_file)
+ {
+ fprintf (dump_file, "II: %p\n", (void *) e->indirect_info->targets);
+ unsigned i = 0;
+ cgraph_node *n;
+ if (e->indirect_info->targets)
+ for (i = 0; e->indirect_info->targets->iterate (i, &n); ++i)
+ fprintf (dump_file, "\t%s\n", n->dump_name ());
+ }
+
+ if (bb_loop_depth (bb) == 0)
+ return;
+
+ if (dump_file)
+ {
+ if (e->callee)
+ fprintf (dump_file, "\tCall in loop (%d): ", bb_loop_depth (bb));
+ else
+ fprintf (dump_file, "\tICall in loop (%d): ", bb_loop_depth (bb));
+ print_gimple_stmt (dump_file, stmt, 0);
+ }
+ (*el_map)[e] = e->sreal_frequency ().to_double ();
+}
+
+/* Walk optimizible cgraph nodes and collect info for edges. */
+
+static void
+analyse_cgraph ()
+{
+ cgraph_node *n;
+ cgraph_edge *e;
+ FOR_EACH_DEFINED_FUNCTION (n)
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "\n\nProcesing function %s\n", n->dump_name ());
+ print_generic_expr (dump_file, n->decl);
+ fprintf (dump_file, "\n");
+ }
+ if (!can_be_optimized (n))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Skip the function\n");
+ continue;
+ }
+
+ /* TODO: maybe remove loop info here. */
+ push_cfun (DECL_STRUCT_FUNCTION (n->decl));
+ calculate_dominance_info (CDI_DOMINATORS);
+ loop_optimizer_init (LOOPS_NORMAL);
+
+ for (e = n->callees; e; e = e->next_callee)
+ analyze_cgraph_edge (e);
+ for (e = n->indirect_calls; e; e = e->next_callee)
+ analyze_cgraph_edge (e);
+
+ free_dominance_info (CDI_DOMINATORS);
+ loop_optimizer_finalize ();
+
+ pop_cfun ();
+ }
+}
+
+/* Save indirect call info to node:icall_target map. */
+
+static void
+prepare_indirect_call_info ()
+{
+ cgraph_node *n, *n2;
+ cgraph_edge *e;
+ FOR_EACH_DEFINED_FUNCTION (n)
+ for (e = n->indirect_calls; e; e = e->next_callee)
+ {
+ if (!e->indirect_info->targets)
+ continue;
+ for (unsigned i = 0; e->indirect_info->targets->iterate (i, &n2); ++i)
+ {
+ if (icn_map->count (n2) == 0)
+ (*icn_map)[n2] = new edge_set;
+ (*icn_map)[n2]->insert (e);
+ }
+ }
+}
+
+static void
+collect_nn_info (struct cgraph_edge *e, struct cgraph_node *n)
+{
+ struct cgraph_node *n2 = e->caller;
+ if (nn_map->count (n2) == 0)
+ (*nn_map)[n2] = new node_set;
+ (*nn_map)[n2]->insert (n);
+ if (nn_map->count (n) != 0)
+ {
+ node_set *set = (*nn_map)[n];
+ for (node_set::const_iterator it = set->begin ();
+ it != set->end (); it++)
+ (*nn_map)[n2]->insert (*it);
+ }
+}
+
+static bool
+check_loop_info_for_cgraph_edge (struct cgraph_edge *e, struct cgraph_node *n,
+ bool &all_in_loop, double &rate)
+{
+ collect_nn_info (e, n);
+ if (el_map->count (e) == 0)
+ {
+ if (dump_file)
+ fprintf (dump_file, "not all: %s->%s\n",
+ e->caller->dump_name (), n->dump_name ());
+ all_in_loop = false;
+ return false;
+ }
+ rate += (*el_map)[e];
+ return true;
+}
+
+static bool
+update_loop_info_for_cgraph_node (struct cgraph_node *n)
+{
+ bool changed = false, all_in_loop = true;
+ double rate = 0.0;
+ struct cgraph_edge *e;
+
+ /* Iterate all direct callers. */
+ if (n->callers)
+ for (e = n->callers; e; e = e->next_caller)
+ if (!check_loop_info_for_cgraph_edge (e, n, all_in_loop, rate))
+ break;
+
+ /* Iterate all possible indirect callers. */
+ edge_set *set = (*icn_map)[n];
+ if (set)
+ for (edge_set::const_iterator it = set->begin (); it != set->end (); it++)
+ if (!check_loop_info_for_cgraph_edge (*it, n, all_in_loop, rate))
+ break;
+
+ /* The node had 0 loop count but the rate is > 0,
+ so something is changed. */
+ if (dump_file)
+ fprintf (dump_file, "%s: all=%d, nl->c=%lu, r=%4.2f\n", n->dump_name (),
+ all_in_loop, nl_map->count (n), rate);
+
+ if (all_in_loop && nl_map->count (n) == 0 && rate > 0.0)
+ {
+ if (dump_file)
+ fprintf (dump_file, "%s: new rate %4.2f\n", n->dump_name (), rate);
+ changed = true;
+ }
+ if (all_in_loop)
+ {
+ (*nl_map)[n] = nl_map->count (n) ? (*nl_map)[n] + rate : rate;
+ for (e = n->callees; e; e = e->next_callee)
+ (*el_map)[e] = el_map->count (e) ? (*el_map)[e] + rate : rate;
+ for (e = n->indirect_calls; e; e = e->next_callee)
+ {
+ (*el_map)[e] = el_map->count (e) ? (*el_map)[e] + rate : rate;
+ if (dump_file)
+ fprintf (dump_file, "%s: reset indirect e=%p to %4.2f\n",
+ n->dump_name (), (void *) e, (*el_map)[e]);
+ }
+ }
+ return changed;
+}
+
+/* Propagate in_loop info over the call graph. */
+
+static void
+propagate_loop_info_in_cgraph ()
+{
+ struct cgraph_node *n;
+ bool changed;
+ unsigned iteration = 0;
+ do
+ {
+ changed = false;
+ if (dump_file)
+ fprintf (dump_file, "\nIteration %u\n", iteration++);
+ FOR_EACH_DEFINED_FUNCTION (n)
+ {
+ if (!n->callers && !(*icn_map)[n])
+ continue;
+ if (update_loop_info_for_cgraph_node (n))
+ changed = true;
+ }
+ } while (changed);
+
+ if (dump_file)
+ {
+ fprintf (dump_file, "\nList of nodes in loops:\n");
+ FOR_EACH_DEFINED_FUNCTION (n)
+ if (nl_map->count (n) != 0)
+ fprintf (dump_file, "%s: %4.2f\n", n->dump_name (), (*nl_map)[n]);
+ fprintf (dump_file, "\nList of callable nodes:\n");
+ FOR_EACH_DEFINED_FUNCTION (n)
+ if (nn_map->count (n) != 0)
+ {
+ node_set *set = (*nn_map)[n];
+ fprintf (dump_file, "%s: ", n->dump_name ());
+ for (node_set::const_iterator it = set->begin ();
+ it != set->end (); it++)
+ fprintf (dump_file, "%s ", (*it)->dump_name ());
+ fprintf (dump_file, "\n");
+ }
+ }
+}
+
+/* Analysis of memory references. */
+
+typedef enum
+{
+ MR_NONE,
+ MR_SIMPLE,
+ MR_POLYNOMIAL,
+ MR_INDIRECT,
+ MR_UNSUPPORTED
+} mr_type;
+const char *mr_type_str[] =
+ {"none", "simple", "poly", "indirect", "unsuppoted"};
+
+struct memref_type;
+typedef std::set<memref_type *> memref_set;
+
+static unsigned max_mr_id = 0;
+typedef struct memref_type
+{
+ unsigned mr_id = 0;
+ mr_type type = MR_NONE;
+ tree mem = NULL_TREE;
+ tree base = NULL_TREE;
+ tree offset = NULL_TREE;
+ vec<gimple *, va_heap, vl_ptr> stmts = vNULL;
+ memref_set used_mrs;
+ bool is_store = false;
+ bool is_incr = false;
+ tree step = NULL_TREE;
+} memref_t;
+
+typedef std::map<tree, memref_t *> tree_memref_map;
+typedef std::map<function *, vec<memref_t *> > function_mrs_map;
+typedef std::map<function *, memref_set *> funct_mrs_map;
+typedef std::map<memref_t *, memref_t *> memref_map;
+typedef std::map<memref_t *, tree> memref_tree_map;
+
+typedef std::set<gimple *> stmt_set;
+typedef std::map<tree, tree> tree_map;
+
+tree_memref_map *tm_map;
+funct_mrs_map *fmrs_map;
+funct_mrs_map *optimize_mrs_map;
+memref_map *mr_candidate_map;
+tree_map *decl_map;
+
+static void analyse_mem_ref (gimple *stmt, tree mem, memref_t* mr);
+
+static memref_t*
+get_memref (gimple *stmt, tree mem, bool is_store)
+{
+ if (tm_map->count (mem))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Found mr %d for %p.\n",
+ (*tm_map)[mem]->mr_id, (void *) mem);
+ return (*tm_map)[mem];
+ }
+
+ memref_t *mr = new memref_t;
+ mr->mr_id = ++max_mr_id;
+ mr->is_store = is_store;
+ mr->mem = mem;
+ (*tm_map)[mem] = mr;
+ if (dump_file)
+ fprintf (dump_file, "Create mr %d for %p.\n",
+ mr->mr_id, (void *) mem);
+ analyse_mem_ref (stmt, mem, mr);
+ return mr;
+}
+
+static void
+print_mrs_ids (memref_set &mrs, const char *start)
+{
+ if (start)
+ fprintf (dump_file, "%s", start);
+ for (memref_set::const_iterator it = mrs.begin (); it != mrs.end (); it++)
+ fprintf (dump_file, "%d ", (*it)->mr_id);
+ fprintf (dump_file, "\n");
+}
+
+static void
+print_memref (memref_t *mr)
+{
+ fprintf (dump_file, "MR (%d) type: %s (%s) mem: ", mr->mr_id,
+ mr_type_str[mr->type], mr->is_store ? "st" : "ld");
+ print_generic_expr (dump_file, mr->mem);
+ fprintf (dump_file, "\nbase: ");
+ if (mr->base)
+ print_generic_expr (dump_file, mr->base);
+ else
+ fprintf (dump_file, "null");
+ fprintf (dump_file, "\noffset: ");
+ if (mr->offset)
+ print_generic_expr (dump_file, mr->offset);
+ else
+ fprintf (dump_file, "null");
+ fprintf (dump_file, "\nstmts:\n");
+ for (unsigned int i = 0; i < mr->stmts.length (); i++)
+ print_gimple_stmt (dump_file, mr->stmts[i], 0);
+ print_mrs_ids (mr->used_mrs, "\tused memrefs: ");
+ if (mr->is_incr)
+ {
+ fprintf (dump_file, "\tis incremental with step: ");
+ print_generic_expr (dump_file, mr->step);
+ }
+ fprintf (dump_file, "\n");
+}
+
+/* If there is a simple load or store to a memory reference in STMT, returns
+ the location of the memory reference, and sets IS_STORE according to whether
+ it is a store or load. Otherwise, returns NULL.
+ TODO: from gcc/tree-ssa-loop-im.c, maybe make it global. */
+
+static tree *
+simple_mem_ref_in_stmt (gimple *stmt, bool *is_store)
+{
+ tree *lhs, *rhs;
+
+ /* Recognize SSA_NAME = MEM and MEM = (SSA_NAME | invariant) patterns. */
+ if (!gimple_assign_single_p (stmt))
+ return NULL;
+
+ lhs = gimple_assign_lhs_ptr (stmt);
+ rhs = gimple_assign_rhs1_ptr (stmt);
+
+ if (TREE_CODE (*lhs) == SSA_NAME && gimple_vuse (stmt))
+ {
+ *is_store = false;
+ return rhs;
+ }
+ else if (gimple_vdef (stmt)
+ && (TREE_CODE (*rhs) == SSA_NAME || is_gimple_min_invariant (*rhs)))
+ {
+ *is_store = true;
+ return lhs;
+ }
+ else
+ return NULL;
+}
+
+static void
+analyse_incremental (gimple *stmt, memref_t* mr)
+{
+ if (!gimple_assign_single_p (stmt))
+ return;
+ tree rhs1, rhs2;
+ /* TODO: maybe support other types of stmts. */
+ while (stmt && is_gimple_assign (stmt))
+ {
+ enum tree_code def_code = gimple_assign_rhs_code (stmt);
+ gimple_rhs_class rhs_class = gimple_assign_rhs_class (stmt);
+ if (dump_file)
+ {
+ fprintf (dump_file, "Incr: in assign (%s)\n",
+ get_tree_code_name (def_code));
+ print_gimple_stmt (dump_file, stmt, 3, TDF_DETAILS);
+ }
+ gcc_assert (def_code != ERROR_MARK);
+ switch (rhs_class)
+ {
+ case GIMPLE_TERNARY_RHS:
+ if (dump_file)
+ fprintf (dump_file, "Incr: unsupported trinary rhs\n");
+ stmt = NULL;
+ break;
+ case GIMPLE_UNARY_RHS:
+ case GIMPLE_SINGLE_RHS:
+ rhs1 = gimple_assign_rhs1 (stmt);
+ if (dump_file)
+ {
+ fprintf (dump_file, "Incr: (%s)",
+ get_tree_code_name (TREE_CODE (rhs1)));
+ print_generic_expr (dump_file, rhs1);
+ fprintf (dump_file, "\n");
+ }
+ if (def_code == SSA_NAME)
+ stmt = SSA_NAME_DEF_STMT (rhs1);
+ else if (def_code == MEM_REF || def_code == COMPONENT_REF
+ || def_code == ARRAY_REF)
+ {
+ /* If we have dereference in address evaluation,
+ it's indirect memory access. */
+ if (dump_file)
+ {
+ if (operand_equal_p (mr->mem, rhs1))
+ fprintf (dump_file, "Incr: the same MEM\n");
+ else
+ fprintf (dump_file, "Incr: diff MEM\n");
+ print_generic_expr (dump_file, rhs1);
+ fprintf (dump_file, " ");
+ print_generic_expr (dump_file, mr->mem);
+ fprintf (dump_file, "\n");
+ }
+ if (operand_equal_p (mr->mem, rhs1) && mr->step)
+ mr->is_incr = true;
+ stmt = NULL;
+ }
+ else
+ {
+ if (dump_file)
+ fprintf (dump_file, "Incr: unsupported unary/single\n");
+ stmt = NULL;
+ }
+ break;
+ case GIMPLE_BINARY_RHS:
+ rhs1 = gimple_assign_rhs1 (stmt);
+ rhs2 = gimple_assign_rhs2 (stmt);
+ if (dump_file)
+ {
+ fprintf (dump_file, "(%s) (%s)",
+ get_tree_code_name (TREE_CODE (rhs1)),
+ get_tree_code_name (TREE_CODE (rhs2)));
+ print_generic_expr (dump_file, rhs1);
+ fprintf (dump_file, " ");
+ print_generic_expr (dump_file, rhs2);
+ fprintf (dump_file, "\n");
+ }
+ /* TODO: extend for other types of incrementation. */
+ if (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == INTEGER_CST)
+ {
+ stmt = SSA_NAME_DEF_STMT (rhs1);
+ mr->step = rhs2;
+ if (dump_file)
+ {
+ fprintf (dump_file, "Incr: const increment stmt: ");
+ print_gimple_stmt (dump_file, stmt, 3, TDF_DETAILS);
+ }
+ }
+ else
+ stmt = NULL;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ }
+ if ((mr->step && !mr->is_incr) || (!mr->step && mr->is_incr))
+ {
+ mr->step = NULL_TREE;
+ mr->is_incr = false;
+ }
+}
+
+static mr_type
+get_memref_type (memref_t *base, memref_t *used, enum tree_code code)
+{
+ /* TODO: improve memref type detection. */
+ enum tree_code base_code = TREE_CODE (base->mem);
+ if (dump_file)
+ fprintf (dump_file, "get_memref_type: base=%d,%d used=%d,%d code=%s "
+ "base_code=%s\n", base->mr_id, base->type,
+ used ? used->mr_id : -1, used ? used->type : -1,
+ get_tree_code_name (code), get_tree_code_name (base_code));
+ if (used)
+ {
+ if (base->type > used->type)
+ return base->type;
+ if (used->type == MR_SIMPLE)
+ return MR_POLYNOMIAL;
+ if (used->type == MR_POLYNOMIAL)
+ return base_code == ARRAY_REF ? MR_POLYNOMIAL : MR_INDIRECT;
+ if (used->type == MR_INDIRECT)
+ return MR_INDIRECT;
+ return MR_UNSUPPORTED;
+ }
+ if (code == MEM_REF || code == ARRAY_REF || code == COMPONENT_REF)
+ return base->type;
+ if (code == POINTER_PLUS_EXPR || code == PLUS_EXPR
+ || code == MINUS_EXPR || code == MULT_EXPR)
+ return base->type <= MR_POLYNOMIAL ? MR_POLYNOMIAL : base->type;
+ return base->type >= MR_INDIRECT ? base->type : MR_INDIRECT;
+}
+
+/* Recursively walk defs of src expression and record used stmts and other mrs.
+ Return a base address candidate if it's found. */
+
+static tree
+analyse_addr_eval (tree src, memref_t* mr)
+{
+ if (TREE_CODE (src) != SSA_NAME)
+ return NULL_TREE;
+ gimple *stmt = SSA_NAME_DEF_STMT (src);
+ if (dump_file)
+ {
+ fprintf (dump_file, "Src_stmt: ");
+ print_gimple_stmt (dump_file, stmt, 0);
+ }
+ if (!is_gimple_assign (stmt))
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "Is not assign, stop analysis: ");
+ print_gimple_stmt (dump_file, stmt, 3, TDF_DETAILS);
+ }
+ mr->type = MR_UNSUPPORTED;
+ mr->stmts.safe_push (stmt);
+ return NULL_TREE;
+ }
+ enum tree_code def_code = gimple_assign_rhs_code (stmt);
+ if (def_code != MEM_REF && def_code != COMPONENT_REF
+ && def_code != ARRAY_REF)
+ mr->stmts.safe_push (stmt);
+ gimple_rhs_class rhs_class = gimple_assign_rhs_class (stmt);
+ tree rhs1, rhs2, base;
+ if (dump_file)
+ fprintf (dump_file, "In assign (%s): ", get_tree_code_name (def_code));
+
+ switch (rhs_class)
+ {
+ case GIMPLE_TERNARY_RHS:
+ if (dump_file)
+ fprintf (dump_file, "Unsupported trinary rhs\n");
+ mr->type = MR_UNSUPPORTED;
+ return NULL_TREE;
+ case GIMPLE_UNARY_RHS:
+ case GIMPLE_SINGLE_RHS:
+ rhs1 = gimple_assign_rhs1 (stmt);
+ if (dump_file)
+ {
+ fprintf (dump_file, "(%s)",
+ get_tree_code_name (TREE_CODE (rhs1)));
+ print_generic_expr (dump_file, rhs1);
+ fprintf (dump_file, "\n");
+ }
+ if (def_code == NOP_EXPR)
+ return analyse_addr_eval (rhs1, mr);
+ else if (def_code == MEM_REF || def_code == COMPONENT_REF
+ || def_code == ARRAY_REF)
+ {
+ memref_t *mr2 = get_memref (stmt, rhs1, false);
+ mr->type = get_memref_type (mr, mr2, def_code);
+ for (memref_set::const_iterator it = mr2->used_mrs.begin ();
+ it != mr2->used_mrs.end (); it++)
+ mr->used_mrs.insert (*it);
+ mr->used_mrs.insert (mr2);
+ return mr2->base;
+ }
+ else
+ {
+ if (dump_file)
+ fprintf (dump_file, "Unsupported unary/single\n");
+ mr->type = MR_UNSUPPORTED;
+ }
+ return NULL_TREE;
+ case GIMPLE_BINARY_RHS:
+ rhs1 = gimple_assign_rhs1 (stmt);
+ rhs2 = gimple_assign_rhs2 (stmt);
+ if (dump_file)
+ {
+ fprintf (dump_file, "(%s) (%s)",
+ get_tree_code_name (TREE_CODE (rhs1)),
+ get_tree_code_name (TREE_CODE (rhs2)));
+ print_generic_expr (dump_file, rhs1);
+ fprintf (dump_file, " ");
+ print_generic_expr (dump_file, rhs2);
+ fprintf (dump_file, "\n");
+ }
+ base = analyse_addr_eval (rhs1, mr);
+ analyse_addr_eval (rhs2, mr);
+ mr->type = get_memref_type (mr, NULL, def_code);
+ return base;
+ default:
+ gcc_unreachable ();
+ }
+ return NULL_TREE;
+}
+
+static tree
+get_mem_ref_address_ssa_name (tree mem, tree base)
+{
+ gcc_assert (TREE_CODE (mem) == MEM_REF);
+ if (base == NULL_TREE)
+ base = get_base_address (mem);
+ tree base_addr = NULL_TREE;
+ if (TREE_CODE (base) == MEM_REF)
+ base_addr = TREE_OPERAND (base, 0);
+ if (base_addr != NULL_TREE && TREE_CODE (base_addr) == SSA_NAME)
+ return base_addr;
+ return NULL_TREE;
+}
+
+static void
+analyse_mem_ref (gimple *stmt, tree mem, memref_t* mr)
+{
+ tree base = get_base_address (mem);
+ if (dump_file)
+ fprintf (dump_file, "Codes: base = %s, mem = %s\n",
+ base ? get_tree_code_name (TREE_CODE (base)) : "null",
+ mem ? get_tree_code_name (TREE_CODE (mem)) : "null");
+
+ mr->stmts.safe_push (stmt);
+ mr->base = base;
+ switch (TREE_CODE (mem))
+ {
+ case COMPONENT_REF:
+ if (mr->is_store)
+ analyse_incremental (stmt, mr);
+ mr->type = MR_SIMPLE;
+ mr->offset = TREE_OPERAND (mem, 1);
+ return;
+ case ARRAY_REF:
+ analyse_addr_eval (TREE_OPERAND (mem, 1), mr);
+ return;
+ case MEM_REF:
+ {
+ tree base_addr = get_mem_ref_address_ssa_name (mem, base);
+ if (dump_file)
+ {
+ fprintf (dump_file, "Base addr (%s): ",
+ base_addr ? get_tree_code_name (TREE_CODE (base_addr))
+ : "null");
+ if (base_addr)
+ print_generic_expr (dump_file, base_addr);
+ fprintf (dump_file, "\n");
+ }
+ if (base_addr)
+ {
+ mr->base = analyse_addr_eval (base_addr, mr);
+ return;
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ mr->type = MR_UNSUPPORTED;
+ mr->base = NULL_TREE;
+}
+
+static void
+analyse_stmt (gimple *stmt)
+{
+ bool is_store;
+ tree *mem = simple_mem_ref_in_stmt (stmt, &is_store);
+ if (!mem)
+ return;
+ if (dump_file)
+ {
+ fprintf (dump_file, "\n%s: mr is found in stmt (%s): ",
+ function_name (cfun), is_store ? "store" : "load");
+ print_gimple_stmt (dump_file, stmt, 3, TDF_DETAILS);
+ }
+ memref_t *mr = get_memref (stmt, *mem, is_store);
+ (*fmrs_map)[cfun]->insert (mr);
+ if (dump_file)
+ print_memref (mr);
+}
+
+/* Scan stmts for indirect stores/loads with bases passed as function args. */
+
+static void
+collect_memrefs_for_cgraph_node (struct cgraph_node *n)
+{
+ if (dump_file)
+ fprintf (dump_file, "\nCollect indirect ptr info in %s\n", n->dump_name ());
+ n->get_body ();
+ function *fn = DECL_STRUCT_FUNCTION (n->decl);
+ gcc_assert (fn && n->has_gimple_body_p ());
+
+ push_cfun (fn);
+ basic_block bb;
+ gimple_stmt_iterator si;
+ (*fmrs_map)[fn] = new memref_set;
+ FOR_EACH_BB_FN (bb, fn)
+ for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
+ {
+ gimple *stmt = gsi_stmt (si);
+ analyse_stmt (stmt);
+ }
+ pop_cfun ();
+}
+
+/* Walk cgraph nodes and collect memory references info. */
+
+static void
+collect_memory_references ()
+{
+ struct cgraph_node *n;
+ /* TODO: collect info only for loops and functions in loops. */
+ FOR_EACH_DEFINED_FUNCTION (n)
+ if (nl_map->count (n) != 0 && n->has_gimple_body_p ())
+ collect_memrefs_for_cgraph_node (n);
+
+ if (dump_file)
+ {
+ fprintf (dump_file, "\n\nDump mem references:\n");
+ FOR_EACH_DEFINED_FUNCTION (n)
+ if (nl_map->count (n) != 0 && n->has_gimple_body_p ())
+ {
+ function *fn = DECL_STRUCT_FUNCTION (n->decl);
+ fprintf (dump_file, "\nIn function %s (%s):\n", function_name (fn),
+ nl_map->count (n) != 0 ? "in loop" : "");
+ for (memref_set::const_iterator it = (*fmrs_map)[fn]->begin ();
+ it != (*fmrs_map)[fn]->end (); it++)
+ print_memref (*it);
+ }
+ }
+}
+
+/* Analysis of loops. */
+
+memref_set *current_incr_mrs;
+memref_set *current_indirect_mrs;
+
+static void
+collect_memref (memref_t *mr, class loop *loop, bool check_loop)
+{
+ gimple *stmt = mr->stmts[0];
+ gcc_assert (stmt);
+ if (check_loop && !flow_bb_inside_loop_p (loop, gimple_bb (stmt)))
+ return;
+
+ /* TODO: Improve base invariant analysis for memrefs which are not local
+ (located in called functions). */
+ bool is_base_inv = false;
+ if (mr->base)
+ is_base_inv = expr_invariant_in_loop_p (loop, mr->base);
+
+ if (dump_file && (mr->type == MR_INDIRECT || mr->is_incr))
+ {
+ fprintf (dump_file, "%s MR (%d): ", mr->is_incr ? "INCR" : "INDIRECT",
+ mr->mr_id);
+ print_generic_expr (dump_file, mr->mem);
+ fprintf (dump_file, "\twith base: ");
+ if (mr->base)
+ print_generic_expr (dump_file, mr->base);
+ else
+ fprintf (dump_file, "null");
+ fprintf (dump_file, " (is_inv=%d)\n", is_base_inv);
+ }
+
+ if (!is_base_inv)
+ return;
+ if (mr->type == MR_INDIRECT)
+ current_indirect_mrs->insert (mr);
+ if (mr->is_incr)
+ current_incr_mrs->insert (mr);
+}
+
+static void
+analyse_callable_function (struct cgraph_node *n, class loop *loop)
+{
+ if (dump_file)
+ fprintf (dump_file, "Callable (%s):\n", n->dump_name ());
+
+ function *fn = DECL_STRUCT_FUNCTION (n->decl);
+ if (fmrs_map->count (fn))
+ for (memref_set::const_iterator it = (*fmrs_map)[fn]->begin ();
+ it != (*fmrs_map)[fn]->end (); it++)
+ collect_memref (*it, loop, false);
+}
+
+static void
+insert_node_with_callable_nodes (node_set &s, struct cgraph_node *n)
+{
+ s.insert (n);
+ if (nn_map->count (n) == 0)
+ return;
+ node_set *set = (*nn_map)[n];
+ for (node_set::const_iterator it = set->begin (); it != set->end (); it++)
+ s.insert ((*it));
+}
+
+static bool
+compatible_memrefs_p (memref_t *mr1, memref_t *mr2, bool &compatible_offset)
+{
+ if (!mr1->base || !mr2->base || !mr2->offset)
+ return false;
+ tree base_type1 = TYPE_MAIN_VARIANT (TREE_TYPE (mr1->base));
+ tree base_type2 = TYPE_MAIN_VARIANT (TREE_TYPE (mr2->base));
+ if (base_type1 != base_type2)
+ return false;
+ if (mr1->offset && mr1->offset == mr2->offset)
+ compatible_offset = true;
+ else
+ compatible_offset = false;
+ return true;
+}
+
+static void
+compare_memrefs (memref_t* mr, memref_t* mr2)
+{
+ /* TODO: improve analysis of memrefs from different functions: take into
+ account data flow and context. */
+ bool compatible_offset = false;
+ if (!compatible_memrefs_p (mr, mr2, compatible_offset))
+ return;
+ if (!compatible_offset)
+ {
+ for (memref_set::const_iterator it = mr->used_mrs.begin ();
+ it != mr->used_mrs.end (); it++)
+ if ((*it)->offset && (*it)->offset == mr2->offset)
+ {
+ compatible_offset = true;
+ if (dump_file)
+ fprintf (dump_file, "Used MR (%d) and INC MR have "
+ "the same offset\n", (*it)->mr_id);
+ break;
+ }
+ }
+ if (!compatible_offset)
+ return;
+ if (dump_file)
+ {
+ fprintf (dump_file, "MR (%d) is optimization candidate with offset: ",
+ mr->mr_id);
+ print_generic_expr (dump_file, mr2->offset);
+ fprintf (dump_file, "\n");
+ }
+
+ if (!mr_candidate_map->count (mr))
+ {
+ (*mr_candidate_map)[mr] = mr2;
+ return;
+ }
+ /* TODO: support analysis with incrementation of different fields. */
+ if ((*mr_candidate_map)[mr]->offset != mr2->offset)
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "It conflicts with previously found MR (%d) "
+ "with offset ", (*mr_candidate_map)[mr]->mr_id);
+ if ((*mr_candidate_map)[mr] != NULL)
+ print_generic_expr (dump_file, (*mr_candidate_map)[mr]->offset);
+ fprintf (dump_file, ", disable the optimization\n");
+ }
+ (*mr_candidate_map)[mr] = NULL;
+ }
+}
+
+/* In the given loop and all functions called from the loop, collect
+ indirect/incremental memrefs with invariant base address and inductive
+ offset. */
+
+static void
+collect_memrefs_for_loop (class loop *loop, struct cgraph_node *n,
+ function *fn)
+{
+ current_incr_mrs = new memref_set;
+ current_indirect_mrs = new memref_set;
+
+ if (dump_file)
+ fprintf (dump_file, "Loop %d\n", loop->num);
+ if (fmrs_map->count (fn))
+ for (memref_set::const_iterator it = (*fmrs_map)[fn]->begin ();
+ it != (*fmrs_map)[fn]->end (); it++)
+ collect_memref (*it, loop, true);
+
+ /* Collect vector of functions called in the loop. */
+ node_set set;
+ struct cgraph_edge *e;
+ struct cgraph_node *n2;
+ for (e = n->callees; e; e = e->next_callee)
+ {
+ gcall *stmt = e->call_stmt;
+ if (!flow_bb_inside_loop_p (loop, gimple_bb (stmt)))
+ continue;
+ insert_node_with_callable_nodes (set, e->callee);
+ }
+ for (e = n->indirect_calls; e; e = e->next_callee)
+ {
+ gcall *stmt = e->call_stmt;
+ if (!flow_bb_inside_loop_p (loop, gimple_bb (stmt))
+ || !e->indirect_info->targets)
+ continue;
+ for (unsigned i = 0; e->indirect_info->targets->iterate (i, &n2); ++i)
+ insert_node_with_callable_nodes (set, n2);
+ }
+ if (set.empty ())
+ return;
+ if (dump_file)
+ fprintf (dump_file, "Go inside all callables of %s\n", n->dump_name ());
+
+ for (node_set::const_iterator it = set.begin (); it != set.end (); it++)
+ analyse_callable_function (*it, loop);
+
+ if (!current_incr_mrs->empty () && !current_indirect_mrs->empty ())
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "Loop has both incr and indirect memrefs\n"
+ "Incr: ");
+ for (memref_set::const_iterator it = current_incr_mrs->begin ();
+ it != current_incr_mrs->end (); it++)
+ fprintf (dump_file, "%d ", (*it)->mr_id);
+ fprintf (dump_file, "\nIndirect: ");
+ for (memref_set::const_iterator it = current_indirect_mrs->begin ();
+ it != current_indirect_mrs->end (); it++)
+ fprintf (dump_file, "%d ", (*it)->mr_id);
+ fprintf (dump_file, "\n");
+ }
+ /* Check if indirect memref has a base address similar to one of
+ incremental memref. */
+ for (memref_set::const_iterator it = current_indirect_mrs->begin ();
+ it != current_indirect_mrs->end (); it++)
+ for (memref_set::const_iterator it2 = current_incr_mrs->begin ();
+ it2 != current_incr_mrs->end (); it2++)
+ compare_memrefs (*it, *it2);
+ }
+
+ delete current_incr_mrs;
+ delete current_indirect_mrs;
+}
+
+static void
+analyse_loops_in_cgraph_node (struct cgraph_node *n)
+{
+ if (dump_file)
+ fprintf (dump_file, "\nAnalyse loops in %s\n", n->dump_name ());
+
+ n->get_body ();
+ function *fn = DECL_STRUCT_FUNCTION (n->decl);
+ gcc_assert (fn && n->has_gimple_body_p ());
+
+ push_cfun (fn);
+ calculate_dominance_info (CDI_DOMINATORS);
+ loop_optimizer_init (LOOPS_NORMAL);
+
+ for (auto loop : loops_list (cfun, 0))
+ {
+ class loop *outer = loop_outer (loop);
+ /* Walk only outermost loops. */
+ if (outer->num != 0)
+ continue;
+ collect_memrefs_for_loop (loop, n, fn);
+ }
+
+ free_dominance_info (CDI_DOMINATORS);
+ loop_optimizer_finalize ();
+ pop_cfun ();
+}
+
+static void
+analyse_loops ()
+{
+ if (dump_file)
+ fprintf (dump_file, "\n\nLoops: procesing functions\n");
+ cgraph_node *n;
+ FOR_EACH_DEFINED_FUNCTION (n)
+ {
+ if (!can_be_optimized (n))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Skip the function\n");
+ continue;
+ }
+ analyse_loops_in_cgraph_node (n);
+ }
+
+ if (dump_file)
+ fprintf (dump_file, "\n\nList of optimization candidates:\n");
+
+ FOR_EACH_DEFINED_FUNCTION (n)
+ {
+ function *fn = DECL_STRUCT_FUNCTION (n->decl);
+ if (!can_be_optimized (n) || !fmrs_map->count (fn))
+ continue;
+ for (memref_map::iterator it = mr_candidate_map->begin ();
+ it != mr_candidate_map->end (); ++it)
+ {
+ memref_t *mr = it->first, *mr2 = it->second;
+ if (mr2 == NULL || !(*fmrs_map)[fn]->count (mr))
+ continue;
+ if (!optimize_mrs_map->count (fn))
+ (*optimize_mrs_map)[fn] = new memref_set;
+ (*optimize_mrs_map)[fn]->insert (mr);
+ }
+ if (dump_file && optimize_mrs_map->count (fn))
+ {
+ fprintf (dump_file, "Function %s\n", n->dump_name ());
+ for (memref_set::const_iterator it
+ = (*optimize_mrs_map)[fn]->begin ();
+ it != (*optimize_mrs_map)[fn]->end (); it++)
+ {
+ memref_t *mr = *it, *mr2 = (*mr_candidate_map)[mr];
+ fprintf (dump_file, "MRs %d,%d with incremental offset ",
+ mr->mr_id, mr2->mr_id);
+ print_generic_expr (dump_file, mr2->offset);
+ fprintf (dump_file, "\n");
+ }
+ }
+ }
+}
+
+/* Reduce the set filtering out memrefs with the same memory references,
+ return the result vector of memrefs. */
+
+static void
+reduce_memref_set (memref_set *set, vec<memref_t *> &vec)
+{
+ for (memref_set::const_iterator it = set->begin ();
+ it != set->end (); it++)
+ {
+ memref_t *mr1 = *it;
+ if (!vec.length ())
+ vec.safe_push (mr1);
+ else
+ {
+ bool inserted = false;
+ for (unsigned int i = 0; i < vec.length (); i++)
+ {
+ /* mr2 is less than current mr1. */
+ memref_t *mr2 = vec[i];
+ if (operand_equal_p (mr1->mem, mr2->mem))
+ {
+ if (dump_file)
+ fprintf (dump_file, "The same mems in MRs %d and %d\n",
+ mr1->mr_id, mr2->mr_id);
+ /* TODO: maybe build new memref which include stmts of both
+ mr1 and mr2. */
+ if ((mr1->is_store && !mr2->is_store)
+ || mr1->stmts.length () > mr2->stmts.length ())
+ {
+ inserted = true;
+ vec[i] = mr1;
+ }
+ }
+ }
+ if (!inserted)
+ vec.safe_push (mr1);
+ }
+ }
+ if (dump_file)
+ {
+ fprintf (dump_file, "MRs (%d) after filtering: ", vec.length ());
+ for (unsigned int i = 0; i < vec.length (); i++)
+ fprintf (dump_file, "%d ", vec[i]->mr_id);
+ fprintf (dump_file, "\n");
+ }
+}
+
+static void
+find_nearest_common_dominator (memref_t *mr, basic_block &dom)
+{
+ for (unsigned int i = 0; i < mr->stmts.length (); i++)
+ {
+ basic_block bb = gimple_bb (mr->stmts[i]);
+ gcc_assert (bb);
+ if (dom == bb)
+ continue;
+ if (dom)
+ dom = nearest_common_dominator (CDI_DOMINATORS, dom, bb);
+ else
+ dom = bb;
+ }
+}
+
+/* Return true if DECL is a parameter or a SSA_NAME for a parameter.
+ TODO: from gcc/tree-inline.c, maybe make it global. */
+
+static bool
+is_parm (tree decl)
+{
+ if (TREE_CODE (decl) == SSA_NAME)
+ {
+ decl = SSA_NAME_VAR (decl);
+ if (!decl)
+ return false;
+ }
+
+ return (TREE_CODE (decl) == PARM_DECL);
+}
+
+/* TODO: the following functions are inspired by remap in gcc/tree-inline.c,
+ maybe we can share some functionality. */
+
+static tree
+remap_name (tree name, gimple *stmt, bool is_lhs)
+{
+ tree new_tree = NULL_TREE;
+ if (decl_map->count (name))
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "Find map: ");
+ print_generic_expr (dump_file, name);
+ fprintf (dump_file, " ");
+ print_generic_expr (dump_file, (*decl_map)[name]);
+ fprintf (dump_file, "\n");
+ }
+ return unshare_expr ((*decl_map)[name]);
+ }
+ if (!is_lhs)
+ return name;
+ if (TREE_CODE (name) == SSA_NAME)
+ {
+ /* Remap anonymous SSA names or SSA names of anonymous decls. */
+ tree var = SSA_NAME_VAR (name);
+ if (!var
+ || (!SSA_NAME_IS_DEFAULT_DEF (name)
+ && VAR_P (var) && !VAR_DECL_IS_VIRTUAL_OPERAND (var)
+ && DECL_ARTIFICIAL (var) && DECL_IGNORED_P (var)
+ && !DECL_NAME (var)))
+ {
+ new_tree = make_ssa_name (TREE_TYPE (name), stmt);
+ if (!var && SSA_NAME_IDENTIFIER (name))
+ SET_SSA_NAME_VAR_OR_IDENTIFIER (new_tree,
+ SSA_NAME_IDENTIFIER (name));
+ SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
+ = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
+ /* So can range-info. */
+ if (!POINTER_TYPE_P (TREE_TYPE (name))
+ && SSA_NAME_RANGE_INFO (name))
+ duplicate_ssa_name_range_info (new_tree,
+ SSA_NAME_RANGE_TYPE (name),
+ SSA_NAME_RANGE_INFO (name));
+ /* TODO: maybe correct the insertion. */
+ (*decl_map)[name] = new_tree;
+ if (dump_file)
+ {
+ fprintf (dump_file, "New map (no var): ");
+ print_generic_expr (dump_file, name);
+ fprintf (dump_file, " ");
+ print_generic_expr (dump_file, new_tree);
+ fprintf (dump_file, "\n");
+ }
+ return new_tree;
+ }
+ /* TODO: maybe remap_name or do the same as before for SSA_NAME_VAR. */
+ new_tree = make_ssa_name (TREE_TYPE (name), stmt);
+ (*decl_map)[name] = new_tree;
+ if (dump_file)
+ {
+ fprintf (dump_file, "New map: ");
+ print_generic_expr (dump_file, name);
+ fprintf (dump_file, " ");
+ print_generic_expr (dump_file, new_tree);
+ fprintf (dump_file, "\n");
+ }
+ }
+ else if (VAR_P (name) || TREE_CODE (name) == PARM_DECL)
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "VAR/PARM: ");
+ print_generic_expr (dump_file, name);
+ fprintf (dump_file, "\n");
+ }
+ return name;
+ }
+ else
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "Unsupported: ");
+ print_generic_expr (dump_file, name);
+ fprintf (dump_file, "\n");
+ }
+ //gcc_unreachable ();
+ return name;
+ }
+ return new_tree;
+}
+
+/* Passed to walk_tree. Copies the node pointed to, if appropriate. */
+
+static tree
+ipa_copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
+{
+ enum tree_code code = TREE_CODE (*tp);
+ enum tree_code_class cl = TREE_CODE_CLASS (code);
+
+ /* We make copies of most nodes. */
+ if (IS_EXPR_CODE_CLASS (cl)
+ || code == TREE_LIST
+ || code == TREE_VEC
+ || code == TYPE_DECL
+ || code == OMP_CLAUSE)
+ {
+ /* Because the chain gets clobbered when we make a copy, we save it
+ here. */
+ tree chain = NULL_TREE, new_tree;
+
+ if (CODE_CONTAINS_STRUCT (code, TS_COMMON))
+ chain = TREE_CHAIN (*tp);
+
+ /* Copy the node. */
+ new_tree = copy_node (*tp);
+
+ *tp = new_tree;
+
+ /* Now, restore the chain, if appropriate. That will cause
+ walk_tree to walk into the chain as well. */
+ if (code == PARM_DECL
+ || code == TREE_LIST
+ || code == OMP_CLAUSE)
+ TREE_CHAIN (*tp) = chain;
+
+ /* For now, we don't update BLOCKs when we make copies. So, we
+ have to nullify all BIND_EXPRs. */
+ if (TREE_CODE (*tp) == BIND_EXPR)
+ BIND_EXPR_BLOCK (*tp) = NULL_TREE;
+ }
+ else if (code == CONSTRUCTOR || code == STATEMENT_LIST)
+ gcc_unreachable ();
+ else if (TREE_CODE_CLASS (code) == tcc_type
+ || TREE_CODE_CLASS (code) == tcc_declaration
+ || TREE_CODE_CLASS (code) == tcc_constant)
+ *walk_subtrees = 0;
+ return NULL_TREE;
+}
+
+/* Remap the GIMPLE operand pointed to by *TP. DATA is really a
+ 'struct walk_stmt_info *'. DATA->INFO is a 'gimple *'.
+ WALK_SUBTREES is used to indicate walk_gimple_op whether to keep
+ recursing into the children nodes of *TP. */
+
+static tree
+remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data)
+{
+ struct walk_stmt_info *wi_p = (struct walk_stmt_info *) data;
+ gimple *stmt = (gimple *) wi_p->info;
+
+ /* For recursive invocations this is no longer the LHS itself. */
+ bool is_lhs = wi_p->is_lhs;
+ wi_p->is_lhs = false;
+
+ if (TREE_CODE (*tp) == SSA_NAME)
+ {
+ *tp = remap_name (*tp, stmt, is_lhs);
+ *walk_subtrees = 0;
+ if (is_lhs)
+ SSA_NAME_DEF_STMT (*tp) = wi_p->stmt;
+ return NULL;
+ }
+ else if (auto_var_in_fn_p (*tp, cfun->decl))
+ {
+ /* Local variables and labels need to be replaced by equivalent
+ variables. We don't want to copy static variables; there's
+ only one of those, no matter how many times we inline the
+ containing function. Similarly for globals from an outer
+ function. */
+ tree new_decl;
+
+ /* Remap the declaration. */
+ new_decl = remap_name (*tp, stmt, is_lhs);
+ gcc_assert (new_decl);
+ /* Replace this variable with the copy. */
+ STRIP_TYPE_NOPS (new_decl);
+ /* ??? The C++ frontend uses void * pointer zero to initialize
+ any other type. This confuses the middle-end type verification.
+ As cloned bodies do not go through gimplification again the fixup
+ there doesn't trigger. */
+ if (TREE_CODE (new_decl) == INTEGER_CST
+ && !useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (new_decl)))
+ new_decl = fold_convert (TREE_TYPE (*tp), new_decl);
+ *tp = new_decl;
+ *walk_subtrees = 0;
+ }
+ else if (TREE_CODE (*tp) == STATEMENT_LIST || TREE_CODE (*tp) == SAVE_EXPR)
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "Unexpected tree: ");
+ print_generic_expr (dump_file, *tp);
+ fprintf (dump_file, "\n");
+ }
+ gcc_unreachable ();
+ }
+ else
+ {
+ /* Otherwise, just copy the node. Note that copy_tree_r already
+ knows not to copy VAR_DECLs, etc., so this is safe. */
+
+ if (TREE_CODE (*tp) == MEM_REF)
+ {
+ /* We need to re-canonicalize MEM_REFs from inline substitutions
+ that can happen when a pointer argument is an ADDR_EXPR.
+ Recurse here manually to allow that. */
+ tree ptr = TREE_OPERAND (*tp, 0);
+ tree type = TREE_TYPE (*tp);
+ tree old = *tp;
+ walk_tree (&ptr, remap_gimple_op_r, data, NULL);
+ *tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
+ TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
+ TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
+ TREE_NO_WARNING (*tp) = TREE_NO_WARNING (old);
+ /* TODO: maybe support this case. */
+ gcc_assert (MR_DEPENDENCE_CLIQUE (old) == 0);
+ /* We cannot propagate the TREE_THIS_NOTRAP flag if we have
+ remapped a parameter as the property might be valid only
+ for the parameter itself. */
+ if (TREE_THIS_NOTRAP (old) && (!is_parm (TREE_OPERAND (old, 0))))
+ TREE_THIS_NOTRAP (*tp) = 1;
+ REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
+ *walk_subtrees = 0;
+ return NULL;
+ }
+
+ /* Here is the "usual case". Copy this tree node, and then
+ tweak some special cases. */
+ ipa_copy_tree_r (tp, walk_subtrees, NULL);
+ gcc_assert (!(TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3)));
+ if (TREE_CODE (*tp) == ADDR_EXPR)
+ {
+ /* TODO: If this used to be invariant, but is not any longer,
+ then regimplification is probably needed. */
+ walk_tree (&TREE_OPERAND (*tp, 0), remap_gimple_op_r, data, NULL);
+ recompute_tree_invariant_for_addr_expr (*tp);
+ *walk_subtrees = 0;
+ }
+ }
+ /* TODO: maybe we need to update TREE_BLOCK (*tp). */
+
+ /* Keep iterating. */
+ return NULL_TREE;
+}
+
+static void
+create_cgraph_edge (cgraph_node *n, gimple *stmt)
+{
+ gcall *call_stmt = dyn_cast <gcall *> (stmt);
+ basic_block bb = gimple_bb (stmt);
+ tree decl = gimple_call_fndecl (call_stmt);
+ if (!decl)
+ return;
+ struct cgraph_edge *e = n->create_edge (cgraph_node::get_create (decl),
+ call_stmt, bb->count);
+ /* TODO: maybe we need to store ipa_call_summary result. */
+ ipa_call_summaries->get_create (e);
+}
+
+/* Insert prefetch intrinsics in this function, return nonzero on success. */
+
+static int
+optimize_function (cgraph_node *n, function *fn)
+{
+ /* In a given function, optimize only indirect memrefs with
+ the same incremental memref.
+ TODO: implement the optimization for other cases. */
+ bool different_incrementals = false;
+ memref_t *first_mr = NULL;
+ memref_set used_mrs;
+ for (memref_set::const_iterator it = (*optimize_mrs_map)[fn]->begin ();
+ it != (*optimize_mrs_map)[fn]->end (); it++)
+ {
+ memref_t *mr = *it;
+ if (!first_mr)
+ first_mr = mr;
+ else if ((*mr_candidate_map)[first_mr] != (*mr_candidate_map)[mr])
+ {
+ different_incrementals = true;
+ break;
+ }
+ for (memref_set::const_iterator it2 = mr->used_mrs.begin ();
+ it2 != mr->used_mrs.end (); it2++)
+ used_mrs.insert (*it2);
+ }
+ if (different_incrementals)
+ {
+ if (dump_file)
+ fprintf (dump_file, "It contains memrefs with different "
+ "incrementals. Skip the case.\n");
+ return 0;
+ }
+ memref_t *inc_mr = (*mr_candidate_map)[first_mr];
+ if (!inc_mr->stmts[0] || !gimple_assign_single_p (inc_mr->stmts[0]))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Incremental MR with unexpected stmt. "
+ "Skip the case.\n");
+ return 0;
+ }
+ if (dump_file && !used_mrs.empty ())
+ print_mrs_ids (used_mrs, "Common list of used mrs:\n");
+
+ /* Find a memref in used mrs which corresponds to the found incremental
+ memref. */
+ memref_t *comp_mr = NULL;
+ for (memref_set::const_iterator it = used_mrs.begin ();
+ it != used_mrs.end (); it++)
+ {
+ bool c_offset;
+ if ((*it)->type != MR_SIMPLE || inc_mr->type != MR_SIMPLE
+ || !compatible_memrefs_p (*it, inc_mr, c_offset))
+ continue;
+ if (c_offset)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Found compatible used MR (%d) and "
+ "incr MR (%d)\n", (*it)->mr_id, inc_mr->mr_id);
+ comp_mr = (*it);
+ }
+ }
+ if (!comp_mr || !comp_mr->stmts[0]
+ || !gimple_assign_single_p (comp_mr->stmts[0]))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Compatible MR in this function is not found "
+ " or it has unexpected stmt. Skip the case.\n");
+ return 0;
+ }
+
+ /* Filter out memrefs with the same memory references.
+ TODO: maybe do the same with used mrs. */
+ vec<memref_t *> vmrs = vNULL;
+ reduce_memref_set ((*optimize_mrs_map)[fn], vmrs);
+
+ /* Find insertion place. Create new BB. */
+ /* TODO: maybe it is useful to process also used_mrs. */
+ basic_block dom_bb = NULL;
+ for (unsigned int i = 0; i < vmrs.length (); i++)
+ find_nearest_common_dominator (vmrs[i], dom_bb);
+
+ if (!dom_bb)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Dominator bb for MRs is not found. "
+ "Skip the case.\n");
+ return 0;
+ }
+ else if (dump_file)
+ fprintf (dump_file, "Dominator bb %d for MRs\n", dom_bb->index);
+
+ split_block (dom_bb, (gimple *) NULL);
+ gimple_stmt_iterator gsi = gsi_last_bb (dom_bb);
+
+ /* Create new inc var. Insert new_var = old_var + step * factor. */
+ decl_map = new tree_map;
+ gcc_assert (comp_mr->stmts[0] && gimple_assign_single_p (comp_mr->stmts[0]));
+ tree inc_var = gimple_assign_lhs (comp_mr->stmts[0]);
+ gimple_seq stmts = NULL;
+ tree var_type = TREE_TYPE (inc_var);
+ enum tree_code inc_code;
+ if (TREE_CODE (var_type) == POINTER_TYPE)
+ inc_code = POINTER_PLUS_EXPR;
+ else
+ inc_code = PLUS_EXPR;
+ tree step = inc_mr->step;
+ unsigned dist_val = tree_to_uhwi (step) * param_ipa_prefetch_distance_factor;
+ tree dist = build_int_cst (TREE_TYPE (step), dist_val);
+ tree new_inc_var = gimple_build (&stmts, inc_code, var_type, inc_var, dist);
+ (*decl_map)[inc_var] = new_inc_var;
+
+ /* Create other new vars. Insert new stmts. */
+ struct walk_stmt_info wi;
+ stmt_set processed_stmts;
+ memref_tree_map mr_new_trees;
+ for (memref_set::const_iterator it = used_mrs.begin ();
+ it != used_mrs.end (); it++)
+ {
+ memref_t *mr = *it;
+ gimple *last_stmt = NULL;
+ if (mr == comp_mr)
+ continue;
+ for (int i = mr->stmts.length () - 1; i >= 0 ; i--)
+ {
+ if (processed_stmts.count (mr->stmts[i]))
+ continue;
+ processed_stmts.insert (mr->stmts[i]);
+ if (dump_file)
+ {
+ fprintf (dump_file, "Copy stmt %d from used MR (%d):\n",
+ i, mr->mr_id);
+ print_gimple_stmt (dump_file, mr->stmts[i], 0);
+ }
+ /* Create a new copy of STMT and duplicate STMT's virtual
+ operands. */
+ gimple *copy = gimple_copy (mr->stmts[i]);
+ gcc_checking_assert (!is_gimple_debug (copy));
+
+ /* Remap all the operands in COPY. */
+ memset (&wi, 0, sizeof (wi));
+ last_stmt = copy;
+ wi.info = copy;
+ walk_gimple_op (copy, remap_gimple_op_r, &wi);
+ if (dump_file)
+ {
+ fprintf (dump_file, "Stmt %d after remap:\n",i);
+ print_gimple_stmt (dump_file, copy, 0);
+ }
+ gimple_seq_add_stmt (&stmts, copy);
+ }
+ gcc_assert (last_stmt);
+ mr_new_trees[mr] = gimple_assign_lhs (last_stmt);
+ if (dump_file)
+ {
+ fprintf (dump_file, "MR (%d) new mem: ", mr->mr_id);
+ print_generic_expr (dump_file, gimple_assign_lhs (last_stmt));
+ fprintf (dump_file, "\n");
+ }
+ }
+ /* On new load check page fault. */
+ /* Insert prefetch instructions. */
+ if (dump_file)
+ fprintf (dump_file, "Evaluate addresses and insert prefetch insn.\n");
+
+ vec<gimple *> pcalls = vNULL;
+ tree local;
+ switch (param_ipa_prefetch_locality)
+ {
+ case 0:
+ local = integer_zero_node;
+ break;
+ case 1:
+ local = integer_one_node;
+ break;
+ case 2:
+ local = build_int_cst (integer_type_node, 2);
+ break;
+ default:
+ case 3:
+ local = integer_three_node;
+ break;
+ }
+ for (unsigned int j = 0; j < vmrs.length (); j++)
+ {
+ memref_t *mr = vmrs[j];
+ /* Don't need to copy the last stmt, since we insert prefetch insn
+ instead of it. */
+ for (int i = mr->stmts.length () - 1; i >= 1 ; i--)
+ {
+ if (processed_stmts.count (mr->stmts[i]))
+ continue;
+ processed_stmts.insert (mr->stmts[i]);
+
+ gimple *copy = gimple_copy (mr->stmts[i]);
+ gcc_checking_assert (!is_gimple_debug (copy));
+
+ /* Remap all the operands in COPY. */
+ memset (&wi, 0, sizeof (wi));
+ wi.info = copy;
+ walk_gimple_op (copy, remap_gimple_op_r, &wi);
+ if (dump_file)
+ {
+ fprintf (dump_file, "Stmt %d after remap:\n",i);
+ print_gimple_stmt (dump_file, copy, 0);
+ }
+ gimple_seq_add_stmt (&stmts, copy);
+ }
+ gimple *last_stmt = mr->stmts[0];
+ gcc_assert (last_stmt);
+ mr_new_trees[mr] = gimple_assign_lhs (last_stmt);
+ tree write_p = mr->is_store ? integer_one_node : integer_zero_node;
+ tree addr = get_mem_ref_address_ssa_name (mr->mem, NULL_TREE);
+ if (decl_map->count (addr))
+ addr = (*decl_map)[addr];
+ last_stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_PREFETCH),
+ 3, addr, write_p, local);
+ pcalls.safe_push (last_stmt);
+ gimple_seq_add_stmt (&stmts, last_stmt);
+ }
+
+ gsi_insert_seq_after (&gsi, stmts, GSI_NEW_STMT);
+ delete decl_map;
+
+ /* Modify cgraph inserting calls to prefetch intrinsics. */
+ for (unsigned i = 0; i < pcalls.length (); i++)
+ create_cgraph_edge (n, pcalls[i]);
+ ipa_update_overall_fn_summary (n);
+
+ return 1;
+}
+
+static int
+insert_prefetch ()
+{
+ int res = 0;
+ cgraph_node *n;
+ FOR_EACH_DEFINED_FUNCTION (n)
+ {
+ function *fn = DECL_STRUCT_FUNCTION (n->decl);
+ if (!optimize_mrs_map->count (fn))
+ continue;
+ if (dump_file)
+ fprintf (dump_file, "Optimize function %s\n", n->dump_name ());
+ push_cfun (DECL_STRUCT_FUNCTION (n->decl));
+ calculate_dominance_info (CDI_DOMINATORS);
+ res |= optimize_function (n, fn);
+ free_dominance_info (CDI_DOMINATORS);
+ pop_cfun ();
+ }
+ return res;
+}
+
+static unsigned int
+ipa_prefetch (void)
+{
+ if (!targetm.have_prefetch ())
+ {
+ if (dump_file)
+ fprintf (dump_file, "Prefetch is not supported by the target.\n");
+ return 0;
+ }
+
+ unsigned int ret = 0;
+ el_map = new edge_in_loop;
+ nl_map = new node_in_loop;
+ icn_map = new node_to_iedge_map;
+ nn_map = new node_to_node_map;
+ tm_map = new tree_memref_map;
+ fmrs_map = new funct_mrs_map;
+ mr_candidate_map = new memref_map;
+ optimize_mrs_map = new funct_mrs_map;
+
+ max_mr_id = 0;
+ /* TODO: check if we really need this init. */
+ if (!builtin_decl_explicit_p (BUILT_IN_PREFETCH))
+ {
+ tree type = build_function_type_list (void_type_node,
+ const_ptr_type_node, NULL_TREE);
+ tree decl = add_builtin_function ("__builtin_prefetch", type,
+ BUILT_IN_PREFETCH, BUILT_IN_NORMAL,
+ NULL, NULL_TREE);
+ DECL_IS_NOVOPS (decl) = true;
+ set_builtin_decl (BUILT_IN_PREFETCH, decl, false);
+ }
+
+ analyse_cgraph ();
+ prepare_indirect_call_info ();
+ propagate_loop_info_in_cgraph ();
+ collect_memory_references ();
+ analyse_loops ();
+
+ /* TODO: implement some specific heuristics. */
+ if (!optimize_mrs_map->empty ())
+ ret = insert_prefetch ();
+
+ delete el_map;
+ delete nl_map;
+ for (node_to_iedge_map::iterator it = icn_map->begin ();
+ it != icn_map->end (); ++it)
+ delete it->second;
+ delete icn_map;
+ for (node_to_node_map::iterator it = nn_map->begin ();
+ it != nn_map->end (); ++it)
+ delete it->second;
+ delete nn_map;
+ for (tree_memref_map::iterator it = tm_map->begin ();
+ it != tm_map->end (); ++it)
+ delete it->second;
+ delete tm_map;
+ for (funct_mrs_map::iterator it = fmrs_map->begin ();
+ it != fmrs_map->end (); ++it)
+ delete it->second;
+ delete fmrs_map;
+ delete mr_candidate_map;
+ delete optimize_mrs_map;
+
+ /* TODO: maybe add other todos. */
+ return ret | TODO_verify_all;
+}
+
+const pass_data pass_data_ipa_prefetch =
+{
+ SIMPLE_IPA_PASS, // type
+ "ipa_prefetch", // name
+ OPTGROUP_NONE, // optinfo_flags
+ TV_IPA_PREFETCH, // tv_id
+ 0, // properties_required
+ 0, // properties_provided
+ 0, // properties_destroyed
+ 0, // todo_flags_start
+ 0, // todo_flags_finish
+};
+
+class pass_ipa_prefetch : public simple_ipa_opt_pass
+{
+public:
+ pass_ipa_prefetch (gcc::context *ctxt)
+ : simple_ipa_opt_pass (pass_data_ipa_prefetch, ctxt)
+ {}
+
+ /* opt_pass methods: */
+ virtual bool gate (function *);
+ virtual unsigned int execute (function *)
+ {
+ return ipa_prefetch ();
+ }
+}; // class pass_ipa_prefetch
+
+bool
+pass_ipa_prefetch::gate (function *)
+{
+ return (optimize >= 3
+ && flag_ipa_prefetch
+ /* Don't bother doing anything if the program has errors. */
+ && !seen_error ()
+ && flag_lto_partition == LTO_PARTITION_ONE
+ /* Only enable struct optimizations in lto or whole_program. */
+ && (in_lto_p || flag_whole_program));
+}
+
+} // anon namespace
+
+simple_ipa_opt_pass *
+make_pass_ipa_prefetch (gcc::context *ctxt)
+{
+ return new pass_ipa_prefetch (ctxt);
+}
diff --git a/gcc/ipa-sra.cc b/gcc/ipa-sra.cc
index 261a72085..5355cf2f4 100644
--- a/gcc/ipa-sra.cc
+++ b/gcc/ipa-sra.cc
@@ -3033,6 +3033,14 @@ process_edge_to_unknown_caller (cgraph_edge *cs)
gcc_checking_assert (from_ifs);
isra_call_summary *csum = call_sums->get (cs);
+ /* TODO: implement better support for call edges inserted after summary
+ collection but before sra wpa invocation. */
+ if (!csum)
+ {
+ csum = call_sums->get_create (cs);
+ csum->m_return_ignored = true;
+ }
+
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Processing an edge to an unknown caller from %s:\n",
cs->caller->dump_name ());
diff --git a/gcc/params.opt b/gcc/params.opt
index 7e5c119cf..5c07e3986 100644
--- a/gcc/params.opt
+++ b/gcc/params.opt
@@ -309,6 +309,14 @@ Maximum pieces that IPA-SRA tracks per formal parameter, as a consequence, also
Common Joined UInteger Var(param_ipa_sra_ptr_growth_factor) Init(2) Param Optimization
Maximum allowed growth of number and total size of new parameters that ipa-sra replaces a pointer to an aggregate with.
+-param=ipa-prefetch-distance-factor=
+Common Joined UInteger Var(param_ipa_prefetch_distance_factor) Init(4) Param Optimization
+The factor represents the number of inductive variable incrementations to evaluate an indirect memory address for IPA prefetch.
+
+-param=ipa-prefetch-locality=
+Common Joined UInteger Var(param_ipa_prefetch_locality) Init(3) Param Optimization
+The flag represents temporal locality values in the following way: 0:pstl1strm, 1:pstl3keep, 2:pstl2keep, 3:pstl1keep.
+
-param=ira-loop-reserved-regs=
Common Joined UInteger Var(param_ira_loop_reserved_regs) Init(2) Param Optimization
The number of registers in each class kept unused by loop invariant motion.
diff --git a/gcc/passes.def b/gcc/passes.def
index b7d4f7b4e..4c1436766 100644
--- a/gcc/passes.def
+++ b/gcc/passes.def
@@ -158,6 +158,7 @@ along with GCC; see the file COPYING3. If not see
NEXT_PASS (pass_ipa_icf);
NEXT_PASS (pass_ipa_devirt);
NEXT_PASS (pass_ipa_icp);
+ NEXT_PASS (pass_ipa_prefetch);
NEXT_PASS (pass_ipa_cp);
NEXT_PASS (pass_ipa_sra);
NEXT_PASS (pass_ipa_cdtor_merge);
diff --git a/gcc/timevar.def b/gcc/timevar.def
index 18a9f62cc..810ae20fd 100644
--- a/gcc/timevar.def
+++ b/gcc/timevar.def
@@ -81,6 +81,7 @@ DEFTIMEVAR (TV_IPA_CONSTANT_PROP , "ipa cp")
DEFTIMEVAR (TV_IPA_INLINING , "ipa inlining heuristics")
DEFTIMEVAR (TV_IPA_FNSPLIT , "ipa function splitting")
DEFTIMEVAR (TV_IPA_COMDATS , "ipa comdats")
+DEFTIMEVAR (TV_IPA_PREFETCH , "ipa prefetch")
DEFTIMEVAR (TV_IPA_STRUCT_REORG , "ipa struct reorg optimization")
DEFTIMEVAR (TV_IPA_OPT , "ipa various optimizations")
DEFTIMEVAR (TV_IPA_LTO_DECOMPRESS , "lto stream decompression")
diff --git a/gcc/tree-pass.h b/gcc/tree-pass.h
index 1733931c3..63f1192ae 100644
--- a/gcc/tree-pass.h
+++ b/gcc/tree-pass.h
@@ -529,6 +529,7 @@ extern ipa_opt_pass_d *make_pass_ipa_icp (gcc::context *ctxt);
extern ipa_opt_pass_d *make_pass_ipa_odr (gcc::context *ctxt);
extern ipa_opt_pass_d *make_pass_ipa_reference (gcc::context *ctxt);
extern ipa_opt_pass_d *make_pass_ipa_pure_const (gcc::context *ctxt);
+extern simple_ipa_opt_pass *make_pass_ipa_prefetch (gcc::context *ctxt);
extern simple_ipa_opt_pass *make_pass_ipa_struct_reorg (gcc::context *ctxt);
extern simple_ipa_opt_pass *make_pass_ipa_pta (gcc::context *ctxt);
extern simple_ipa_opt_pass *make_pass_ipa_tm (gcc::context *ctxt);
--
2.33.0
Reference in New Issue View Git Blame Copy Permalink