1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2018 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
51 #include "stringpool.h"
52 #include "attribs.h"
53 #include "omp-simd-clone.h"
54
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
57
58 static unsigned HOST_WIDE_INT
simd_clone_subparts(tree vectype)59 simd_clone_subparts (tree vectype)
60 {
61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
62 }
63
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
66
67 static struct cgraph_simd_clone *
simd_clone_struct_alloc(int nargs)68 simd_clone_struct_alloc (int nargs)
69 {
70 struct cgraph_simd_clone *clone_info;
71 size_t len = (sizeof (struct cgraph_simd_clone)
72 + nargs * sizeof (struct cgraph_simd_clone_arg));
73 clone_info = (struct cgraph_simd_clone *)
74 ggc_internal_cleared_alloc (len);
75 return clone_info;
76 }
77
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
79
80 static inline void
simd_clone_struct_copy(struct cgraph_simd_clone * to,struct cgraph_simd_clone * from)81 simd_clone_struct_copy (struct cgraph_simd_clone *to,
82 struct cgraph_simd_clone *from)
83 {
84 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
85 + ((from->nargs - from->inbranch)
86 * sizeof (struct cgraph_simd_clone_arg))));
87 }
88
89 /* Return vector of parameter types of function FNDECL. This uses
90 TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
92
93 static vec<tree>
simd_clone_vector_of_formal_parm_types(tree fndecl)94 simd_clone_vector_of_formal_parm_types (tree fndecl)
95 {
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
97 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
98 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
99 unsigned int i;
100 tree arg;
101 FOR_EACH_VEC_ELT (args, i, arg)
102 args[i] = TREE_TYPE (args[i]);
103 return args;
104 }
105
106 /* Given a simd function in NODE, extract the simd specific
107 information from the OMP clauses passed in CLAUSES, and return
108 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
109 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
110 otherwise set to FALSE. */
111
112 static struct cgraph_simd_clone *
simd_clone_clauses_extract(struct cgraph_node * node,tree clauses,bool * inbranch_specified)113 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
114 bool *inbranch_specified)
115 {
116 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
117 tree t;
118 int n;
119 *inbranch_specified = false;
120
121 n = args.length ();
122 if (n > 0 && args.last () == void_type_node)
123 n--;
124
125 /* Allocate one more than needed just in case this is an in-branch
126 clone which will require a mask argument. */
127 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
128 clone_info->nargs = n;
129
130 if (!clauses)
131 goto out;
132
133 clauses = TREE_VALUE (clauses);
134 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
135 goto out;
136
137 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
138 {
139 switch (OMP_CLAUSE_CODE (t))
140 {
141 case OMP_CLAUSE_INBRANCH:
142 clone_info->inbranch = 1;
143 *inbranch_specified = true;
144 break;
145 case OMP_CLAUSE_NOTINBRANCH:
146 clone_info->inbranch = 0;
147 *inbranch_specified = true;
148 break;
149 case OMP_CLAUSE_SIMDLEN:
150 clone_info->simdlen
151 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
152 break;
153 case OMP_CLAUSE_LINEAR:
154 {
155 tree decl = OMP_CLAUSE_DECL (t);
156 tree step = OMP_CLAUSE_LINEAR_STEP (t);
157 int argno = TREE_INT_CST_LOW (decl);
158 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
159 {
160 enum cgraph_simd_clone_arg_type arg_type;
161 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
162 switch (OMP_CLAUSE_LINEAR_KIND (t))
163 {
164 case OMP_CLAUSE_LINEAR_REF:
165 arg_type
166 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
167 break;
168 case OMP_CLAUSE_LINEAR_UVAL:
169 arg_type
170 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
171 break;
172 case OMP_CLAUSE_LINEAR_VAL:
173 case OMP_CLAUSE_LINEAR_DEFAULT:
174 arg_type
175 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
176 break;
177 default:
178 gcc_unreachable ();
179 }
180 else
181 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
182 clone_info->args[argno].arg_type = arg_type;
183 clone_info->args[argno].linear_step = tree_to_shwi (step);
184 gcc_assert (clone_info->args[argno].linear_step >= 0
185 && clone_info->args[argno].linear_step < n);
186 }
187 else
188 {
189 if (POINTER_TYPE_P (args[argno]))
190 step = fold_convert (ssizetype, step);
191 if (!tree_fits_shwi_p (step))
192 {
193 warning_at (OMP_CLAUSE_LOCATION (t), 0,
194 "ignoring large linear step");
195 args.release ();
196 return NULL;
197 }
198 else if (integer_zerop (step))
199 {
200 warning_at (OMP_CLAUSE_LOCATION (t), 0,
201 "ignoring zero linear step");
202 args.release ();
203 return NULL;
204 }
205 else
206 {
207 enum cgraph_simd_clone_arg_type arg_type;
208 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
209 switch (OMP_CLAUSE_LINEAR_KIND (t))
210 {
211 case OMP_CLAUSE_LINEAR_REF:
212 arg_type
213 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
214 break;
215 case OMP_CLAUSE_LINEAR_UVAL:
216 arg_type
217 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
218 break;
219 case OMP_CLAUSE_LINEAR_VAL:
220 case OMP_CLAUSE_LINEAR_DEFAULT:
221 arg_type
222 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
223 break;
224 default:
225 gcc_unreachable ();
226 }
227 else
228 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
229 clone_info->args[argno].arg_type = arg_type;
230 clone_info->args[argno].linear_step = tree_to_shwi (step);
231 }
232 }
233 break;
234 }
235 case OMP_CLAUSE_UNIFORM:
236 {
237 tree decl = OMP_CLAUSE_DECL (t);
238 int argno = tree_to_uhwi (decl);
239 clone_info->args[argno].arg_type
240 = SIMD_CLONE_ARG_TYPE_UNIFORM;
241 break;
242 }
243 case OMP_CLAUSE_ALIGNED:
244 {
245 /* Ignore aligned (x) for declare simd, for the ABI we really
246 need an alignment specified. */
247 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE)
248 break;
249 tree decl = OMP_CLAUSE_DECL (t);
250 int argno = tree_to_uhwi (decl);
251 clone_info->args[argno].alignment
252 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
253 break;
254 }
255 default:
256 break;
257 }
258 }
259
260 out:
261 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
262 {
263 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
264 "ignoring %<#pragma omp declare simd%> on function "
265 "with %<_Atomic%> qualified return type");
266 args.release ();
267 return NULL;
268 }
269
270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
271 if (TYPE_ATOMIC (args[argno])
272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
273 {
274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
277 args.release ();
278 return NULL;
279 }
280
281 args.release ();
282 return clone_info;
283 }
284
285 /* Given a SIMD clone in NODE, calculate the characteristic data
286 type and return the coresponding type. The characteristic data
287 type is computed as described in the Intel Vector ABI. */
288
289 static tree
simd_clone_compute_base_data_type(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)290 simd_clone_compute_base_data_type (struct cgraph_node *node,
291 struct cgraph_simd_clone *clone_info)
292 {
293 tree type = integer_type_node;
294 tree fndecl = node->decl;
295
296 /* a) For non-void function, the characteristic data type is the
297 return type. */
298 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
299 type = TREE_TYPE (TREE_TYPE (fndecl));
300
301 /* b) If the function has any non-uniform, non-linear parameters,
302 then the characteristic data type is the type of the first
303 such parameter. */
304 else
305 {
306 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
307 for (unsigned int i = 0; i < clone_info->nargs; ++i)
308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
309 {
310 type = map[i];
311 break;
312 }
313 map.release ();
314 }
315
316 /* c) If the characteristic data type determined by a) or b) above
317 is struct, union, or class type which is pass-by-value (except
318 for the type that maps to the built-in complex data type), the
319 characteristic data type is int. */
320 if (RECORD_OR_UNION_TYPE_P (type)
321 && !aggregate_value_p (type, NULL)
322 && TREE_CODE (type) != COMPLEX_TYPE)
323 return integer_type_node;
324
325 /* d) If none of the above three classes is applicable, the
326 characteristic data type is int. */
327
328 return type;
329
330 /* e) For Intel Xeon Phi native and offload compilation, if the
331 resulting characteristic data type is 8-bit or 16-bit integer
332 data type, the characteristic data type is int. */
333 /* Well, we don't handle Xeon Phi yet. */
334 }
335
336 static tree
simd_clone_mangle(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)337 simd_clone_mangle (struct cgraph_node *node,
338 struct cgraph_simd_clone *clone_info)
339 {
340 char vecsize_mangle = clone_info->vecsize_mangle;
341 char mask = clone_info->inbranch ? 'M' : 'N';
342 unsigned int simdlen = clone_info->simdlen;
343 unsigned int n;
344 pretty_printer pp;
345
346 gcc_assert (vecsize_mangle && simdlen);
347
348 pp_string (&pp, "_ZGV");
349 pp_character (&pp, vecsize_mangle);
350 pp_character (&pp, mask);
351 pp_decimal_int (&pp, simdlen);
352
353 for (n = 0; n < clone_info->nargs; ++n)
354 {
355 struct cgraph_simd_clone_arg arg = clone_info->args[n];
356
357 switch (arg.arg_type)
358 {
359 case SIMD_CLONE_ARG_TYPE_UNIFORM:
360 pp_character (&pp, 'u');
361 break;
362 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
363 pp_character (&pp, 'l');
364 goto mangle_linear;
365 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
366 pp_character (&pp, 'R');
367 goto mangle_linear;
368 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
369 pp_character (&pp, 'L');
370 goto mangle_linear;
371 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
372 pp_character (&pp, 'U');
373 goto mangle_linear;
374 mangle_linear:
375 gcc_assert (arg.linear_step != 0);
376 if (arg.linear_step > 1)
377 pp_unsigned_wide_integer (&pp, arg.linear_step);
378 else if (arg.linear_step < 0)
379 {
380 pp_character (&pp, 'n');
381 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
382 arg.linear_step));
383 }
384 break;
385 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
386 pp_string (&pp, "ls");
387 pp_unsigned_wide_integer (&pp, arg.linear_step);
388 break;
389 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
390 pp_string (&pp, "Rs");
391 pp_unsigned_wide_integer (&pp, arg.linear_step);
392 break;
393 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
394 pp_string (&pp, "Ls");
395 pp_unsigned_wide_integer (&pp, arg.linear_step);
396 break;
397 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
398 pp_string (&pp, "Us");
399 pp_unsigned_wide_integer (&pp, arg.linear_step);
400 break;
401 default:
402 pp_character (&pp, 'v');
403 }
404 if (arg.alignment)
405 {
406 pp_character (&pp, 'a');
407 pp_decimal_int (&pp, arg.alignment);
408 }
409 }
410
411 pp_underscore (&pp);
412 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
413 if (*str == '*')
414 ++str;
415 pp_string (&pp, str);
416 str = pp_formatted_text (&pp);
417
418 /* If there already is a SIMD clone with the same mangled name, don't
419 add another one. This can happen e.g. for
420 #pragma omp declare simd
421 #pragma omp declare simd simdlen(8)
422 int foo (int, int);
423 if the simdlen is assumed to be 8 for the first one, etc. */
424 for (struct cgraph_node *clone = node->simd_clones; clone;
425 clone = clone->simdclone->next_clone)
426 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
427 return NULL_TREE;
428
429 return get_identifier (str);
430 }
431
432 /* Create a simd clone of OLD_NODE and return it. */
433
434 static struct cgraph_node *
simd_clone_create(struct cgraph_node * old_node)435 simd_clone_create (struct cgraph_node *old_node)
436 {
437 struct cgraph_node *new_node;
438 if (old_node->definition)
439 {
440 if (!old_node->has_gimple_body_p ())
441 return NULL;
442 old_node->get_body ();
443 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
444 false, NULL, NULL,
445 "simdclone");
446 }
447 else
448 {
449 tree old_decl = old_node->decl;
450 tree new_decl = copy_node (old_node->decl);
451 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
452 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
453 SET_DECL_RTL (new_decl, NULL);
454 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
455 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
456 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
457 if (old_node->in_other_partition)
458 new_node->in_other_partition = 1;
459 }
460 if (new_node == NULL)
461 return new_node;
462
463 DECL_BUILT_IN_CLASS (new_node->decl) = NOT_BUILT_IN;
464 DECL_FUNCTION_CODE (new_node->decl) = (enum built_in_function) 0;
465 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
466 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
467 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
468 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
469 DECL_VISIBILITY_SPECIFIED (new_node->decl)
470 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
471 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
472 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
473 if (DECL_ONE_ONLY (old_node->decl))
474 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
475
476 /* The method cgraph_version_clone_with_body () will force the new
477 symbol local. Undo this, and inherit external visibility from
478 the old node. */
479 new_node->local.local = old_node->local.local;
480 new_node->externally_visible = old_node->externally_visible;
481
482 return new_node;
483 }
484
485 /* Adjust the return type of the given function to its appropriate
486 vector counterpart. Returns a simd array to be used throughout the
487 function as a return value. */
488
489 static tree
simd_clone_adjust_return_type(struct cgraph_node * node)490 simd_clone_adjust_return_type (struct cgraph_node *node)
491 {
492 tree fndecl = node->decl;
493 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
494 unsigned int veclen;
495 tree t;
496
497 /* Adjust the function return type. */
498 if (orig_rettype == void_type_node)
499 return NULL_TREE;
500 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
501 t = TREE_TYPE (TREE_TYPE (fndecl));
502 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
503 veclen = node->simdclone->vecsize_int;
504 else
505 veclen = node->simdclone->vecsize_float;
506 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t));
507 if (veclen > node->simdclone->simdlen)
508 veclen = node->simdclone->simdlen;
509 if (POINTER_TYPE_P (t))
510 t = pointer_sized_int_node;
511 if (veclen == node->simdclone->simdlen)
512 t = build_vector_type (t, node->simdclone->simdlen);
513 else
514 {
515 t = build_vector_type (t, veclen);
516 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
517 }
518 TREE_TYPE (TREE_TYPE (fndecl)) = t;
519 if (!node->definition)
520 return NULL_TREE;
521
522 t = DECL_RESULT (fndecl);
523 /* Adjust the DECL_RESULT. */
524 gcc_assert (TREE_TYPE (t) != void_type_node);
525 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
526 relayout_decl (t);
527
528 tree atype = build_array_type_nelts (orig_rettype,
529 node->simdclone->simdlen);
530 if (veclen != node->simdclone->simdlen)
531 return build1 (VIEW_CONVERT_EXPR, atype, t);
532
533 /* Set up a SIMD array to use as the return value. */
534 tree retval = create_tmp_var_raw (atype, "retval");
535 gimple_add_tmp_var (retval);
536 return retval;
537 }
538
539 /* Each vector argument has a corresponding array to be used locally
540 as part of the eventual loop. Create such temporary array and
541 return it.
542
543 PREFIX is the prefix to be used for the temporary.
544
545 TYPE is the inner element type.
546
547 SIMDLEN is the number of elements. */
548
549 static tree
create_tmp_simd_array(const char * prefix,tree type,int simdlen)550 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
551 {
552 tree atype = build_array_type_nelts (type, simdlen);
553 tree avar = create_tmp_var_raw (atype, prefix);
554 gimple_add_tmp_var (avar);
555 return avar;
556 }
557
558 /* Modify the function argument types to their corresponding vector
559 counterparts if appropriate. Also, create one array for each simd
560 argument to be used locally when using the function arguments as
561 part of the loop.
562
563 NODE is the function whose arguments are to be adjusted.
564
565 Returns an adjustment vector that will be filled describing how the
566 argument types will be adjusted. */
567
568 static ipa_parm_adjustment_vec
simd_clone_adjust_argument_types(struct cgraph_node * node)569 simd_clone_adjust_argument_types (struct cgraph_node *node)
570 {
571 vec<tree> args;
572 ipa_parm_adjustment_vec adjustments;
573
574 if (node->definition)
575 args = ipa_get_vector_of_formal_parms (node->decl);
576 else
577 args = simd_clone_vector_of_formal_parm_types (node->decl);
578 adjustments.create (args.length ());
579 unsigned i, j, veclen;
580 struct ipa_parm_adjustment adj;
581 struct cgraph_simd_clone *sc = node->simdclone;
582
583 for (i = 0; i < sc->nargs; ++i)
584 {
585 memset (&adj, 0, sizeof (adj));
586 tree parm = args[i];
587 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
588 adj.base_index = i;
589 adj.base = parm;
590
591 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
592 sc->args[i].orig_type = parm_type;
593
594 switch (sc->args[i].arg_type)
595 {
596 default:
597 /* No adjustment necessary for scalar arguments. */
598 adj.op = IPA_PARM_OP_COPY;
599 break;
600 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
601 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
602 if (node->definition)
603 sc->args[i].simd_array
604 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
605 TREE_TYPE (parm_type),
606 sc->simdlen);
607 adj.op = IPA_PARM_OP_COPY;
608 break;
609 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
610 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
611 case SIMD_CLONE_ARG_TYPE_VECTOR:
612 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
613 veclen = sc->vecsize_int;
614 else
615 veclen = sc->vecsize_float;
616 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type));
617 if (veclen > sc->simdlen)
618 veclen = sc->simdlen;
619 adj.arg_prefix = "simd";
620 if (POINTER_TYPE_P (parm_type))
621 adj.type = build_vector_type (pointer_sized_int_node, veclen);
622 else
623 adj.type = build_vector_type (parm_type, veclen);
624 sc->args[i].vector_type = adj.type;
625 for (j = veclen; j < sc->simdlen; j += veclen)
626 {
627 adjustments.safe_push (adj);
628 if (j == veclen)
629 {
630 memset (&adj, 0, sizeof (adj));
631 adj.op = IPA_PARM_OP_NEW;
632 adj.arg_prefix = "simd";
633 adj.base_index = i;
634 adj.type = sc->args[i].vector_type;
635 }
636 }
637
638 if (node->definition)
639 sc->args[i].simd_array
640 = create_tmp_simd_array (DECL_NAME (parm)
641 ? IDENTIFIER_POINTER (DECL_NAME (parm))
642 : NULL, parm_type, sc->simdlen);
643 }
644 adjustments.safe_push (adj);
645 }
646
647 if (sc->inbranch)
648 {
649 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
650
651 memset (&adj, 0, sizeof (adj));
652 adj.op = IPA_PARM_OP_NEW;
653 adj.arg_prefix = "mask";
654
655 adj.base_index = i;
656 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
657 veclen = sc->vecsize_int;
658 else
659 veclen = sc->vecsize_float;
660 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
661 if (veclen > sc->simdlen)
662 veclen = sc->simdlen;
663 if (sc->mask_mode != VOIDmode)
664 adj.type
665 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
666 else if (POINTER_TYPE_P (base_type))
667 adj.type = build_vector_type (pointer_sized_int_node, veclen);
668 else
669 adj.type = build_vector_type (base_type, veclen);
670 adjustments.safe_push (adj);
671
672 for (j = veclen; j < sc->simdlen; j += veclen)
673 adjustments.safe_push (adj);
674
675 /* We have previously allocated one extra entry for the mask. Use
676 it and fill it. */
677 sc->nargs++;
678 if (sc->mask_mode != VOIDmode)
679 base_type = boolean_type_node;
680 if (node->definition)
681 {
682 sc->args[i].orig_arg
683 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
684 if (sc->mask_mode == VOIDmode)
685 sc->args[i].simd_array
686 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
687 else if (veclen < sc->simdlen)
688 sc->args[i].simd_array
689 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
690 else
691 sc->args[i].simd_array = NULL_TREE;
692 }
693 sc->args[i].orig_type = base_type;
694 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
695 }
696
697 if (node->definition)
698 ipa_modify_formal_parameters (node->decl, adjustments);
699 else
700 {
701 tree new_arg_types = NULL_TREE, new_reversed;
702 bool last_parm_void = false;
703 if (args.length () > 0 && args.last () == void_type_node)
704 last_parm_void = true;
705
706 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
707 j = adjustments.length ();
708 for (i = 0; i < j; i++)
709 {
710 struct ipa_parm_adjustment *adj = &adjustments[i];
711 tree ptype;
712 if (adj->op == IPA_PARM_OP_COPY)
713 ptype = args[adj->base_index];
714 else
715 ptype = adj->type;
716 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
717 }
718 new_reversed = nreverse (new_arg_types);
719 if (last_parm_void)
720 {
721 if (new_reversed)
722 TREE_CHAIN (new_arg_types) = void_list_node;
723 else
724 new_reversed = void_list_node;
725 }
726
727 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
728 TYPE_ARG_TYPES (new_type) = new_reversed;
729 TREE_TYPE (node->decl) = new_type;
730
731 adjustments.release ();
732 }
733 args.release ();
734 return adjustments;
735 }
736
737 /* Initialize and copy the function arguments in NODE to their
738 corresponding local simd arrays. Returns a fresh gimple_seq with
739 the instruction sequence generated. */
740
741 static gimple_seq
simd_clone_init_simd_arrays(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments)742 simd_clone_init_simd_arrays (struct cgraph_node *node,
743 ipa_parm_adjustment_vec adjustments)
744 {
745 gimple_seq seq = NULL;
746 unsigned i = 0, j = 0, k;
747
748 for (tree arg = DECL_ARGUMENTS (node->decl);
749 arg;
750 arg = DECL_CHAIN (arg), i++, j++)
751 {
752 if (adjustments[j].op == IPA_PARM_OP_COPY
753 || POINTER_TYPE_P (TREE_TYPE (arg)))
754 continue;
755
756 node->simdclone->args[i].vector_arg = arg;
757
758 tree array = node->simdclone->args[i].simd_array;
759 if (node->simdclone->mask_mode != VOIDmode
760 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
761 {
762 if (array == NULL_TREE)
763 continue;
764 unsigned int l
765 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
766 for (k = 0; k <= l; k++)
767 {
768 if (k)
769 {
770 arg = DECL_CHAIN (arg);
771 j++;
772 }
773 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
774 array, size_int (k), NULL, NULL);
775 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
776 gimplify_and_add (t, &seq);
777 }
778 continue;
779 }
780 if (simd_clone_subparts (TREE_TYPE (arg)) == node->simdclone->simdlen)
781 {
782 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
783 tree ptr = build_fold_addr_expr (array);
784 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
785 build_int_cst (ptype, 0));
786 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
787 gimplify_and_add (t, &seq);
788 }
789 else
790 {
791 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg));
792 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
793 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
794 {
795 tree ptr = build_fold_addr_expr (array);
796 int elemsize;
797 if (k)
798 {
799 arg = DECL_CHAIN (arg);
800 j++;
801 }
802 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
803 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
804 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
805 build_int_cst (ptype, k * elemsize));
806 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
807 gimplify_and_add (t, &seq);
808 }
809 }
810 }
811 return seq;
812 }
813
814 /* Callback info for ipa_simd_modify_stmt_ops below. */
815
816 struct modify_stmt_info {
817 ipa_parm_adjustment_vec adjustments;
818 gimple *stmt;
819 /* True if the parent statement was modified by
820 ipa_simd_modify_stmt_ops. */
821 bool modified;
822 };
823
824 /* Callback for walk_gimple_op.
825
826 Adjust operands from a given statement as specified in the
827 adjustments vector in the callback data. */
828
829 static tree
ipa_simd_modify_stmt_ops(tree * tp,int * walk_subtrees,void * data)830 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
831 {
832 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
833 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
834 tree *orig_tp = tp;
835 if (TREE_CODE (*tp) == ADDR_EXPR)
836 tp = &TREE_OPERAND (*tp, 0);
837 struct ipa_parm_adjustment *cand = NULL;
838 if (TREE_CODE (*tp) == PARM_DECL)
839 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
840 else
841 {
842 if (TYPE_P (*tp))
843 *walk_subtrees = 0;
844 }
845
846 tree repl = NULL_TREE;
847 if (cand)
848 repl = unshare_expr (cand->new_decl);
849 else
850 {
851 if (tp != orig_tp)
852 {
853 *walk_subtrees = 0;
854 bool modified = info->modified;
855 info->modified = false;
856 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
857 if (!info->modified)
858 {
859 info->modified = modified;
860 return NULL_TREE;
861 }
862 info->modified = modified;
863 repl = *tp;
864 }
865 else
866 return NULL_TREE;
867 }
868
869 if (tp != orig_tp)
870 {
871 repl = build_fold_addr_expr (repl);
872 gimple *stmt;
873 if (is_gimple_debug (info->stmt))
874 {
875 tree vexpr = make_node (DEBUG_EXPR_DECL);
876 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
877 DECL_ARTIFICIAL (vexpr) = 1;
878 TREE_TYPE (vexpr) = TREE_TYPE (repl);
879 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
880 repl = vexpr;
881 }
882 else
883 {
884 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
885 repl = gimple_assign_lhs (stmt);
886 }
887 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
888 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
889 *orig_tp = repl;
890 }
891 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
892 {
893 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
894 *tp = vce;
895 }
896 else
897 *tp = repl;
898
899 info->modified = true;
900 return NULL_TREE;
901 }
902
903 /* Traverse the function body and perform all modifications as
904 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
905 modified such that the replacement/reduction value will now be an
906 offset into the corresponding simd_array.
907
908 This function will replace all function argument uses with their
909 corresponding simd array elements, and ajust the return values
910 accordingly. */
911
912 static void
ipa_simd_modify_function_body(struct cgraph_node * node,ipa_parm_adjustment_vec adjustments,tree retval_array,tree iter)913 ipa_simd_modify_function_body (struct cgraph_node *node,
914 ipa_parm_adjustment_vec adjustments,
915 tree retval_array, tree iter)
916 {
917 basic_block bb;
918 unsigned int i, j, l;
919
920 /* Re-use the adjustments array, but this time use it to replace
921 every function argument use to an offset into the corresponding
922 simd_array. */
923 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
924 {
925 if (!node->simdclone->args[i].vector_arg)
926 continue;
927
928 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
929 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
930 adjustments[j].new_decl
931 = build4 (ARRAY_REF,
932 basetype,
933 node->simdclone->args[i].simd_array,
934 iter,
935 NULL_TREE, NULL_TREE);
936 if (adjustments[j].op == IPA_PARM_OP_NONE
937 && simd_clone_subparts (vectype) < node->simdclone->simdlen)
938 j += node->simdclone->simdlen / simd_clone_subparts (vectype) - 1;
939 }
940
941 l = adjustments.length ();
942 tree name;
943
944 FOR_EACH_SSA_NAME (i, name, cfun)
945 {
946 if (SSA_NAME_VAR (name)
947 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
948 {
949 for (j = 0; j < l; j++)
950 if (SSA_NAME_VAR (name) == adjustments[j].base
951 && adjustments[j].new_decl)
952 {
953 tree base_var;
954 if (adjustments[j].new_ssa_base == NULL_TREE)
955 {
956 base_var
957 = copy_var_decl (adjustments[j].base,
958 DECL_NAME (adjustments[j].base),
959 TREE_TYPE (adjustments[j].base));
960 adjustments[j].new_ssa_base = base_var;
961 }
962 else
963 base_var = adjustments[j].new_ssa_base;
964 if (SSA_NAME_IS_DEFAULT_DEF (name))
965 {
966 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
967 gimple_stmt_iterator gsi = gsi_after_labels (bb);
968 tree new_decl = unshare_expr (adjustments[j].new_decl);
969 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
970 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
971 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
972 gimple *stmt = gimple_build_assign (name, new_decl);
973 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
974 }
975 else
976 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
977 }
978 }
979 }
980
981 struct modify_stmt_info info;
982 info.adjustments = adjustments;
983
984 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
985 {
986 gimple_stmt_iterator gsi;
987
988 gsi = gsi_start_bb (bb);
989 while (!gsi_end_p (gsi))
990 {
991 gimple *stmt = gsi_stmt (gsi);
992 info.stmt = stmt;
993 struct walk_stmt_info wi;
994
995 memset (&wi, 0, sizeof (wi));
996 info.modified = false;
997 wi.info = &info;
998 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
999
1000 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
1001 {
1002 tree retval = gimple_return_retval (return_stmt);
1003 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
1004 e->flags |= EDGE_FALLTHRU;
1005 if (!retval)
1006 {
1007 gsi_remove (&gsi, true);
1008 continue;
1009 }
1010
1011 /* Replace `return foo' with `retval_array[iter] = foo'. */
1012 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1013 retval_array, iter, NULL, NULL);
1014 stmt = gimple_build_assign (ref, retval);
1015 gsi_replace (&gsi, stmt, true);
1016 info.modified = true;
1017 }
1018
1019 if (info.modified)
1020 {
1021 update_stmt (stmt);
1022 /* If the above changed the var of a debug bind into something
1023 different, remove the debug stmt. We could also for all the
1024 replaced parameters add VAR_DECLs for debug info purposes,
1025 add debug stmts for those to be the simd array accesses and
1026 replace debug stmt var operand with that var. Debugging of
1027 vectorized loops doesn't work too well, so don't bother for
1028 now. */
1029 if ((gimple_debug_bind_p (stmt)
1030 && !DECL_P (gimple_debug_bind_get_var (stmt)))
1031 || (gimple_debug_source_bind_p (stmt)
1032 && !DECL_P (gimple_debug_source_bind_get_var (stmt))))
1033 {
1034 gsi_remove (&gsi, true);
1035 continue;
1036 }
1037 if (maybe_clean_eh_stmt (stmt))
1038 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1039 }
1040 gsi_next (&gsi);
1041 }
1042 }
1043 }
1044
1045 /* Helper function of simd_clone_adjust, return linear step addend
1046 of Ith argument. */
1047
1048 static tree
simd_clone_linear_addend(struct cgraph_node * node,unsigned int i,tree addtype,basic_block entry_bb)1049 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1050 tree addtype, basic_block entry_bb)
1051 {
1052 tree ptype = NULL_TREE;
1053 switch (node->simdclone->args[i].arg_type)
1054 {
1055 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1056 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1057 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1058 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1059 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1060 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1061 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1062 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1063 break;
1064 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1065 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1066 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1067 break;
1068 default:
1069 gcc_unreachable ();
1070 }
1071
1072 unsigned int idx = node->simdclone->args[i].linear_step;
1073 tree arg = node->simdclone->args[idx].orig_arg;
1074 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1075 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1076 gimple *g;
1077 tree ret;
1078 if (is_gimple_reg (arg))
1079 ret = get_or_create_ssa_default_def (cfun, arg);
1080 else
1081 {
1082 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1083 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1084 ret = gimple_assign_lhs (g);
1085 }
1086 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1087 {
1088 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1089 build_simple_mem_ref (ret));
1090 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1091 ret = gimple_assign_lhs (g);
1092 }
1093 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1094 {
1095 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1096 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1097 ret = gimple_assign_lhs (g);
1098 }
1099 if (POINTER_TYPE_P (ptype))
1100 {
1101 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1102 if (size && TREE_CODE (size) == INTEGER_CST)
1103 {
1104 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1105 ret, fold_convert (addtype, size));
1106 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1107 ret = gimple_assign_lhs (g);
1108 }
1109 }
1110 return ret;
1111 }
1112
1113 /* Adjust the argument types in NODE to their appropriate vector
1114 counterparts. */
1115
1116 static void
simd_clone_adjust(struct cgraph_node * node)1117 simd_clone_adjust (struct cgraph_node *node)
1118 {
1119 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1120
1121 targetm.simd_clone.adjust (node);
1122
1123 tree retval = simd_clone_adjust_return_type (node);
1124 ipa_parm_adjustment_vec adjustments
1125 = simd_clone_adjust_argument_types (node);
1126
1127 push_gimplify_context ();
1128
1129 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1130
1131 /* Adjust all uses of vector arguments accordingly. Adjust all
1132 return values accordingly. */
1133 tree iter = create_tmp_var (unsigned_type_node, "iter");
1134 tree iter1 = make_ssa_name (iter);
1135 tree iter2 = NULL_TREE;
1136 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1137 adjustments.release ();
1138
1139 /* Initialize the iteration variable. */
1140 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1141 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1142 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1143 /* Insert the SIMD array and iv initialization at function
1144 entry. */
1145 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1146
1147 pop_gimplify_context (NULL);
1148
1149 gimple *g;
1150 basic_block incr_bb = NULL;
1151 struct loop *loop = NULL;
1152
1153 /* Create a new BB right before the original exit BB, to hold the
1154 iteration increment and the condition/branch. */
1155 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1156 {
1157 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1158 incr_bb = create_empty_bb (orig_exit);
1159 incr_bb->count = profile_count::zero ();
1160 add_bb_to_loop (incr_bb, body_bb->loop_father);
1161 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1162 {
1163 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1164 redirect_edge_succ (e, incr_bb);
1165 incr_bb->count += e->count ();
1166 }
1167 }
1168 else if (node->simdclone->inbranch)
1169 {
1170 incr_bb = create_empty_bb (entry_bb);
1171 incr_bb->count = profile_count::zero ();
1172 add_bb_to_loop (incr_bb, body_bb->loop_father);
1173 }
1174
1175 if (incr_bb)
1176 {
1177 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1178 gsi = gsi_last_bb (incr_bb);
1179 iter2 = make_ssa_name (iter);
1180 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1181 build_int_cst (unsigned_type_node, 1));
1182 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1183
1184 /* Mostly annotate the loop for the vectorizer (the rest is done
1185 below). */
1186 loop = alloc_loop ();
1187 cfun->has_force_vectorize_loops = true;
1188 loop->safelen = node->simdclone->simdlen;
1189 loop->force_vectorize = true;
1190 loop->header = body_bb;
1191 }
1192
1193 /* Branch around the body if the mask applies. */
1194 if (node->simdclone->inbranch)
1195 {
1196 gsi = gsi_last_bb (loop->header);
1197 tree mask_array
1198 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1199 tree mask;
1200 if (node->simdclone->mask_mode != VOIDmode)
1201 {
1202 tree shift_cnt;
1203 if (mask_array == NULL_TREE)
1204 {
1205 tree arg = node->simdclone->args[node->simdclone->nargs
1206 - 1].vector_arg;
1207 mask = get_or_create_ssa_default_def (cfun, arg);
1208 shift_cnt = iter1;
1209 }
1210 else
1211 {
1212 tree maskt = TREE_TYPE (mask_array);
1213 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1214 c = node->simdclone->simdlen / (c + 1);
1215 int s = exact_log2 (c);
1216 gcc_assert (s > 0);
1217 c--;
1218 tree idx = make_ssa_name (TREE_TYPE (iter1));
1219 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1220 build_int_cst (NULL_TREE, s));
1221 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1222 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1223 tree aref = build4 (ARRAY_REF,
1224 TREE_TYPE (TREE_TYPE (mask_array)),
1225 mask_array, idx, NULL, NULL);
1226 g = gimple_build_assign (mask, aref);
1227 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1228 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1229 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1230 build_int_cst (TREE_TYPE (iter1), c));
1231 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1232 }
1233 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1234 RSHIFT_EXPR, mask, shift_cnt);
1235 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1236 mask = gimple_assign_lhs (g);
1237 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1238 BIT_AND_EXPR, mask,
1239 build_int_cst (TREE_TYPE (mask), 1));
1240 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1241 mask = gimple_assign_lhs (g);
1242 }
1243 else
1244 {
1245 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1246 tree aref = build4 (ARRAY_REF,
1247 TREE_TYPE (TREE_TYPE (mask_array)),
1248 mask_array, iter1, NULL, NULL);
1249 g = gimple_build_assign (mask, aref);
1250 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1251 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1252 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1253 {
1254 aref = build1 (VIEW_CONVERT_EXPR,
1255 build_nonstandard_integer_type (bitsize, 0),
1256 mask);
1257 mask = make_ssa_name (TREE_TYPE (aref));
1258 g = gimple_build_assign (mask, aref);
1259 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1260 }
1261 }
1262
1263 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1264 NULL, NULL);
1265 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1266 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1267 e->probability = profile_probability::unlikely ().guessed ();
1268 incr_bb->count += e->count ();
1269 edge fallthru = FALLTHRU_EDGE (loop->header);
1270 fallthru->flags = EDGE_FALSE_VALUE;
1271 fallthru->probability = profile_probability::likely ().guessed ();
1272 }
1273
1274 basic_block latch_bb = NULL;
1275 basic_block new_exit_bb = NULL;
1276
1277 /* Generate the condition. */
1278 if (incr_bb)
1279 {
1280 gsi = gsi_last_bb (incr_bb);
1281 g = gimple_build_cond (LT_EXPR, iter2,
1282 build_int_cst (unsigned_type_node,
1283 node->simdclone->simdlen),
1284 NULL, NULL);
1285 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1286 edge e = split_block (incr_bb, gsi_stmt (gsi));
1287 latch_bb = e->dest;
1288 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1289 loop->latch = latch_bb;
1290
1291 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1292
1293 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1294
1295 /* FIXME: Do we need to distribute probabilities for the conditional? */
1296 new_e->probability = profile_probability::guessed_never ();
1297 /* The successor of incr_bb is already pointing to latch_bb; just
1298 change the flags.
1299 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1300 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1301 }
1302
1303 gphi *phi = create_phi_node (iter1, body_bb);
1304 edge preheader_edge = find_edge (entry_bb, body_bb);
1305 edge latch_edge = NULL;
1306 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1307 UNKNOWN_LOCATION);
1308 if (incr_bb)
1309 {
1310 latch_edge = single_succ_edge (latch_bb);
1311 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1312
1313 /* Generate the new return. */
1314 gsi = gsi_last_bb (new_exit_bb);
1315 if (retval
1316 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1317 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1318 retval = TREE_OPERAND (retval, 0);
1319 else if (retval)
1320 {
1321 retval = build1 (VIEW_CONVERT_EXPR,
1322 TREE_TYPE (TREE_TYPE (node->decl)),
1323 retval);
1324 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1325 false, GSI_CONTINUE_LINKING);
1326 }
1327 g = gimple_build_return (retval);
1328 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1329 }
1330
1331 /* Handle aligned clauses by replacing default defs of the aligned
1332 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1333 lhs. Handle linear by adding PHIs. */
1334 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1335 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1336 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1337 || !is_gimple_reg_type
1338 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1339 {
1340 tree orig_arg = node->simdclone->args[i].orig_arg;
1341 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1342 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1343 else
1344 {
1345 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1346 gimple_add_tmp_var (iter1);
1347 }
1348 gsi = gsi_after_labels (entry_bb);
1349 g = gimple_build_assign (iter1, orig_arg);
1350 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1351 gsi = gsi_after_labels (body_bb);
1352 g = gimple_build_assign (orig_arg, iter1);
1353 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1354 }
1355 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1356 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1357 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1358 == REFERENCE_TYPE
1359 && TREE_ADDRESSABLE
1360 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1361 {
1362 tree orig_arg = node->simdclone->args[i].orig_arg;
1363 tree def = ssa_default_def (cfun, orig_arg);
1364 if (def && !has_zero_uses (def))
1365 {
1366 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1367 gimple_add_tmp_var (iter1);
1368 gsi = gsi_after_labels (entry_bb);
1369 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1370 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1371 gsi = gsi_after_labels (body_bb);
1372 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1373 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1374 }
1375 }
1376 else if (node->simdclone->args[i].alignment
1377 && node->simdclone->args[i].arg_type
1378 == SIMD_CLONE_ARG_TYPE_UNIFORM
1379 && (node->simdclone->args[i].alignment
1380 & (node->simdclone->args[i].alignment - 1)) == 0
1381 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1382 == POINTER_TYPE)
1383 {
1384 unsigned int alignment = node->simdclone->args[i].alignment;
1385 tree orig_arg = node->simdclone->args[i].orig_arg;
1386 tree def = ssa_default_def (cfun, orig_arg);
1387 if (def && !has_zero_uses (def))
1388 {
1389 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1390 gimple_seq seq = NULL;
1391 bool need_cvt = false;
1392 gcall *call
1393 = gimple_build_call (fn, 2, def, size_int (alignment));
1394 g = call;
1395 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1396 ptr_type_node))
1397 need_cvt = true;
1398 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1399 gimple_call_set_lhs (g, t);
1400 gimple_seq_add_stmt_without_update (&seq, g);
1401 if (need_cvt)
1402 {
1403 t = make_ssa_name (orig_arg);
1404 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1405 gimple_seq_add_stmt_without_update (&seq, g);
1406 }
1407 gsi_insert_seq_on_edge_immediate
1408 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1409
1410 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1411 node->create_edge (cgraph_node::get_create (fn),
1412 call, entry_bb->count);
1413
1414 imm_use_iterator iter;
1415 use_operand_p use_p;
1416 gimple *use_stmt;
1417 tree repl = gimple_get_lhs (g);
1418 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1419 if (is_gimple_debug (use_stmt) || use_stmt == call)
1420 continue;
1421 else
1422 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1423 SET_USE (use_p, repl);
1424 }
1425 }
1426 else if ((node->simdclone->args[i].arg_type
1427 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1428 || (node->simdclone->args[i].arg_type
1429 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1430 || (node->simdclone->args[i].arg_type
1431 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1432 || (node->simdclone->args[i].arg_type
1433 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1434 {
1435 tree orig_arg = node->simdclone->args[i].orig_arg;
1436 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1437 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1438 tree def = NULL_TREE;
1439 if (TREE_ADDRESSABLE (orig_arg))
1440 {
1441 def = make_ssa_name (TREE_TYPE (orig_arg));
1442 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1443 if (incr_bb)
1444 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1445 gsi = gsi_after_labels (entry_bb);
1446 g = gimple_build_assign (def, orig_arg);
1447 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1448 }
1449 else
1450 {
1451 def = ssa_default_def (cfun, orig_arg);
1452 if (!def || has_zero_uses (def))
1453 def = NULL_TREE;
1454 else
1455 {
1456 iter1 = make_ssa_name (orig_arg);
1457 if (incr_bb)
1458 iter2 = make_ssa_name (orig_arg);
1459 }
1460 }
1461 if (def)
1462 {
1463 phi = create_phi_node (iter1, body_bb);
1464 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1465 if (incr_bb)
1466 {
1467 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1468 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1469 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1470 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1471 ? TREE_TYPE (orig_arg) : sizetype;
1472 tree addcst = simd_clone_linear_addend (node, i, addtype,
1473 entry_bb);
1474 gsi = gsi_last_bb (incr_bb);
1475 g = gimple_build_assign (iter2, code, iter1, addcst);
1476 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1477 }
1478
1479 imm_use_iterator iter;
1480 use_operand_p use_p;
1481 gimple *use_stmt;
1482 if (TREE_ADDRESSABLE (orig_arg))
1483 {
1484 gsi = gsi_after_labels (body_bb);
1485 g = gimple_build_assign (orig_arg, iter1);
1486 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1487 }
1488 else
1489 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1490 if (use_stmt == phi)
1491 continue;
1492 else
1493 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1494 SET_USE (use_p, iter1);
1495 }
1496 }
1497 else if (node->simdclone->args[i].arg_type
1498 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1499 || (node->simdclone->args[i].arg_type
1500 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1501 {
1502 tree orig_arg = node->simdclone->args[i].orig_arg;
1503 tree def = ssa_default_def (cfun, orig_arg);
1504 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1505 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1506 if (def && !has_zero_uses (def))
1507 {
1508 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1509 iter1 = make_ssa_name (orig_arg);
1510 if (incr_bb)
1511 iter2 = make_ssa_name (orig_arg);
1512 tree iter3 = make_ssa_name (rtype);
1513 tree iter4 = make_ssa_name (rtype);
1514 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1515 gsi = gsi_after_labels (entry_bb);
1516 gimple *load
1517 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1518 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1519
1520 tree array = node->simdclone->args[i].simd_array;
1521 TREE_ADDRESSABLE (array) = 1;
1522 tree ptr = build_fold_addr_expr (array);
1523 phi = create_phi_node (iter1, body_bb);
1524 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1525 if (incr_bb)
1526 {
1527 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1528 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1529 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1530 gsi = gsi_last_bb (incr_bb);
1531 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1532 }
1533
1534 phi = create_phi_node (iter4, body_bb);
1535 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1536 if (incr_bb)
1537 {
1538 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1539 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1540 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1541 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1542 ? TREE_TYPE (iter3) : sizetype;
1543 tree addcst = simd_clone_linear_addend (node, i, addtype,
1544 entry_bb);
1545 g = gimple_build_assign (iter5, code, iter4, addcst);
1546 gsi = gsi_last_bb (incr_bb);
1547 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1548 }
1549
1550 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1551 gsi = gsi_after_labels (body_bb);
1552 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1553
1554 imm_use_iterator iter;
1555 use_operand_p use_p;
1556 gimple *use_stmt;
1557 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1558 if (use_stmt == load)
1559 continue;
1560 else
1561 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1562 SET_USE (use_p, iter1);
1563
1564 if (!TYPE_READONLY (rtype) && incr_bb)
1565 {
1566 tree v = make_ssa_name (rtype);
1567 tree aref = build4 (ARRAY_REF, rtype, array,
1568 size_zero_node, NULL_TREE,
1569 NULL_TREE);
1570 gsi = gsi_after_labels (new_exit_bb);
1571 g = gimple_build_assign (v, aref);
1572 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1573 g = gimple_build_assign (build_simple_mem_ref (def), v);
1574 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1575 }
1576 }
1577 }
1578
1579 calculate_dominance_info (CDI_DOMINATORS);
1580 if (loop)
1581 add_loop (loop, loop->header->loop_father);
1582 update_ssa (TODO_update_ssa);
1583
1584 pop_cfun ();
1585 }
1586
1587 /* If the function in NODE is tagged as an elemental SIMD function,
1588 create the appropriate SIMD clones. */
1589
1590 void
expand_simd_clones(struct cgraph_node * node)1591 expand_simd_clones (struct cgraph_node *node)
1592 {
1593 tree attr = lookup_attribute ("omp declare simd",
1594 DECL_ATTRIBUTES (node->decl));
1595 if (attr == NULL_TREE
1596 || node->global.inlined_to
1597 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1598 return;
1599
1600 /* Ignore
1601 #pragma omp declare simd
1602 extern int foo ();
1603 in C, there we don't know the argument types at all. */
1604 if (!node->definition
1605 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1606 return;
1607
1608 /* Call this before creating clone_info, as it might ggc_collect. */
1609 if (node->definition && node->has_gimple_body_p ())
1610 node->get_body ();
1611
1612 do
1613 {
1614 /* Start with parsing the "omp declare simd" attribute(s). */
1615 bool inbranch_clause_specified;
1616 struct cgraph_simd_clone *clone_info
1617 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1618 &inbranch_clause_specified);
1619 if (clone_info == NULL)
1620 continue;
1621
1622 int orig_simdlen = clone_info->simdlen;
1623 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1624 /* The target can return 0 (no simd clones should be created),
1625 1 (just one ISA of simd clones should be created) or higher
1626 count of ISA variants. In that case, clone_info is initialized
1627 for the first ISA variant. */
1628 int count
1629 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1630 base_type, 0);
1631 if (count == 0)
1632 continue;
1633
1634 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1635 also create one inbranch and one !inbranch clone of it. */
1636 for (int i = 0; i < count * 2; i++)
1637 {
1638 struct cgraph_simd_clone *clone = clone_info;
1639 if (inbranch_clause_specified && (i & 1) != 0)
1640 continue;
1641
1642 if (i != 0)
1643 {
1644 clone = simd_clone_struct_alloc (clone_info->nargs
1645 + ((i & 1) != 0));
1646 simd_clone_struct_copy (clone, clone_info);
1647 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1648 and simd_clone_adjust_argument_types did to the first
1649 clone's info. */
1650 clone->nargs -= clone_info->inbranch;
1651 clone->simdlen = orig_simdlen;
1652 /* And call the target hook again to get the right ISA. */
1653 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1654 base_type,
1655 i / 2);
1656 if ((i & 1) != 0)
1657 clone->inbranch = 1;
1658 }
1659
1660 /* simd_clone_mangle might fail if such a clone has been created
1661 already. */
1662 tree id = simd_clone_mangle (node, clone);
1663 if (id == NULL_TREE)
1664 continue;
1665
1666 /* Only when we are sure we want to create the clone actually
1667 clone the function (or definitions) or create another
1668 extern FUNCTION_DECL (for prototypes without definitions). */
1669 struct cgraph_node *n = simd_clone_create (node);
1670 if (n == NULL)
1671 continue;
1672
1673 n->simdclone = clone;
1674 clone->origin = node;
1675 clone->next_clone = NULL;
1676 if (node->simd_clones == NULL)
1677 {
1678 clone->prev_clone = n;
1679 node->simd_clones = n;
1680 }
1681 else
1682 {
1683 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1684 clone->prev_clone->simdclone->next_clone = n;
1685 node->simd_clones->simdclone->prev_clone = n;
1686 }
1687 symtab->change_decl_assembler_name (n->decl, id);
1688 /* And finally adjust the return type, parameters and for
1689 definitions also function body. */
1690 if (node->definition)
1691 simd_clone_adjust (n);
1692 else
1693 {
1694 simd_clone_adjust_return_type (n);
1695 simd_clone_adjust_argument_types (n);
1696 }
1697 }
1698 }
1699 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1700 }
1701
1702 /* Entry point for IPA simd clone creation pass. */
1703
1704 static unsigned int
ipa_omp_simd_clone(void)1705 ipa_omp_simd_clone (void)
1706 {
1707 struct cgraph_node *node;
1708 FOR_EACH_FUNCTION (node)
1709 expand_simd_clones (node);
1710 return 0;
1711 }
1712
1713 namespace {
1714
1715 const pass_data pass_data_omp_simd_clone =
1716 {
1717 SIMPLE_IPA_PASS, /* type */
1718 "simdclone", /* name */
1719 OPTGROUP_OMP, /* optinfo_flags */
1720 TV_NONE, /* tv_id */
1721 ( PROP_ssa | PROP_cfg ), /* properties_required */
1722 0, /* properties_provided */
1723 0, /* properties_destroyed */
1724 0, /* todo_flags_start */
1725 0, /* todo_flags_finish */
1726 };
1727
1728 class pass_omp_simd_clone : public simple_ipa_opt_pass
1729 {
1730 public:
pass_omp_simd_clone(gcc::context * ctxt)1731 pass_omp_simd_clone(gcc::context *ctxt)
1732 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1733 {}
1734
1735 /* opt_pass methods: */
1736 virtual bool gate (function *);
execute(function *)1737 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1738 };
1739
1740 bool
gate(function *)1741 pass_omp_simd_clone::gate (function *)
1742 {
1743 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1744 }
1745
1746 } // anon namespace
1747
1748 simple_ipa_opt_pass *
make_pass_omp_simd_clone(gcc::context * ctxt)1749 make_pass_omp_simd_clone (gcc::context *ctxt)
1750 {
1751 return new pass_omp_simd_clone (ctxt);
1752 }
1753