PaStiX Handbook  6.2.1
starpu.c
Go to the documentation of this file.
1 /**
2  *
3  * @file starpu.c
4  *
5  * @copyright 2017-2021 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
6  * Univ. Bordeaux. All rights reserved.
7  *
8  * @version 6.2.1
9  * @author Pierre Ramet
10  * @author Mathieu Faverge
11  * @date 2021-06-21
12  *
13  * @addtogroup pastix_starpu
14  * @{
15  *
16  **/
17 #include "common.h"
18 #if !defined(PASTIX_WITH_STARPU)
19 #error "This file should not be compiled if Starpu is not enabled"
20 #endif
21 #include <stdio.h>
22 #include "pastix_starpu.h"
23 
24 #if defined(PASTIX_STARPU_HETEROPRIO)
25 #include <starpu_heteroprio.h>
26 /**
27  *******************************************************************************
28  *
29  * @brief Inits the heteroprio priorities, mappings and accelerations.
30  *
31  * This function initializes the heteroprio system in an arbitrary manner for now.
32  * Used as a callback by starpu directly.
33  *
34  ******************************************************************************/
35 void
36 init_heteroprio( unsigned ctx )
37 {
38  unsigned idx;
39  /* CPU uses 4 buckets and visits them in the natural order */
40  starpu_heteroprio_set_nb_prios( ctx, STARPU_CPU_WORKER, BucketNumber );
41  /* It uses direct mapping idx => idx */
42  for ( idx = 0; idx < BucketNumber; ++idx ) {
43  starpu_heteroprio_set_mapping( ctx, STARPU_CPU_WORKER, idx, idx );
44  /* If there are no CUDA workers, we must tell that CPU is faster */
45  starpu_heteroprio_set_faster_arch( ctx, STARPU_CPU_WORKER, idx );
46  }
47  if ( starpu_cuda_worker_get_count() ) {
48  const int cuda_matching[] = { BucketGEMM2D, BucketTRSM2D, BucketGEMM1D };
49  const float cuda_factor = 125.0f / starpu_cpu_worker_get_count();
50  const float cuda_factors[] = { cuda_factor, cuda_factor, cuda_factor };
51  /* CUDA is enabled and uses 2 buckets */
52  starpu_heteroprio_set_nb_prios( ctx, STARPU_CUDA_WORKER, 3 );
53 
54  for ( idx = 0; idx < 3; ++idx ) {
55  /* CUDA has its own mapping */
56  starpu_heteroprio_set_mapping( ctx, STARPU_CUDA_WORKER, idx, cuda_matching[idx] );
57  /* For its buckets, CUDA is the fastest */
58  starpu_heteroprio_set_faster_arch( ctx, STARPU_CUDA_WORKER, cuda_matching[idx] );
59  /* And CPU is slower by a factor dependant on the bucket */
60  starpu_heteroprio_set_arch_slow_factor(
61  ctx, STARPU_CPU_WORKER, cuda_matching[idx], cuda_factors[idx] );
62  }
63  }
64 }
65 #endif
66 
67 /**
68  *******************************************************************************
69  *
70  * @brief Startup the StarPU runtime system.
71  *
72  * This function initialize and startup the StarPU runtime system with PaStix
73  * configuration variables
74  *
75  *******************************************************************************
76  *
77  * @param[inout] pastix
78  * The main pastix_data structure.
79  *
80  * @param[inout] argc
81  * The number of arguments of the main program.
82  *
83  * @param[inout] argv
84  * The list of argument given to the main program.
85  *
86  * @param[in] bindtab
87  * The binding array of size the number of threads if a specific
88  * binding is required, NULL otherwise.
89  *
90  ******************************************************************************/
91 void
92 pastix_starpu_init( pastix_data_t *pastix,
93  int *argc, char **argv[],
94  const int *bindtab )
95 {
96  struct starpu_conf *conf;
97  pastix_int_t *iparm = pastix->iparm;
98  int rc;
99 
100  if ( pastix->starpu != NULL )
101  return;
102 
103  pastix->starpu = malloc(sizeof(struct starpu_conf));
104  starpu_conf_init( pastix->starpu );
105 
106 #if defined(PASTIX_STARPU_PROFILING_LOG)
108  profiling_log_init( pastix->dir_local );
109 #endif
110 
111  /* Force no GPUs if CUDA has not been enabled in PaStiX */
112 #if !defined(PASTIX_WITH_CUDA)
113  iparm[IPARM_GPU_NBR] = 0;
114 #endif
115 
116  conf = pastix->starpu;
117  conf->ncpus = pastix_imax( 1, (iparm[IPARM_THREAD_NBR] - iparm[IPARM_GPU_NBR] - 1) );
118  conf->ncuda = iparm[IPARM_GPU_NBR];
119  conf->nopencl = 0;
120 
121 #if defined(PASTIX_STARPU_HETEROPRIO)
122  /*
123  * Set scheduling to heteroprio in any case if requested at compilation
124  */
125  conf->sched_policy_name = "heteroprio";
126 #if !defined(HAVE_STARPU_SCHED_POLICY_CALLBACK )
127  conf->sched_policy_init = &init_heteroprio;
128 #else
129  conf->sched_policy_callback = &init_heteroprio;
130 #endif
131 #else /* PASTIX_STARPU_HETEROPRIO */
132 
133  if (conf->ncuda > 0) {
134 #if defined(PASTIX_GENERATE_MODEL)
135  pastix_print( pastix->procnum, 0,
136  "WARNING: PaStiX compiled with -DPASTIX_GENERATE_MODEL forces:\n"
137  " - a single event per stream\n"
138  " - a single stream per GPU\n"
139  " - restore the automatic detection of the number of threads\n" );
140 
141  conf->ncpus = -1;
142 
143  pastix_setenv( "STARPU_NWORKER_PER_CUDA", "1", 1 );
144  pastix_setenv( "STARPU_CUDA_PIPELINE", "0", 1 );
145 #endif
146 
147  conf->sched_policy_name = "dmdas";
148  }
149  else {
150  /*
151  * Set scheduling to "ws"/"lws" if no cuda devices used because it
152  * behaves better on homogneneous architectures. If the user wants
153  * to use another scheduling strategy, he can set STARPU_SCHED
154  * env. var. to whatever he wants
155  */
156 #if (STARPU_MAJOR_VERSION > 1) || ((STARPU_MAJOR_VERSION == 1) && (STARPU_MINOR_VERSION >= 2))
157  conf->sched_policy_name = "lws";
158 #else
159  conf->sched_policy_name = "ws";
160 #endif
161  }
162 #endif /* PASTIX_STARPU_HETEROPRIO */
163 
164  if ( bindtab != NULL ) {
165  int i;
166 
167  assert( iparm[IPARM_THREAD_NBR] < STARPU_NMAXWORKERS );
168  conf->use_explicit_workers_bindid = 1;
169 
170  for(i=0; i < pastix_imin( iparm[IPARM_THREAD_NBR], STARPU_NMAXWORKERS ); i++) {
171  conf->workers_bindid[i] = bindtab[i];
172  }
173  }
174 #if defined(STARPU_USE_FXT)
175  starpu_fxt_autostart_profiling( 0 ); /* FxT starts profiling upon explicit call only */
176 #endif
177  rc = starpu_init( conf );
178 
179  starpu_malloc_on_node_set_default_flags( STARPU_MAIN_RAM,
180  STARPU_MALLOC_PINNED
181  | STARPU_MALLOC_COUNT
182 #if defined(PASTIX_STARPU_SIMULATION)
183  | STARPU_MALLOC_SIMULATION_FOLDED
184 #endif
185  );
186 
187 #if defined(PASTIX_WITH_MPI)
188 #if defined(PASTIX_DEBUG_MPI) && !defined(PASTIX_STARPU_SIMULATION)
189  {
190  int flag = 0;
191  MPI_Initialized( &flag );
192  assert( flag );
193  }
194 #endif
195  starpu_mpi_init_comm( argc, argv, 0, pastix->inter_node_comm );
197 #endif
199 
200 #if defined(PASTIX_WITH_CUDA) && !defined(PASTIX_STARPU_SIMULATION)
201  starpu_cublas_init();
202 #endif
203 
204  /* Suspend threads until we need them */
205  starpu_pause();
206 
207  assert( pastix->starpu != NULL );
208 
209  (void)argc; (void)argv;
210  (void)rc;
211 }
212 
213 /**
214  *******************************************************************************
215  *
216  * @brief Finalize the StarPU runtime system.
217  *
218  * This function stop the StarPU runtime system.
219  *
220  *******************************************************************************
221  *
222  * @param[inout] pastix
223  * The main pastix_data structure.
224  *
225  ******************************************************************************/
226 void
228 {
229  if ( pastix->starpu != NULL ) {
230  starpu_resume();
231 
233 
234  profiling_display_allinfo();
235  profiling_log_fini();
236 
237 #if defined(PASTIX_WITH_MPI)
238  starpu_mpi_shutdown();
239 #endif
240 #if defined(PASTIX_WITH_CUDA) && !defined(PASTIX_STARPU_SIMULATION)
241  starpu_cublas_shutdown();
242 #endif
243  starpu_shutdown();
244 
245  free( pastix->starpu );
246  pastix->starpu = NULL;
247  }
248 }
249 
250 /**
251  * @}
252  */
pastix_starpu_finalize
void pastix_starpu_finalize(pastix_data_t *pastix)
Finalize the StarPU runtime system.
Definition: starpu.c:227
pastix_starpu_tag_init
int pastix_starpu_tag_init(pastix_data_t *pastix)
pastix_gendirectories
void pastix_gendirectories(pastix_data_t *pastix_data)
Generate a unique temporary directory to store output files.
Definition: api.c:69
pastix_starpu_interface_init
void pastix_starpu_interface_init()
Initialize the interface ID.
Definition: pastix_starpu_interface.c:546
IPARM_THREAD_NBR
@ IPARM_THREAD_NBR
Definition: api.h:117
pastix_starpu_init
void pastix_starpu_init(pastix_data_t *pastix, int *argc, char **argv[], const int *bindtab)
Startup the StarPU runtime system.
Definition: starpu.c:92
IPARM_GPU_NBR
@ IPARM_GPU_NBR
Definition: api.h:121
pastix_starpu.h
pastix
BEGIN_C_DECLS int pastix(pastix_data_t **pastix_data, PASTIX_Comm pastix_comm, pastix_int_t n, pastix_int_t *colptr, pastix_int_t *rowptr, void *values, pastix_int_t *perm, pastix_int_t *invp, void *B, pastix_int_t nrhs, pastix_int_t *iparm, double *dparm)
Main function for compatibility with former releases.
Definition: pastix.c:103
pastix_starpu_interface_fini
void pastix_starpu_interface_fini()
Finalize the interface and reset the ID.
Definition: pastix_starpu_interface.c:570