PaStiX Handbook  6.3.1
codelet_blok_cgemmsp.c
Go to the documentation of this file.
1 /**
2  *
3  * @file codelet_blok_cgemmsp.c
4  *
5  * StarPU codelets for blas-like functions
6  *
7  * @copyright 2016-2023 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
8  * Univ. Bordeaux. All rights reserved.
9  *
10  * @version 6.3.1
11  * @author Mathieu Faverge
12  * @author Pierre Ramet
13  * @author Ian Masliah
14  * @author Tom Moenne-Loccoz
15  * @date 2023-07-21
16  *
17  * @generated from /builds/solverstack/pastix/sopalin/starpu/codelet_blok_zgemmsp.c, normal z -> c, Thu Nov 23 09:55:56 2023
18  *
19  * @addtogroup pastix_starpu
20  * @{
21  *
22  **/
23 #ifndef DOXYGEN_SHOULD_SKIP_THIS
24 #define _GNU_SOURCE
25 #endif /* DOXYGEN_SHOULD_SKIP_THIS */
26 #include "common.h"
27 #include "blend/solver.h"
28 #include "sopalin/sopalin_data.h"
29 #include "pastix_ccores.h"
30 #if defined(PASTIX_WITH_CUDA)
31 #include "pastix_ccuda.h"
32 #endif
33 #include "pastix_starpu.h"
34 #include "pastix_cstarpu.h"
35 #include "codelets.h"
36 
37 /**
38  * @brief Main structure for all tasks of blok_cgemmsp type
39  */
40 struct cl_blok_cgemmsp_args_s {
41  profile_data_t profile_data;
42  sopalin_data_t *sopalin_data;
43  pastix_trans_t trans;
44  const SolverCblk *cblk;
45  SolverCblk *fcblk;
46  pastix_int_t blok_mk;
47  pastix_int_t blok_nk;
48  pastix_int_t blok_mn;
49 };
50 
51 #if defined( PASTIX_STARPU_PROFILING )
52 /**
53  * @brief Functions to profile the codelet
54  *
55  * Two levels of profiling are available:
56  * 1) A generic one that returns the flops per worker
57  * 2) A more detailed one that generate logs of the performance for each kernel
58  */
59 starpu_profile_t blok_cgemmsp_profile = {
60  .next = NULL,
61  .name = "blok_cgemmsp"
62 };
63 
64 /**
65  * @brief Profiling registration function
66  */
67 void blok_cgemmsp_profile_register( void ) __attribute__( ( constructor ) );
68 void
69 blok_cgemmsp_profile_register( void )
70 {
71  profiling_register_cl( &blok_cgemmsp_profile );
72 }
73 
74 #ifndef DOXYGEN_SHOULD_SKIP_THIS
75 #if defined(PASTIX_STARPU_PROFILING_LOG)
76 static void
77 cl_profiling_cb_blok_cgemmsp( void *callback_arg )
78 {
79  cl_profiling_callback( callback_arg );
80 
81  struct starpu_task *task = starpu_task_get_current();
82  struct starpu_profiling_task_info *info = task->profiling_info;
83 
84  /* Quick return */
85  if ( info == NULL ) {
86  return;
87  }
88 
89  struct cl_blok_cgemmsp_args_s *args = (struct cl_blok_cgemmsp_args_s *) callback_arg;
90  pastix_fixdbl_t flops = args->profile_data.flops;
91  pastix_fixdbl_t duration = starpu_timing_timespec_delay_us( &info->start_time, &info->end_time );
92  pastix_fixdbl_t speed = flops / ( 1000.0 * duration );
93 
94  pastix_int_t M = blok_rownbr_ext( args->cblk->fblokptr + args->blok_mk );
95  pastix_int_t N = blok_rownbr_ext( args->cblk->fblokptr + args->blok_nk );
96  pastix_int_t K = cblk_colnbr( args->cblk );
97 
98  cl_profiling_log_register( task->name, "blok_cgemmsp", M, N, K, flops, speed );
99 }
100 #endif
101 
102 #if defined(PASTIX_STARPU_PROFILING_LOG)
103 static void (*blok_cgemmsp_callback)(void*) = cl_profiling_cb_blok_cgemmsp;
104 #else
105 static void (*blok_cgemmsp_callback)(void*) = cl_profiling_callback;
106 #endif
107 #endif /* DOXYGEN_SHOULD_SKIP_THIS */
108 
109 #endif /* defined( PASTIX_STARPU_PROFILING ) */
110 
111 /**
112  *******************************************************************************
113  *
114  * @brief Cost model function
115  *
116  * The user can switch from the pastix static model to an history based model
117  * computed automatically.
118  *
119  *******************************************************************************
120  *
121  * @param[in] task
122  * TODO
123  *
124  * @param[in] arch
125  * TODO
126  *
127  * @param[in] nimpl
128  * TODO
129  *
130  *******************************************************************************
131  *
132  * @retval TODO
133  *
134  *******************************************************************************/
135 static inline pastix_fixdbl_t
136 fct_blok_cgemmsp_cost( struct starpu_task *task,
137  struct starpu_perfmodel_arch *arch,
138  unsigned nimpl )
139 {
140  struct cl_blok_cgemmsp_args_s *args = (struct cl_blok_cgemmsp_args_s *)(task->cl_arg);
141 
142  pastix_fixdbl_t cost = 0.;
143  pastix_fixdbl_t *coefs;
144  pastix_int_t M = blok_rownbr_ext( args->cblk->fblokptr + args->blok_mk );
145  pastix_int_t N = blok_rownbr_ext( args->cblk->fblokptr + args->blok_nk );
146  pastix_int_t K = cblk_colnbr( args->cblk );
147 
148  switch( arch->devices->type ) {
149  case STARPU_CPU_WORKER:
150  coefs = &(args->sopalin_data->cpu_models->coefficients[PastixComplex32-2][PastixKernelGEMMBlok2d2d][0]);
151  break;
152  case STARPU_CUDA_WORKER:
153  coefs = &(args->sopalin_data->gpu_models->coefficients[PastixComplex32-2][PastixKernelGEMMBlok2d2d][0]);
154  break;
155  default:
156  assert(0);
157  return 0.;
158  }
159 
160  /* Get cost in us */
161  cost = modelsGetCost3Param( coefs, M, N, K ) * 1e6;
162 
163  (void)nimpl;
164  return cost;
165 }
166 
167 
168 #ifndef DOXYGEN_SHOULD_SKIP_THIS
169 static struct starpu_perfmodel starpu_blok_cgemmsp_model = {
170 #if defined(PASTIX_STARPU_COST_PER_ARCH)
171  .type = STARPU_PER_ARCH,
172  .arch_cost_function = fct_blok_cgemmsp_cost,
173 #else
174  .type = STARPU_HISTORY_BASED,
175 #endif
176  .symbol = "blok_cgemmsp",
177 };
178 #endif /* DOXYGEN_SHOULD_SKIP_THIS */
179 
180 #if !defined(PASTIX_STARPU_SIMULATION)
181 /**
182  *******************************************************************************
183  *
184  * @brief StarPU CPU implementation
185  *
186  *******************************************************************************
187  *
188  * @param[in] descr
189  * TODO
190  *
191  * @param[in] cl_arg
192  * TODO
193  *
194  *******************************************************************************/
195 static void
196 fct_blok_cgemmsp_cpu( void *descr[], void *cl_arg )
197 {
198  struct cl_blok_cgemmsp_args_s *args = (struct cl_blok_cgemmsp_args_s *)cl_arg;
199  const void *A;
200  const void *B;
201  void *C;
202 
203  A = pastix_starpu_blok_get_ptr( descr[0] );
204  B = pastix_starpu_blok_get_ptr( descr[1] );
205  C = pastix_starpu_blok_get_ptr( descr[2] );
206 
207  assert( args->cblk->cblktype & CBLK_TASKS_2D );
208  assert( args->fcblk->cblktype & CBLK_TASKS_2D );
209 
210  args->profile_data.flops = cpublok_cgemmsp( args->trans,
211  args->cblk, args->fcblk,
212  args->blok_mk, args->blok_nk, args->blok_mn,
213  A, B, C,
214  &(args->sopalin_data->solvmtx->lowrank) );
215 }
216 
217 /**
218  * @brief StarPU GPU implementation
219  */
220 #if defined(PASTIX_WITH_CUDA)
221 static void
222 fct_blok_cgemmsp_gpu( void *descr[], void *cl_arg )
223 {
224  struct cl_blok_cgemmsp_args_s *args = (struct cl_blok_cgemmsp_args_s *)cl_arg;
225  const void *A;
226  const void *B;
227  void *C;
228 
229  A = pastix_starpu_blok_get_ptr( descr[0] );
230  B = pastix_starpu_blok_get_ptr( descr[1] );
231  C = pastix_starpu_blok_get_ptr( descr[2] );
232 
233  assert( args->cblk->cblktype & CBLK_TASKS_2D );
234  assert( args->fcblk->cblktype & CBLK_TASKS_2D );
235 
236  args->profile_data.flops = gpublok_cgemmsp( args->trans,
237  args->cblk, args->fcblk,
238  args->blok_mk, args->blok_nk, args->blok_mn,
239  A, B, C,
240  &(args->sopalin_data->solvmtx->lowrank),
241  starpu_cuda_get_local_stream() );
242 }
243 #endif /* defined(PASTIX_WITH_CUDA) */
244 #endif /* !defined(PASTIX_STARPU_SIMULATION) */
245 
246 #ifndef DOXYGEN_SHOULD_SKIP_THIS
247 CODELETS_GPU( blok_cgemmsp, 3, STARPU_CUDA_ASYNC );
248 #endif /* DOXYGEN_SHOULD_SKIP_THIS */
249 
250 /**
251  *******************************************************************************
252  *
253  * @brief TODO
254  *
255  *******************************************************************************
256  *
257  * @param[in] sopalin_data
258  * TODO
259  *
260  * @param[in] sideA
261  * TODO
262  *
263  * @param[in] sideB
264  * TODO
265  *
266  * @param[in] trans
267  * TODO
268  *
269  * @param[in] cblk
270  * TODO
271  *
272  * @param[in] fcblk
273  * TODO
274  *
275  * @param[in] blokA
276  * TODO
277  *
278  * @param[in] blokB
279  * TODO
280  *
281  * @param[in] prio
282  * TODO
283  *
284  *******************************************************************************/
285 void
286 starpu_task_blok_cgemmsp( sopalin_data_t *sopalin_data,
287  pastix_coefside_t sideA,
288  pastix_coefside_t sideB,
289  pastix_trans_t trans,
290  SolverCblk *cblk,
291  SolverCblk *fcblk,
292  const SolverBlok *blokA,
293  const SolverBlok *blokB,
294  int prio )
295 {
296  struct cl_blok_cgemmsp_args_s *cl_arg = NULL;
297  long long execute_where = cl_blok_cgemmsp_any.where;
298  int need_exec = 1;
299 #if defined(PASTIX_DEBUG_STARPU) || defined(PASTIX_STARPU_PROFILING_LOG)
300  char *task_name;
301 #endif
302 
303  pastix_int_t frownum;
304  pastix_int_t lrownum;
305  pastix_int_t blok_mn = 0, j = 0;
306  pastix_int_t blok_mk = blokA - cblk->fblokptr;
307  pastix_int_t blok_nk = blokB - cblk->fblokptr;
308  SolverBlok *blokC = fcblk->fblokptr;
309 
310  assert( blok_nk <= blok_mk );
311 
312  do {
313  frownum = blokC->frownum;
314  lrownum = blokC->lrownum;
315  blok_mn += j;
316  j = 1;
317 
318  /* Increase lrownum as long as blocks are facing the same cblk */
319  while( (blokC < fcblk[1].fblokptr-1) &&
320  (blokC[0].fcblknm == blokC[1].fcblknm) &&
321  (blokC[0].lcblknm == blokC[1].lcblknm) )
322  {
323  blokC++; j++;
324  lrownum = blokC->lrownum;
325  }
326  blokC++;
327  }
328  while( !((blokA->frownum >= frownum) &&
329  (blokA->lrownum <= lrownum)) );
330 
331  blokC = fcblk->fblokptr + blok_mn;
332 
333  assert( blokA->lcblknm == blokB->lcblknm );
334  assert( blokB->fcblknm == blokC->lcblknm );
335  assert( blokC->frownum <= blokA->frownum );
336  assert( blokA[-1].fcblknm != blokA[0].fcblknm );
337  assert( blokB[-1].fcblknm != blokB[0].fcblknm );
338  assert( (blok_mn == 0) || (blokC[-1].fcblknm != blokC[0].fcblknm) );
339 
340  /*
341  * Check if it needs to be submitted
342  */
343 #if defined(PASTIX_WITH_MPI)
344  {
345  int need_submit = 0;
346  if ( cblk->ownerid == sopalin_data->solvmtx->clustnum ) {
347  need_submit = 1;
348  }
349  if ( fcblk->ownerid == sopalin_data->solvmtx->clustnum ) {
350  need_submit = 1;
351  }
352  else {
353  need_exec = 0;
354  }
355  if ( starpu_mpi_cached_receive( blokC->handler[sideA] ) ) {
356  need_submit = 1;
357  }
358  if ( !need_submit ) {
359  return;
360  }
361  }
362 #endif
363 
364 #if !defined(HAVE_STARPU_DATA_PARTITION_CLEAN_NODE)
365  /* Mark the read cblk as partitionned */
366  cblk->partitioned |= (sideA + 1);
367  cblk->partitioned |= (sideB + 1);
368 #endif
369 
370  /*
371  * Create the arguments array
372  */
373  if ( need_exec ) {
374  cl_arg = malloc( sizeof( struct cl_blok_cgemmsp_args_s ) );
375  cl_arg->sopalin_data = sopalin_data;
376 #if defined(PASTIX_STARPU_PROFILING)
377  cl_arg->profile_data.measures = blok_cgemmsp_profile.measures;
378  cl_arg->profile_data.flops = NAN;
379 #endif
380  cl_arg->trans = trans;
381  cl_arg->cblk = cblk;
382  cl_arg->fcblk = fcblk;
383  cl_arg->blok_mk = blok_mk;
384  cl_arg->blok_nk = blok_nk;
385  cl_arg->blok_mn = blok_mn;
386 
387 #if defined(PASTIX_WITH_CUDA)
388  if ( (cblk->cblktype & CBLK_COMPRESSED) ||
389  (fcblk->cblktype & CBLK_COMPRESSED) )
390  {
391  /* Disable CUDA */
392  execute_where &= (~STARPU_CUDA);
393  }
394 #endif
395  }
396 
397 #if defined(PASTIX_DEBUG_STARPU) || defined(PASTIX_STARPU_PROFILING_LOG)
398  /* This actually generates a memory leak */
399  asprintf( &task_name, "%s( %ld, %ld, %ld, %ld )",
400  cl_blok_cgemmsp_any.name,
401  (long)(blokA - sopalin_data->solvmtx->bloktab),
402  (long)(blokB - sopalin_data->solvmtx->bloktab),
403  (long)(blokC - sopalin_data->solvmtx->bloktab),
404  (long)sideA );
405 #endif
406 
407  pastix_starpu_insert_task(
408  &cl_blok_cgemmsp_any,
409  STARPU_CL_ARGS, cl_arg, sizeof( struct cl_blok_cgemmsp_args_s ),
410  STARPU_EXECUTE_WHERE, execute_where,
411 #if defined(PASTIX_STARPU_PROFILING)
412  STARPU_CALLBACK_WITH_ARG_NFREE, blok_cgemmsp_callback, cl_arg,
413 #endif
414  STARPU_R, blokA->handler[sideA],
415  STARPU_R, blokB->handler[sideB],
416  STARPU_RW, blokC->handler[sideA],
417 #if defined(PASTIX_DEBUG_STARPU) || defined(PASTIX_STARPU_PROFILING_LOG)
418  STARPU_NAME, task_name,
419 #endif
420 #if defined(PASTIX_STARPU_HETEROPRIO)
421  STARPU_PRIORITY, BucketGEMM2D,
422 #else
423  STARPU_PRIORITY, prio,
424 #endif
425  0);
426  (void)prio;
427 }
428 
429 /**
430  * @}
431  */
pastix_int_t
BEGIN_C_DECLS typedef int pastix_int_t
Definition: datatypes.h:51
pastix_fixdbl_t
double pastix_fixdbl_t
Definition: datatypes.h:65
PastixKernelGEMMBlok2d2d
@ PastixKernelGEMMBlok2d2d
Definition: kernels_enums.h:67
cpublok_cgemmsp
pastix_fixdbl_t cpublok_cgemmsp(pastix_trans_t transB, const SolverCblk *cblk, SolverCblk *fcblk, pastix_int_t blok_mk, pastix_int_t blok_nk, pastix_int_t blok_mn, const void *A, const void *B, void *C, const pastix_lr_t *lowrank)
Compute the CPU gemm associated to a couple of off-diagonal blocks.
Definition: core_cgemmsp.c:1671
pastix_trans_t
enum pastix_trans_e pastix_trans_t
Transpostion.
pastix_coefside_t
enum pastix_coefside_e pastix_coefside_t
Data blocks used in the kernel.
fct_blok_cgemmsp_cost
static pastix_fixdbl_t fct_blok_cgemmsp_cost(struct starpu_task *task, struct starpu_perfmodel_arch *arch, unsigned nimpl)
Cost model function.
Definition: codelet_blok_cgemmsp.c:136
fct_blok_cgemmsp_cpu
static void fct_blok_cgemmsp_cpu(void *descr[], void *cl_arg)
StarPU CPU implementation.
Definition: codelet_blok_cgemmsp.c:196
starpu_task_blok_cgemmsp
void starpu_task_blok_cgemmsp(sopalin_data_t *sopalin_data, pastix_coefside_t sideA, pastix_coefside_t sideB, pastix_trans_t trans, SolverCblk *cblk, SolverCblk *fcblk, const SolverBlok *blokA, const SolverBlok *blokB, int prio)
StarPU GPU implementation.
Definition: codelet_blok_cgemmsp.c:286
profile_data_s
Base structure to all codelet arguments that include the profiling data.
Definition: pastix_starpu.h:182
cost
static double cost(symbol_cblk_t *cblk)
Computes the cost of a cblk.
Definition: symbol_reorder.c:288
pastix_ccores.h
pastix_ccuda.h
pastix_cstarpu.h
pastix_starpu.h
blok_rownbr_ext
static pastix_int_t blok_rownbr_ext(const SolverBlok *blok)
Compute the number of rows of a contiguous block in front of the same cblk.
Definition: solver.h:402
solver_blok_s::lrownum
pastix_int_t lrownum
Definition: solver.h:143
solver_blok_s::handler
void * handler[2]
Definition: solver.h:138
cblk_colnbr
static pastix_int_t cblk_colnbr(const SolverCblk *cblk)
Compute the number of columns in a column block.
Definition: solver.h:324
solver_blok_s::fcblknm
pastix_int_t fcblknm
Definition: solver.h:140
solver_blok_s::frownum
pastix_int_t frownum
Definition: solver.h:142
solver_cblk_s::fblokptr
SolverBlok * fblokptr
Definition: solver.h:163
solver_blok_s::lcblknm
pastix_int_t lcblknm
Definition: solver.h:139
solver_cblk_s::cblktype
int8_t cblktype
Definition: solver.h:159
solver_cblk_s::ownerid
int ownerid
Definition: solver.h:175
solver_cblk_s::partitioned
int8_t partitioned
Definition: solver.h:160
solver_blok_s
Solver block structure.
Definition: solver.h:137
solver_cblk_s
Solver column block structure.
Definition: solver.h:156