SysMan.c 20.6 KB
Newer Older
1
/* ----------------------------------------------------------------------------
2
   Time-stamp: <Wed Mar 21 2001 17:16:28 Stardate: [-30]6363.59 hwloidl>
3
4
5
6
7

   GUM System Manager Program
   Handles startup, shutdown and global synchronisation of the parallel system.

   The Parade/AQUA Projects, Glasgow University, 1994-1995.
8
   GdH/APART Projects, Heriot-Watt University, Edinburgh, 1997-2000.
9
10
11
 
   ------------------------------------------------------------------------- */

12
//@node GUM System Manager Program, , , 
13
14
15
16
17
18
19
//@section GUM System Manager Program

//@menu
//* General docu::		
//* Includes::			
//* Macros etc::		
//* Variables::			
20
21
//* Prototypes::		
//* Aux startup and shutdown fcts::  
22
//* Main fct::			
23
//* Message handlers::		
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
//* Auxiliary fcts::		
//* Index::			
//@end menu

//@node General docu, Includes, GUM System Manager Program, GUM System Manager Program
//@subsection General docu

/*
The Sysman task currently controls initiation, termination, of a
parallel Haskell program running under GUM. In the future it may
control global GC synchronisation and statistics gathering. Based on
K. Hammond's SysMan.lc in Graph for PVM. SysMan is unusual in that it
is not part of the executable produced by ghc: it is a free-standing
program that spawns PVM tasks (logical PEs) to evaluate the
program. After initialisation it runs in parallel with the PE tasks,
awaiting messages.

OK children, buckle down for some serious weirdness, it works like this ...

o The argument vector (argv) for SysMan has one the following 2 shapes:

-------------------------------------------------------------------------------
| SysMan path | debug flag | pvm-executable path | Num. PEs | Program Args ...|
-------------------------------------------------------------------------------

-------------------------------------------------------------------
| SysMan path | pvm-executable path | Num. PEs | Program Args ... |
-------------------------------------------------------------------

The "pvm-executable path" is an absolute path of where PVM stashes the
code for each PE. The arguments passed on to each PE-executable
spawned by PVM are:

-------------------------------
| Num. PEs | Program Args ... |
-------------------------------

The arguments passed to the Main-thread PE-executable are

-------------------------------------------------------------------
| main flag | pvm-executable path | Num. PEs | Program Args ... |
-------------------------------------------------------------------

o SysMan's algorithm is as follows.

o use PVM to spawn (nPE-1) PVM tasks 
o fork SysMan to create the main-thread PE. This permits the main-thread to 
71
72
73
74
  read and write to stdin and stdout. 
o  Wait for all the PE-tasks to reply back saying they are ready and if they were the
  main thread or not.
o Broadcast an array of the PE task-ids out to all of the PE-tasks.
75
o Enter a loop awaiting incoming messages, e.g. failure, Garbage-collection, 
76
  termination.
77
78
79
80
81
82
83
84
85
86
87

The forked Main-thread algorithm, in SysMan, is as follows.

o disconnects from PVM.
o sets a flag in argv to indicate that it is the main thread.
o `exec's a copy of the pvm-executable (i.e. the program being run)


The pvm-executable run by each PE-task, is initialised as follows.

o Registers with PVM, obtaining a task-id.
88
89
90
91
92
93
o If it was main it gets SysMan's task-id from argv otherwise it can use pvm_parent.
oSends a ready message to SysMan together with a flag indicating if it was main or not.
o Receives from SysMan the array of task-ids of the other PEs.
o If the number of task-ids sent was larger than expected then it must have been a task
  generated after the rest of the program had started, so it sends its own task-id message
  to all the tasks it was told about.
94
95
96
97
98
99
100
o Begins execution.

*/

//@node Includes, Macros etc, General docu, GUM System Manager Program
//@subsection Includes

101
102
103
/* Evidently not Posix */
/* #include "PosixSource.h" */

104
105
106
107
#include "Rts.h"
#include "ParTypes.h"
#include "LLC.h"
#include "Parallel.h"
108
#include "ParallelRts.h" // stats only
109
110
111
112
113
114
115
116

//@node Macros etc, Variables, Includes, GUM System Manager Program
//@subsection Macros etc

/* SysMan is put on top of the GHC routine that does the RtsFlags handling.
   So, we cannot use the standard macros. For the time being we use a macro
   that is fixed at compile time.
*/
117
118
119
120
121

#ifdef IF_PAR_DEBUG
#undef IF_PAR_DEBUG
#endif
    
122
/* debugging enabled */
123
//#define IF_PAR_DEBUG(c,s)  { s; } 
124
/* debugging disabled */
125
126
127
#define IF_PAR_DEBUG(c,s)  /* nothing */

void *stgMallocBytes (int n, char *msg);
128

129
//@node Variables, Prototypes, Macros etc, GUM System Manager Program
130
131
132
133
//@subsection Variables

/*
   The following definitions included so that SysMan can be linked with Low
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
   Level Communications module (LLComms). They are not used in SysMan.  
*/
GlobalTaskId         mytid; 

static unsigned      PEsArrived = 0;
static GlobalTaskId  gtids[MAX_PES];
static GlobalTaskId  sysman_id, sender_id;
static unsigned      PEsTerminated = 0;
static rtsBool       Finishing = rtsFalse;
static long          PEbuffer[MAX_PES];
nat                  nSpawn = 0;    // current no. of spawned tasks (see gtids)
nat                  nPEs = 0;      // number of PEs specified on startup
nat                  nextPE;
/* PVM-ish variables */
char                 *petask, *pvmExecutable;
char                 **pargv;
int                  cc, spawn_flag = PvmTaskDefault;

#if 0 && defined(PAR_TICKY)
/* ToDo: use allGlobalParStats to collect stats of all PEs */
GlobalParStats *allGlobalParStats[MAX_PES];
#endif

//@node Prototypes, Aux startup and shutdown fcts, Variables, GUM System Manager Program
//@subsection Prototypes

/* prototypes for message handlers called from the main loop of SysMan */
void newPE(int nbytes, int opcode, int sender_id);
void readyPE(int nbytes, int opcode, int sender_id);
void finishPE(int nbytes, int opcode, int sender_id, int exit_code);

//@node Aux startup and shutdown fcts, Main fct, Prototypes, GUM System Manager Program
//@subsection Aux startup and shutdown fcts

/* 
   Create the PE Tasks. We spawn (nPEs-1) pvm threads: the Main Thread 
   (which starts execution and performs IO) is created by forking SysMan 
*/
static int
createPEs(int total_nPEs) {
  int i, spawn_nPEs, iSpawn = 0, nArch, nHost;
  struct pvmhostinfo *hostp; 
  int sysman_host;

  spawn_nPEs = total_nPEs-1;
  if (spawn_nPEs > 0) {
    IF_PAR_DEBUG(verbose,
		 fprintf(stderr, "==== [%x] Spawning %d PEs(%s) ...\n", 
			 sysman_id, spawn_nPEs, petask);
		 fprintf(stderr, "  args: ");
		 for (i = 0; pargv[i]; ++i)
		   fprintf(stderr, "%s, ", pargv[i]);
		 fprintf(stderr, "\n"));

    pvm_config(&nHost,&nArch,&hostp);
    sysman_host=pvm_tidtohost(sysman_id);
	
    /* create PEs on the specific machines in the specified order! */
    for (i=0; (iSpawn<spawn_nPEs) && (i<nHost); i++)
      if (hostp[i].hi_tid != sysman_host) { 
	checkComms(pvm_spawn(petask, pargv, spawn_flag+PvmTaskHost, 
			     hostp[i].hi_name, 1, gtids+iSpawn),
		   "SysMan startup");
	IF_PAR_DEBUG(verbose,
		     fprintf(stderr, "==== [%x] Spawned PE %d onto %s\n",
			     sysman_id, i, hostp[i].hi_name));
	iSpawn++;
      }
      
    /* create additional PEs anywhere you like */
    if (iSpawn<spawn_nPEs) { 
      checkComms(pvm_spawn(petask, pargv, spawn_flag, "", 
			   spawn_nPEs-iSpawn, gtids+iSpawn),
		 "SysMan startup");
	IF_PAR_DEBUG(verbose,
		     fprintf(stderr,"==== [%x] Spawned %d additional PEs anywhere\n",
			     sysman_id, spawn_nPEs-iSpawn));
      }   
    }

#if 0
  /* old code with random placement of PEs; make that a variant? */
# error "Broken startup in SysMan"
  { /* let pvm place the PEs anywhere; not used anymore */
    checkComms(pvm_spawn(petask, pargv, spawn_flag, "", spawn_nPEs, gtids),"SysMan startup");
    IF_PAR_DEBUG(verbose,
		 fprintf(stderr,"==== [%x] Spawned\n", sysman_id));
    
  }
#endif    

  // iSpawn=spawn_nPEs; 

  return iSpawn;
}

/* 
   Check if this pvm task is in the list of tasks we spawned and are waiting 
   on, if so then remove it.
*/

static rtsBool 
alreadySpawned (GlobalTaskId g) { 
  unsigned int i;

  for (i=0; i<nSpawn; i++)
    if (g==gtids[i]) { 
      nSpawn--;
      gtids[i] = gtids[nSpawn];  //the last takes its place
      return rtsTrue;
    }
  return rtsFalse;
}
247

248
249
250
251
static void 
broadcastFinish(void) { 
  int i,j;
  int tids[MAX_PES];  /* local buffer of all surviving PEs */
252

253
254
255
  for (i=0, j=0; i<nPEs; i++) 
    if (PEbuffer[i]) 
      tids[j++]=PEbuffer[i]; //extract valid tids
256

257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
  IF_PAR_DEBUG(verbose,
    fprintf(stderr,"==== [%x] Broadcasting Finish to %d PEs; initiating shutdown\n", 
	sysman_id, j));

  /* ToDo: move into LLComms.c */	    	    	    
  pvm_initsend(PvmDataDefault);
  pvm_mcast(tids,j,PP_FINISH);
}

static void 
broadcastPEtids (void) { 
  nat i; 

  IF_PAR_DEBUG(verbose,
    fprintf(stderr,"==== [%x] SysMan sending PE table to all PEs\n", sysman_id);
    /* debugging */
    fprintf(stderr,"++++ [%x] PE table as seen by SysMan:\n", mytid);
    for (i = 0; i < nPEs; i++) { 
      fprintf(stderr,"++++ PEbuffer[%d] = %x\n", i, PEbuffer[i]);
    }      	
  )

  broadcastOpN(PP_PETIDS, PEGROUP, nPEs, &PEbuffer);
}

//@node Main fct, Message handlers, Aux startup and shutdown fcts, GUM System Manager Program
283
284
285
//@subsection Main fct

//@cindex main
286
int 
287
288
main (int argc, char **argv) {
  int rbufid;
289
290
  int opcode, nbytes, nSpawn;
  unsigned int i;
291
292
293
  
  setbuf(stdout, NULL);  // disable buffering of stdout
  setbuf(stderr, NULL);  // disable buffering of stderr
294
295
296

  IF_PAR_DEBUG(verbose,
	       fprintf(stderr,
297
	               "==== RFP: GdH enabled SysMan reporting for duty\n"));
298
299
300
301
302
303
304
305
306
  
  if (argc > 1) {
    if (*argv[1] == '-') {
      spawn_flag = PvmTaskDebug;
      argv[1] = argv[0];
      argv++; argc--;
    }
    sysman_id = pvm_mytid();  /* This must be the first PVM call */
    
307
308
309
310
    if (sysman_id<0) { 
	fprintf(stderr, "==== PVM initialisation failure\n");  
      	exit(EXIT_FAILURE);  
    }
311
312
313
314
315
316
317
318
    
    /* 
       Get the full path and filename of the pvm executable (stashed in some
       PVM directory), and the number of PEs from the command line.
    */
    pvmExecutable = argv[1];
    nPEs = atoi(argv[2]);
    
319
320
321
322
323
324
325
326
327
328
329
330
    if (nPEs==0) { 
      /* as usual 0 means infinity: use all PEs specified in PVM config */
      int nArch, nHost;
      struct pvmhostinfo *hostp; 

      /* get info on PVM config */
      pvm_config(&nHost,&nArch,&hostp);
      nPEs=nHost;
      sprintf(argv[2],"%d",nPEs); /* ToCheck: does this work on all archs */
    }	

    /* get the name of the binary to execute */
331
332
333
334
    if ((petask = getenv(PETASK)) == NULL)  // PETASK set by driver
      petask = PETASK;

    IF_PAR_DEBUG(verbose,
335
336
		 fprintf(stderr,"==== [%x] nPEs: %d; executable: |%s|\n", 
			sysman_id, nPEs, petask));
337
    
338
339
340
341
342
343
    /* Check that we can create the number of PE and IMU tasks requested.
                                                     ^^^
       This comment is most entertaining since we haven't been using IMUs 
       for the last 10 years or so -- HWL */
    if ((nPEs > MAX_PES) || (nPEs<1)) {
      fprintf(stderr,"==** SysMan: No more than %d PEs allowed (%d requested)\n     Reconfigure GUM setting MAX_PE in ghc/includes/Parallel.h to a higher value\n", 
344
	   MAX_PES, nPEs);
345
      exit(EXIT_FAILURE);
346
347
    }

348
349
    IF_PAR_DEBUG(verbose,
		   fprintf(stderr,"==== [%x] is SysMan Task\n", sysman_id));
350

351
352
353
354
355
356
357
358
359
360
361
    /* Initialise the PE task arguments from Sysman's arguments */
    pargv = argv + 2;

    /* Initialise list of all PE identifiers */
    PEsArrived=0;  
    nextPE=1;
    for (i=0; i<nPEs; i++)
      PEbuffer[i]=0;
    
    /* start up the required number of PEs */
    nSpawn = createPEs(nPEs);
362
363
364
365
366
367
    
    /* 
       Create the MainThread PE by forking SysMan. This arcane coding 
       is required to allow MainThread to read stdin and write to stdout.
       PWT 18/1/96 
    */
368
    //nPEs++;                /* Record that the number of PEs is increasing */
369
    if ((cc = fork())) {
370
371
372
373
      checkComms(cc,"SysMan fork");         /* Parent continues as SysMan */
                  
      PEbuffer[0]=0;    /* we accept the first main and assume its valid. */
      PEsArrived=1;     /* assume you've got main                         */
374
375

      IF_PAR_DEBUG(verbose,
376
		   fprintf(stderr,"==== [%x] Sysman successfully initialized!\n",
377
378
379
380
381
382
383
384
			   sysman_id));

//@cindex message handling loop
      /* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
      /* Main message handling loop                                         */
      /* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
      /* Process incoming messages */
      while (1) {
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
	if ((rbufid = pvm_recv(ANY_TASK, ANY_OPCODE)) < 0) {
	  pvm_perror("==** Sysman: Receiving Message (pvm_recv)");
          /* never reached */
        }

	pvm_bufinfo(rbufid, &nbytes, &opcode, &sender_id);

        /* very low level debugging
	IF_PAR_DEBUG(verbose,
	             fprintf(stderr,"== [%x] SysMan: Message received by SysMan: rbufid=%x, nbytes = %d, opcode = %x, sender_id = %x\n",
                     sysman_id, rbufid, nbytes, opcode, sender_id));
	*/

	switch (opcode) {
	    
	  case PP_NEWPE: /* a new PE is registering for work */
	    newPE(nbytes, opcode, sender_id);
	    break;

          case PP_READY: /* startup complete; let PEs start working */
	    readyPE(nbytes, opcode, sender_id);
	    break;

	      
	  case PP_GC_INIT: /* start global GC */
410
	    /* This Function not yet implemented for GUM */
411
412
	    fprintf(stderr,"==** Global GC requested by PE %x. Not yet implemented for GUM!\n", 
		    sender_id);
413
414
	    break;
	    
415
416
	  case PP_STATS_ON: /* enable statistics gathering */
	    fprintf(stderr,"==** PP_STATS_ON requested by %x. Not yet implemented for GUM!\n", 
417
418
419
		  sender_id);
	    break;

420
421
	  case PP_STATS_OFF: /* disable statistics gathering */
	    fprintf(stderr,"==** PP_STATS_OFF requested by %x. Not yet implemented for GUM!\n", 
422
423
424
425
		  sender_id);
	    break;
	    
	  case PP_FINISH:
426
427
428
429
430
	    { 
              int exit_code = getExitCode(nbytes, &sender_id);
	      finishPE(nbytes, opcode, sender_id, exit_code);
	      break;

431
432
433
434
435
436
	  default:
	    {
	     /*		          
	      char *opname = GetOpName(opcode);
	      fprintf(stderr,"Sysman: Unrecognised opcode %s (%x)\n",
	                      opname,opcode);	*/
437
438
	      fprintf(stderr,"==** Qagh: Sysman: Unrecognised opcode (%x)\n",
		      opcode);
439
440
441
	    }
	    break;
	  } 	/* switch */
442
	}	/* else */
443
      }		/* while 1 */
444
445
446
447
448
449
450
451
452
453
454
      /* end of SysMan!! */
    } else {	
      /* forked main thread begins here */
      IF_PAR_DEBUG(verbose,
	           fprintf(stderr, "==== Main Thread PE has been forked; doing an execv(%s,...)\n", 
	           pvmExecutable));
      pvmendtask();		 // Disconnect from PVM to avoid confusion:
                                 // executable reconnects 
      
      // RFP: assumes that length(arvv[0])>=9 !!!
      sprintf(argv[0],"-%08X",sysman_id);  /*flag that its the Main Thread PE and include sysman's id*/
455
      execv(pvmExecutable,argv); /* Parent task becomes Main Thread PE */
456
457
458
    }           /* else */
  }		/* argc > 1 */  
}		/* main */
459

460
461
//@node Message handlers, Auxiliary fcts, Main fct, GUM System Manager Program
//@subsection Message handlers
462
463

/*
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
   Received PP_NEWPE:
   A new PE has been added to the configuration.
*/
void
newPE(int nbytes, int opcode, int sender_id) { 
  IF_PAR_DEBUG(verbose,
	       fprintf(stderr,"==== [%x] SysMan detected a new host\n",
		       sysman_id));

  /* Determine the new machine... assume its the last on the config list? */
  if (nSpawn < MAX_PES) { 
    int nArch,nHost;
    struct pvmhostinfo *hostp; 

    /* get conmfiguration of PVM machine */
    pvm_config(&nHost,&nArch,&hostp);	      
    nHost--;
    checkComms(pvm_spawn(petask, pargv, spawn_flag+PvmTaskHost, 
			 hostp[nHost].hi_name, 1, gtids+nSpawn),
	       "SysMan loop");
    nSpawn++;
    IF_PAR_DEBUG(verbose,
		 fprintf(stderr, "==== [%x] Spawned onto %s\n",
			 sysman_id, hostp[nHost].hi_name));
  }
}
	  
/* 
   Received PP_READY:
   Let it be known that PE @sender_id@ participates in the computation.
*/
void
readyPE(int nbytes, int opcode, int sender_id) { 
  int i = 0, flag = 1;
  long isMain;
  int nArch, nHost;
  struct pvmhostinfo *hostp; 

  //ASSERT(opcode==PP_READY);

  IF_PAR_DEBUG(verbose,
	       fprintf(stderr,"==== [%x] SysMan received PP_READY message from %x\n",
		       sysman_id, sender_id));

    pvm_config(&nHost,&nArch,&hostp);

  GetArg1(isMain);
	      
  //if ((isMain && (PEbuffer[0]==0)) || alreadySpawned(sender_id)) { 
    if (nPEs >= MAX_PES) { 
      fprintf(stderr,"==== [%x] SysMan doesn't need PE %d (max %d PEs allowed)\n",
	      sysman_id, sender_id, MAX_PES);
      pvm_kill(sender_id); 
    } else { 
      if (isMain) { 
	IF_PAR_DEBUG(verbose,
		     fprintf(stderr,"==== [%x] SysMan found Main PE %x\n", 
			     sysman_id, sender_id));
	PEbuffer[0]=sender_id;
      } else { 
	/* search for PE in list of PEs */
	for(i=1; i<nPEs; i++)
	  if (PEbuffer[i]==sender_id) { 
	    flag=0;
	    break;
	  }
	/* it's a new PE: add it to the list of PEs */
	if (flag)  
	  PEbuffer[nextPE++] = sender_id; 
	
	IF_PAR_DEBUG(verbose,
		     fprintf(stderr,"==== [%x] SysMan: found PE %d as [%x] on host %s\n", 
			     sysman_id, PEsArrived, sender_id, hostp[PEsArrived].hi_name));

	PEbuffer[PEsArrived++] = sender_id;
      }

		
      /* enable better handling of unexpected terminations */
      checkComms( pvm_notify(PvmTaskExit, PP_FINISH, 1, &sender_id),
	          "SysMan loop");

      /* finished registration of all PEs => enable notification */
      if ((PEsArrived==nPEs) && PEbuffer[0]) { 
	checkComms( pvm_notify(PvmHostAdd, PP_NEWPE, -1, 0),
                    "SysMan startup");
	IF_PAR_DEBUG(verbose,
		     fprintf(stderr,"==== [%x] SysMan initialising notificaton for new hosts\n", sysman_id));
      }
		
      /* finished notification => send off the PE ids */
      if ((PEsArrived>=nPEs) && PEbuffer[0]) { 
        if (PEsArrived>nPEs) {
	IF_PAR_DEBUG(verbose,	
		     fprintf(stderr,"==== [%x] Weird: %d PEs registered, but we only asked for %d\n", sysman_id, PEsArrived, nPEs));
	// nPEs=PEsArrived;
        }
	broadcastPEtids();
      }
563
564
565
    }
}

566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
/* 
   Received PP_FINISH:
   Shut down the corresponding PE. Check whether it is a regular shutdown
   or an uncontrolled termination.
*/
void
finishPE(int nbytes, int opcode, int sender_id, int exitCode) { 
  int i;

  IF_PAR_DEBUG(verbose,
	       fprintf(stderr,"==== [%x] SysMan received PP_FINISH message from %x (exit code: %d)\n",
		       sysman_id, sender_id, exitCode));

  /* Is it relevant to us? Count the first message */
  for (i=0; i<nPEs; i++) 
    if (PEbuffer[i] == sender_id) { 
      PEsTerminated++;
      PEbuffer[i]=0; 
	
      /* handle exit code */
      if (exitCode<0) {           /* a task exit before a controlled finish? */
	fprintf(stderr,"==== [%x] Termination at %x with exit(%d)\n", 
		sysman_id, sender_id, exitCode);
      } else if (exitCode>0) {		           /* an abnormal exit code? */
	fprintf(stderr,"==== [%x] Uncontrolled termination at %x with exit(%d)\n", 
		sysman_id, sender_id, exitCode);	
      } else if (!Finishing) {             /* exitCode==0 which is good news */
        if (i!=0) {          /* someone other than main PE terminated first? */
	 fprintf(stderr,"==== [%x] Unexpected early termination at %x\n", 
		 sysman_id, sender_id);	
	} else {
         /* start shutdown by broadcasting FINISH to other PEs */
	 IF_PAR_DEBUG(verbose,
	              fprintf(stderr,"==== [%x] Initiating shutdown (requested by [%x] RIP) (exit code: %d)\n", sysman_id, sender_id, exitCode));
         Finishing = rtsTrue;
         broadcastFinish();
        }
      }	else {
         /* we are in a shutdown already */
	IF_PAR_DEBUG(verbose,
		     fprintf(stderr,"==== [%x] Finish from %x during shutdown (%d PEs terminated so far; %d total)\n", 
			     sysman_id, sender_id, PEsTerminated, nPEs));
      }

      if (PEsTerminated >= nPEs) { 
        IF_PAR_DEBUG(verbose,
          fprintf(stderr,"==== [%x] Global Shutdown, Goodbye!! (SysMan has received FINISHes from all PEs)\n", sysman_id));
        //broadcastFinish();
	/* received finish from everybody; now, we can exit, too */
        exit(EXIT_SUCCESS); /* Qapla'! */
      }
    }
}	
	    
//@node Auxiliary fcts, Index, Message handlers, GUM System Manager Program
//@subsection Auxiliary fcts

623
624
625
626
/* Needed here because its used in loads of places like LLComms etc */

//@cindex stg_exit

627
628
629
630
631
632
/* 
 * called from STG-land to exit the program
 */

void  
stg_exit(I_ n)
633
{
634
635
636
637
638
639
  fprintf(stderr, "==// [%x] %s in SysMan code; sending PP_FINISH to all PEs ...\n", 
	    mytid,(n!=0)?"FAILURE":"FINISH");
  broadcastFinish();
  //broadcastFinish();
  pvm_exit();
  exit(n);
640
641
642
643
644
645
646
647
648
649
650
}

//@node Index,  , Auxiliary fcts, GUM System Manager Program
//@subsection Index

//@index
//* main::  @cindex\s-+main
//* message handling loop::  @cindex\s-+message handling loop
//* stgMallocBytes::  @cindex\s-+stgMallocBytes
//* stg_exit::  @cindex\s-+stg_exit
//@end index