VMS/VMS_Implementations/VMS_impls/VMS__MC_shared_impl

view CoreController.c @ 257:f5b110414453

fix coding standard to work with -std=gnu99 -Wall
author Nina Engelhardt <nengel@mailbox.tu-berlin.de>
date Tue, 25 Sep 2012 16:11:15 +0200
parents 4c7414df4f0e
children 094ad1bdeaec
line source
1 /*
2 * Copyright 2010 OpenSourceStewardshipFoundation
3 *
4 * Licensed under BSD
5 */
8 #include "VMS.h"
10 #include <stdlib.h>
11 #include <stdio.h>
12 #include <time.h>
14 #include <pthread.h>
15 #include <sched.h>
17 #include <unistd.h>
18 ssize_t read(int fd, void *buf, size_t count);
20 //===================== Functions local to this file =======================
21 void *terminateCoreController(SlaveVP *currSlv);
23 void
24 doBackoff_for_TooLongToGetLock( int32 numTriesToGetLock, uint32 *seed1,
25 uint32 *seed2 );
26 void
27 doBackoff_for_TooLongWithNoWork( int32 numRepsWithNoWork, uint32 *seed1,
28 uint32 *seed2 );
30 //===========================================================================
33 /*The Core Controller is logically "beneath" the masterVP and slave VPs. Its
34 * job is to control which of those VPs the core animates. Any time one of
35 * those VPs suspends, the suspend-primitive switches the core over to
36 * animating the core controller. The core controller then follows a very
37 * basic pattern to choose which VP will get animated next, then switches
38 * the core over to animating that VP. So, all VPs switch the core to
39 * core controller, which then chooses which VP the core animates next.
40 *
41 *The way the core controller decides which VP to switch the core to next is:
42 * 1) There are a number of "animation slots", which the master VP fills up
43 * with slave VPs that are ready to be animated. So, the core controller
44 * just iterates through the animation slots. When the next slot has a
45 * slave VP in it, the core controller switches the core over to animate
46 * that slave.
47 * 2) When the core controller checks a animation slot, and it's empty,
48 * then the controller switches the core over to animating the master VP,
49 * whose job is to find more slave VPs ready, and assign those to
50 * animation slots.
51 *
52 *So, in effect, a animation slot functions as another layer of virtual
53 * processor. A slot has the logical meaning of being an animator that
54 * animates the slave assigned to it. However, the core controller sits
55 * below the slots, and sequences down them, assigning the actual physical
56 * core to each slot, in turn.
57 *The reason for having the animation slots and core controller is to
58 * amortize the overhead of switching to the master VP and running it. With
59 * multiple animation slots, the time to switch-to-master and the code in
60 * the animation master is divided by the number of animation slots.
61 *The core controller and animation slots are not fundamental parts of VMS,
62 * but rather optimizations put into the shared-semantic-state version of
63 * VMS. Other versions of VMS will not have a core controller nor scheduling
64 * slots.
65 *
66 *The core controller "owns" the physical core, in effect, and is the
67 * function given to the pthread's creation call. Hence, it contains code
68 * related to pthread startup, synchronizing the controllers to all start
69 * at the same time-point, and pinning the pthreads to physical cores.
70 *
71 */
72 void *
73 coreController( void *paramsIn )
74 {
75 int32 thisCoresIdx;
76 int32 numRepetitionsWithNoWork;
77 SlaveVP *currVP;
78 AnimSlot *currSlot, **animSlots;
79 int32 currSlotIdx;
80 volatile int32 *addrOfMasterLock; //thing pointed to is volatile, not ptr
81 SlaveVP *thisCoresMasterVP;
82 //Variables used for pthread related things
83 ThdParams *thisCoresThdParams;
84 cpu_set_t coreMask; //used during pinning pthread to CPU core
85 int32 errorCode;
86 //Variables used during measurements
87 #ifdef MEAS__TURN_ON_SYSTEM_MEAS
88 TSCountLowHigh endSusp;
89 #endif
90 //Variables used in random-backoff, for master-lock and waiting for work
91 uint32_t seed1 = rand()%1000; // init random number generator for backoffs
92 uint32_t seed2 = rand()%1000;
95 //=============== Initializations ===================
96 thisCoresThdParams = (ThdParams *)paramsIn;
97 thisCoresIdx = thisCoresThdParams->coreNum;
99 //Assembly that saves addr of label of return instr -- label in assmbly
100 recordCoreCtlrReturnLabelAddr((void**)&(_VMSMasterEnv->coreCtlrReturnPt));
102 animSlots = _VMSMasterEnv->allAnimSlots[thisCoresIdx];
103 currSlotIdx = 0; //start at slot 0, go up until one empty, then do master
104 numRepetitionsWithNoWork = 0;
105 addrOfMasterLock = &(_VMSMasterEnv->masterLock);
106 thisCoresMasterVP = _VMSMasterEnv->masterVPs[thisCoresIdx];
108 //==================== pthread related stuff ======================
109 //pin the pthread to the core -- takes away Linux control
110 //Linux requires pinning to be done inside the thread-function
111 //Designate a core by a 1 in bit-position corresponding to the core
112 CPU_ZERO(&coreMask); //initialize mask bits to zero
113 CPU_SET(thisCoresThdParams->coreNum,&coreMask); //set bit repr the coreNum
114 pthread_t selfThd = pthread_self();
115 errorCode =
116 pthread_setaffinity_np( selfThd, sizeof(coreMask), &coreMask);
117 if(errorCode){ printf("\n pinning thd to core failed \n"); exit(0); }
119 //make sure the controllers all start at same time, by making them wait
120 pthread_mutex_lock( &suspendLock );
121 while( !(_VMSMasterEnv->setupComplete) )
122 { pthread_cond_wait( &suspendCond, &suspendLock );
123 }
124 pthread_mutex_unlock( &suspendLock );
126 HOLISTIC__CoreCtrl_Setup;
128 DEBUG__printf1(TRUE, "started coreCtrlr on core %d", thisCoresIdx );
130 //====================== The Core Controller ======================
131 while(1) //An endless loop is just one way of doing the control structure
132 { //Assembly code switches the core between animating a VP and
133 // animating this core controller. The switch is done by
134 // changing the stack-pointer and frame-pointer and then doing
135 // an assembly jmp. When reading this code, the effect is
136 // that the "switchToSlv()" at the end of the loop is sort of a
137 // "warp in time" -- the core disappears inside this, jmps to
138 // animating a VP, and when that VP suspends, the suspend
139 // jmps back. This has the effect of "returning" from the
140 // switchToSlv() call. Then control loops back to here.
141 //Alternatively, the VP suspend primitive could just not bother
142 // returning from switchToSlv, and instead jmp directly to here.
144 if( currSlotIdx >= NUM_ANIM_SLOTS ) goto switchToMaster;
145 currSlot = animSlots[ currSlotIdx ];
147 if( ! currSlot->needsSlaveAssigned ) //slot does have slave assigned
148 { if(currSlot->slaveAssignedToSlot->typeOfVP == Idle){
149 numRepetitionsWithNoWork ++;
150 } else {
151 numRepetitionsWithNoWork = 0; //reset back2back master count
152 }
153 currSlotIdx ++;
154 currVP = currSlot->slaveAssignedToSlot;
155 HOLISTIC__Record_last_work;
156 }
157 else //slot is empty, so switch to master
158 {
159 switchToMaster:
160 currSlotIdx = 0; //doing switch to master, so start over at slot 0
161 currVP = NULL;
163 MEAS__Capture_Pre_Master_Lock_Point;
164 HOLISTIC__Record_AppResponderInvocation_start;
166 int numTriesToGetLock = 0; int gotLock = 0;
167 while( currVP == NULL ) //keep going until get master lock
168 {
169 //At this point, first thing to do is get lock. But, want to
170 // reduce lock contention from cores with no work, so first
171 // check if this is a core with no work, and busy wait if so.
172 //Then, if it's been way too long without work, yield pthread
173 if( numRepetitionsWithNoWork > NUM_REPS_W_NO_WORK_BEFORE_BACKOFF)
174 doBackoff_for_TooLongWithNoWork( numRepetitionsWithNoWork, &seed1, &seed2 );
175 if( numRepetitionsWithNoWork > NUM_REPS_W_NO_WORK_BEFORE_YIELD )
176 { numRepetitionsWithNoWork = 0; pthread_yield(); }
179 //Now, try to get the lock
180 gotLock = __sync_bool_compare_and_swap( addrOfMasterLock,
181 UNLOCKED, LOCKED );
182 if( gotLock )
183 { //At this point, have run out of slaves, so tried to get
184 // the master lock, and have successfully gotten it.
185 //So, set the currVP to this core's masterVP and break out
186 // of the get-lock loop. Below, assembly code will switch
187 // the core over to animating the masterVP. When it's
188 // done, the masterVP will use assembly to switch the core
189 // back to animating this core controller
190 currVP = thisCoresMasterVP;
191 numRepetitionsWithNoWork += 1;
192 break; //end while -- have a VP to animate now
193 }
194 //Get here only when failed to get lock
196 numTriesToGetLock++; //if too many, means too much contention
197 if( numTriesToGetLock > NUM_TRIES_BEFORE_DO_BACKOFF )
198 doBackoff_for_TooLongToGetLock( numTriesToGetLock, &seed1, &seed2 );
199 if( numTriesToGetLock > MASTERLOCK_RETRIES_BEFORE_YIELD )
200 { numTriesToGetLock = 0; pthread_yield(); }
201 }
202 MEAS__Capture_Post_Master_Lock_Point;
203 }
205 HOLISTIC__Record_Work_start;
207 switchToSlv(currVP); //Slave suspend makes core "return" from this call
208 flushRegisters(); //prevent GCC optimization from doing bad things
210 HOLISTIC__Record_Work_end;
212 MEAS__Capture_End_Susp_in_CoreCtlr_ForSys;
214 }//while(1)
215 }
217 /*Shutdown of VMS involves several steps, of which this is the last. This
218 * function is jumped to from the asmTerminateCoreCtrl, which is in turn
219 * called from endOSThreadFn, which is the top-level-fn of the shutdown
220 * slaves.
221 */
222 void *
223 terminateCoreCtlr(SlaveVP *currSlv)
224 {
225 //first, free shutdown Slv that jumped here, then end the pthread
226 VMS_int__dissipate_slaveVP( currSlv );
227 pthread_exit( NULL );
228 }
230 uint32_t
231 randomNumber()
232 {
233 _VMSMasterEnv->seed1 = (uint32)(36969 * (_VMSMasterEnv->seed1 & 65535) +
234 (_VMSMasterEnv->seed1 >> 16) );
235 _VMSMasterEnv->seed2 = (uint32)(18000 * (_VMSMasterEnv->seed2 & 65535) +
236 (_VMSMasterEnv->seed2 >> 16) );
237 return (_VMSMasterEnv->seed1 << 16) + _VMSMasterEnv->seed2;
238 }
242 /*Busy-wait for a random number of cycles -- chooses number of cycles
243 * differently than for the too-many-tries-to-get-lock backoff
244 */
245 void
246 doBackoff_for_TooLongWithNoWork( int32 numRepsWithNoWork, uint32 *seed1,
247 uint32 *seed2 )
248 { int32 i, waitIterations;
249 volatile double fakeWorkVar; //busy-wait fake work
251 //Get a random number of iterations to busy-wait. The % is a simple
252 // way to set the maximum value that can be generated.
253 waitIterations =
254 randomNumber(seed1, seed2) %
255 (numRepsWithNoWork * numRepsWithNoWork * NUM_CORES);
256 for( i = 0; i < waitIterations; i++ )
257 { fakeWorkVar += (fakeWorkVar + 32.0) / 2.0; //busy-wait
258 }
259 }
261 /*Busy-waits for a random number of cycles -- chooses number of cycles
262 * differently than for the no-work backoff
263 */
264 void
265 doBackoff_for_TooLongToGetLock( int32 numTriesToGetLock, uint32 *seed1,
266 uint32 *seed2 )
267 { int32 i, waitIterations;
268 volatile double fakeWorkVar; //busy-wait fake work
270 waitIterations =
271 randomNumber(seed1, seed2) %
272 (numTriesToGetLock * GET_LOCK_BACKOFF_WEIGHT);
273 //addToHist( wait_iterations, coreLoopThdParams->wait_iterations_hist );
274 for( i = 0; i < waitIterations; i++ )
275 { fakeWorkVar += (fakeWorkVar + 32.0) / 2.0; //busy-wait
276 }
277 }
280 #ifdef DEBUG__TURN_ON_SEQUENTIAL_MODE
282 //===========================================================================
283 /*This sequential version does the same as threaded, except doesn't do the
284 * pin-threads part, nor the wait until setup complete and acquire master
285 * lock parts.
286 */
287 void *
288 coreCtlr_Seq( void *paramsIn )
289 {
290 int32 thisCoresIdx;
291 int32 numRepetitionsWithNoWork;
292 SlaveVP *currVP;
293 AnimSlot *currSlot, **animSlots;
294 int32 currSlotIdx;
295 int32 *addrOfMasterLock;
296 SlaveVP *thisCoresMasterVP;
298 //=============== Initializations ===================
299 thisCoresIdx = 0; //sequential version
300 animSlots = _VMSMasterEnv->allAnimSlots[thisCoresIdx];
301 currSlotIdx = 0; //start at slot 0, go up until one empty, then do master
302 numRepetitionsWithNoWork = 0;
303 addrOfMasterLock = &(_VMSMasterEnv->masterLock);
304 thisCoresMasterVP = _VMSMasterEnv->masterVPs[thisCoresIdx];
306 //Assembly that saves addr of label of return instr -- label in assmbly
307 recordCoreCtlrReturnLabelAddr((void**)&(_VMSMasterEnv->coreCtlrReturnPt));
310 //====================== The Core Controller ======================
311 while(1)
312 {
313 if( currSlotIdx >= NUM_ANIM_SLOTS ) goto switchToMaster;
314 currSlot = animSlots[ currSlotIdx ];
316 if( ! currSlot->needsSlaveAssigned ) //slot does have slave assigned
317 { numRepetitionsWithNoWork = 0; //reset B2B master count
318 currSlotIdx ++;
319 currVP = currSlot->slaveAssignedToSlot;
320 }
321 else //slot is empty, so switch to master
322 {
323 switchToMaster:
324 currSlotIdx = 0; //doing switch to master, so start over at slot 0
326 currVP = thisCoresMasterVP;
328 MEAS__Capture_Pre_Master_Lock_Point; //back to back because
329 MEAS__Capture_Post_Master_Lock_Point; // sequential version
331 if( numRepetitionsWithNoWork > NUM_REPS_W_NO_WORK_BEFORE_YIELD )
332 { printf("Lots of reps w/o work\n");
333 exit(0); //if no work, no way to ever get it in sequential!
334 }
335 numRepetitionsWithNoWork += 1;
336 }
338 switchToSlv(currVP); //Slave suspend makes core "return" from this call
339 flushRegisters(); //prevent GCC optimization from doing bad things
341 MEAS__Capture_End_Susp_in_CoreCtlr_ForSys;
343 } //while(1)
344 }
345 #endif