1.1 --- a/MasterLoop.c	Mon Aug 09 02:24:31 2010 -0700
     1.2 +++ b/MasterLoop.c	Wed Sep 01 08:23:39 2010 -0700
     1.3 @@ -42,74 +42,77 @@
     1.4   * all the others -- then does any extra setup needed and puts it into the
     1.5   * work queue.
     1.6   *However means have to make masterEnv a global static volatile the same way
     1.7 - * did with workQ in core loop.  -- for performance, put the
     1.8 + * did with readyToAnimateQ in core loop.  -- for performance, put the
     1.9   * jump to the core loop directly in here, and have it directly jump back.
    1.10 + *
    1.11 + *
    1.12 + *Aug 18, 2010 -- Going to a separate MasterVP for each core, to see if this
    1.13 + * avoids the suspected bug in the system stack that causes bizarre faults
    1.14 + * at random places in the system code.
    1.15 + *
    1.16 + *So, this function is coupled to each of the MasterVPs, -- meaning this
    1.17 + * function can't rely on a particular stack and frame -- each MasterVP that
    1.18 + * animates this function has a different one.
    1.19 + *
    1.20 + *At this point, the masterLoop does not write itself into the queue anymore,
    1.21 + * instead, the coreLoop acquires the masterLock when it has nothing to
    1.22 + * animate, and then animates its own masterLoop.  However, still try to put
    1.23 + * several AppVPs into the queue to amortize the startup cost of switching
    1.24 + * to the MasterVP.  Note, don't have to worry about latency of requests much
    1.25 + * because most requests generate work for same core -- only latency issue
    1.26 + * is case when other cores starved and one core's requests generate work
    1.27 + * for them -- so keep max in queue to 3 or 4..
    1.28   */
    1.29 -void masterLoop( void *initData, VirtProcr *masterPr )
    1.30 +void masterLoop( void *initData, VirtProcr *animatingPr )
    1.31   { 
    1.32 -   int             slotIdx, numFilled, filledSlotIdx, masterHasBeenQueued;
    1.33 +   int             slotIdx;
    1.34     VirtProcr      *schedVirtPr;
    1.35 -   SchedSlot      *currSlot, **schedSlots, **filledSlots;
    1.36 +   SchedSlot      *currSlot, **schedSlots;
    1.37     MasterEnv      *masterEnv;
    1.38 -   VMSQueueStruc  *workQ;
    1.39 -   void           *jmpPt, *stackPtrAddr, *framePtrAddr, *stillRunningAddr;
    1.40 -   void           *coreLoopFramePtr, *coreLoopStackPtr, *semanticEnv;
    1.41 +   VMSQueueStruc  *readyToAnimateQ;
    1.42     
    1.43     SlaveScheduler  slaveScheduler;
    1.44     RequestHandler  requestHandler;
    1.45 +   void           *semanticEnv;
    1.46  
    1.47 -      //this will run as the first virt processor in workQ, and will be a
    1.48 -      // new born -- so will do all the GCC-generated allocating space on
    1.49 -      // the stack owned by master virt procr -- and will run this last bit
    1.50 -      // of setup code..
    1.51 +   int             thisCoresIdx;
    1.52 +   VirtProcr      *masterPr;
    1.53 +   volatile        VirtProcr *volatileMasterPr;
    1.54 +   
    1.55 +   volatileMasterPr = animatingPr;
    1.56 +   masterPr         = volatileMasterPr; //used to force re-define after jmp
    1.57 +
    1.58 +      //First animation of each MasterVP will in turn animate this part
    1.59 +      // of setup code.. (VP creator sets up the stack as if this function
    1.60 +      // was called normally, but actually get here by jmp)
    1.61 +      //So, setup values about stack ptr, jmp pt and all that
    1.62     masterPr->nextInstrPt = &&masterLoopStartPt;
    1.63  
    1.64 -      //The second time MasterVP comes out of queue, the first animation of
    1.65 -      // it hasn't written the stackPtr and framePtr yet -- but the second
    1.66 -      // animation has already had its stackPtr and framePtr set to the old
    1.67 -      // value by the coreLoop.  Fix this by writing the correct stack and
    1.68 -      // frame pointers here, at which point they're correct in the first
    1.69 -      // animation of MasterVP.
    1.70 -      //TODO: remove writing stackPtr and framePtr at the bottom, for eff
    1.71 -   stackPtrAddr      = &(masterPr->stackPtr);
    1.72 -   framePtrAddr      = &(masterPr->framePtr);
    1.73  
    1.74 -   asm volatile("movl %0,     %%eax;  \
    1.75 -                 movl %%esp, (%%eax); \
    1.76 -                 movl %1,     %%eax;  \
    1.77 -                 movl %%ebp, (%%eax); "
    1.78 -   /* outputs */ : "=g" (stackPtrAddr), "=g" (framePtrAddr)                 \
    1.79 -   /* inputs  */ :                                                          \
    1.80 -   /* clobber */ : "memory", "%eax", "%ebx"                                 \
    1.81 -                );
    1.82 +      //Note, got rid of writing the stack and frame ptr up here, because
    1.83 +      // only one
    1.84 +      // core can ever animate a given MasterVP, so don't need to communicate
    1.85 +      // new frame and stack ptr to the MasterVP storage before a second
    1.86 +      // version of that MasterVP can get animated on a different core.
    1.87 +      //Also got rid of the busy-wait.
    1.88  
    1.89 -
    1.90 +   
    1.91     masterLoopStartPt:
    1.92  
    1.93 -      //if another reference to same Master VirtProcr still going, busy-wait
    1.94 -      //Could put this lower, but don't want to think about shared stack..
    1.95 -   while( _VMSMasterEnv->stillRunning ) /*busy wait*/ ;
    1.96 -      //TODO: want to do busy-wait as assembly, to be sure stack not touched?
    1.97 +   masterEnv        = _VMSMasterEnv;
    1.98     
    1.99 -      //this is the only master running now, set flag again
   1.100 -   _VMSMasterEnv->stillRunning = TRUE;
   1.101 -   masterEnv = _VMSMasterEnv;
   1.102 +//TODO: check that compiles so that always re-define from frame-storage
   1.103 +   masterPr         = volatileMasterPr;  //just to make sure after jmp
   1.104 +   thisCoresIdx     = masterPr->coreAnimatedBy;
   1.105 +   readyToAnimateQ  = masterEnv->readyToAnimateQs[thisCoresIdx];
   1.106 +   schedSlots       = masterEnv->allSchedSlots[thisCoresIdx];
   1.107  
   1.108 -      //TODO: gdb -- check that a volatile _VMSMasterEnv and _VMSWorkQ means
   1.109 -      // all these will be re-filled every time jump here..
   1.110 -   workQ            = _VMSWorkQ;
   1.111     requestHandler   = masterEnv->requestHandler;
   1.112     slaveScheduler   = masterEnv->slaveScheduler;
   1.113 -   schedSlots       = masterEnv->schedSlots;
   1.114 -   filledSlots      = masterEnv->filledSlots;
   1.115 -   masterPr         = masterEnv->masterVirtPr;  //post-jmp clobbered, re-load
   1.116     semanticEnv      = masterEnv->semanticEnv;
   1.117  
   1.118 -      //prepare for scheduling
   1.119 -   numFilled = 0;
   1.120 -   masterHasBeenQueued = FALSE;
   1.121  
   1.122 -      //Poll each slot's Done flag -- slot 0 reserved for master, start at 1
   1.123 +      //Poll each slot's Done flag
   1.124     for( slotIdx = 0; slotIdx < NUM_SCHED_SLOTS; slotIdx++)
   1.125      {
   1.126        currSlot = schedSlots[ slotIdx ];
   1.127 @@ -125,63 +128,31 @@
   1.128        if( currSlot->needsProcrAssigned )
   1.129         {    //give slot a new virt procr
   1.130           schedVirtPr =
   1.131 -          (*slaveScheduler)( semanticEnv );
   1.132 +          (*slaveScheduler)( semanticEnv, thisCoresIdx );
   1.133           
   1.134           if( schedVirtPr != NULL )
   1.135            { currSlot->procrAssignedToSlot = schedVirtPr;
   1.136              schedVirtPr->schedSlot        = currSlot;
   1.137              currSlot->needsProcrAssigned  = FALSE;
   1.138  
   1.139 -            filledSlots[ numFilled ]      = currSlot;
   1.140 -
   1.141 -            writeVMSQ( schedVirtPr, workQ );
   1.142 -            numFilled += 1;
   1.143 -            
   1.144 -            if( numFilled == masterEnv->numToPrecede )
   1.145 -             {
   1.146 -               writeVMSQ( masterEnv->masterVirtPr, workQ );
   1.147 -               masterHasBeenQueued = TRUE;
   1.148 -             }
   1.149 -
   1.150 +            writeSRSWQ( schedVirtPr, readyToAnimateQ );
   1.151            }
   1.152         }
   1.153      }
   1.154  
   1.155 -   if( !masterHasBeenQueued )
   1.156 -    {
   1.157 -      writeVMSQ( masterEnv->masterVirtPr, workQ );
   1.158 -    }
   1.159  
   1.160 -      //Adjust the number to precede, for next round -- assume rate of
   1.161 -      // finishing work is stable -- which is a bad assumption!  But, just
   1.162 -      // want something working for the moment, look at dynamic behavior
   1.163 -      // later
   1.164 -//TODO: look at dynamic behavior -- time-average numToPrecede or something
   1.165 -   if( numFilled < NUM_CORES - 1 )
   1.166 -    { 
   1.167 -      masterEnv->numToPrecede = 1;
   1.168 -    }
   1.169 -   else
   1.170 -    { masterEnv->numToPrecede = numFilled - NUM_CORES + 1;
   1.171 -    }
   1.172 -
   1.173 -      //Save stack ptr and frame -- don't need to, take out later, but safe
   1.174 -      // Also, wait to set stillRunning to FALSE until just before jump, to
   1.175 -      // be safe -- although the two simulatneously animated MasterLoops
   1.176 -      // are on different cores, so have different stacks, so no worries
   1.177 -      // there.
   1.178 -      //Restore CoreLoop's stack frame (and stack pointer, to be safe)
   1.179 +      //Save stack ptr and frame, restore CoreLoop's stack and frame,
   1.180 +      // and clear the MasterLock
   1.181        //TODO: cafefully verify don't need to force saving anything to stack
   1.182        // before jumping back to core loop.
   1.183 +   void           *stackPtrAddr, *framePtrAddr, *masterLockAddr;
   1.184 +   void           *jmpPt, *coreLoopFramePtr, *coreLoopStackPtr;
   1.185 +
   1.186     stackPtrAddr      = &(masterPr->stackPtr);
   1.187     framePtrAddr      = &(masterPr->framePtr);
   1.188 -   stillRunningAddr  = &(_VMSMasterEnv->stillRunning); //when race condition
   1.189 -      //arises, stillRunning is shared between the two cores both animating
   1.190 -      // MasterLoop -- but those two cores have different esp & ebp, so safe
   1.191 -      // to change stack and frame pointer here, without one messing up other
   1.192 -      // one
   1.193 +   masterLockAddr    = &(_VMSMasterEnv->masterLock);
   1.194  
   1.195 -   jmpPt             = masterPr->coreLoopStartPt;
   1.196 +   jmpPt             = _VMSMasterEnv->coreLoopStartPt;
   1.197     coreLoopFramePtr  = masterPr->coreLoopFramePtr;//need this only
   1.198     coreLoopStackPtr  = masterPr->coreLoopStackPtr;//shouldn't need -- safety
   1.199     
   1.200 @@ -196,7 +167,7 @@
   1.201                   movl $0x0, (%%ebx);  \
   1.202                   jmp  %%eax;"         \
   1.203     /* outputs */ : "=g" (stackPtrAddr), "=g" (framePtrAddr),                \
   1.204 -                   "=g"(stillRunningAddr)                                   \
   1.205 +                   "=g"(masterLockAddr)                                   \
   1.206     /* inputs  */ : "g" (jmpPt), "g"(coreLoopStackPtr), "g"(coreLoopFramePtr)\
   1.207     /* clobber */ : "memory", "%eax", "%ebx", "%ecx", "%edx", "%edi", "%esi" \
   1.208                  );//can probably make clobber list empty -- but safe for now