1.1 --- a/MasterLoop.c	Wed Sep 07 19:36:46 2011 +0200
     1.2 +++ b/MasterLoop.c	Mon Sep 19 16:12:01 2011 +0200
     1.3 @@ -11,12 +11,21 @@
     1.4  
     1.5  #include "VMS.h"
     1.6  #include "ProcrContext.h"
     1.7 -
     1.8 +#include "scheduling.h"
     1.9 +#include "inter_VMS_requests.h"
    1.10 +#include "inter_VMS_requests_handler.h"
    1.11  
    1.12  //===========================================================================
    1.13  void inline
    1.14  stealWorkInto( SchedSlot *currSlot, VMSQueueStruc *readyToAnimateQ,
    1.15 -               VirtProcr *masterPr );
    1.16 +               VirtProcr *masterPr);
    1.17 +
    1.18 +void inline
    1.19 +handleInterMasterReq( InterMasterReqst *currReq, void *_semEnv,
    1.20 +                                                    VirtProcr *masterPr);
    1.21 +
    1.22 +void inline
    1.23 +handleInterVMSCoreReq( InterVMSCoreReqst *currReq, VirtProcr *masterPr);
    1.24  
    1.25  //===========================================================================
    1.26  
    1.27 @@ -49,9 +58,7 @@
    1.28   *So VMS__init just births the master virtual processor same way it births
    1.29   * all the others -- then does any extra setup needed and puts it into the
    1.30   * work queue.
    1.31 - *However means have to make masterEnv a global static volatile the same way
    1.32 - * did with readyToAnimateQ in core loop.  -- for performance, put the
    1.33 - * jump to the core loop directly in here, and have it directly jump back.
    1.34 + *However means have to make masterEnv a global static volatile.
    1.35   *
    1.36   *
    1.37   *Aug 18, 2010 -- Going to a separate MasterVP for each core, to see if this
    1.38 @@ -60,7 +67,7 @@
    1.39   *
    1.40   *So, this function is coupled to each of the MasterVPs, -- meaning this
    1.41   * function can't rely on a particular stack and frame -- each MasterVP that
    1.42 - * animates this function has a different one.
    1.43 + * animates this function has a different stack.
    1.44   *
    1.45   *At this point, the masterLoop does not write itself into the queue anymore,
    1.46   * instead, the coreLoop acquires the masterLock when it has nothing to
    1.47 @@ -89,39 +96,30 @@
    1.48     
    1.49     volatileMasterPr = animatingPr;
    1.50     masterPr         = (VirtProcr*)volatileMasterPr; //used to force re-define after jmp
    1.51 +   masterEnv        = (MasterEnv*)_VMSMasterEnv;
    1.52  
    1.53        //First animation of each MasterVP will in turn animate this part
    1.54        // of setup code.. (VP creator sets up the stack as if this function
    1.55        // was called normally, but actually get here by jmp)
    1.56 -      //So, setup values about stack ptr, jmp pt and all that
    1.57 -   //masterPr->nextInstrPt = &&masterLoopStartPt;
    1.58  
    1.59 -
    1.60 -      //Note, got rid of writing the stack and frame ptr up here, because
    1.61 -      // only one
    1.62 -      // core can ever animate a given MasterVP, so don't need to communicate
    1.63 -      // new frame and stack ptr to the MasterVP storage before a second
    1.64 -      // version of that MasterVP can get animated on a different core.
    1.65 -      //Also got rid of the busy-wait.
    1.66 -
    1.67 -   
    1.68 -   //masterLoopStartPt:
    1.69 -   while(1){
    1.70 -       
    1.71 +      //Sept 2011
    1.72 +      //Old code jumped directly to this point, but doesn't work on x64
    1.73 +	  // So, just make this an endless loop, and do assembly function at end
    1.74 +	  // that saves its own return addr, then jumps to core_loop.
    1.75 +   while(1)
    1.76 +   {       
    1.77     //============================= MEASUREMENT STUFF ========================
    1.78     #ifdef MEAS__TIME_MASTER
    1.79        //Total Master time includes one coreloop time -- just assume the core
    1.80 -      // loop time is same for Master as for AppVPs, even though it may be
    1.81 +      // loop time is same for Master as is for AppVPs, even though it may be
    1.82        // smaller due to higher predictability of the fixed jmp.
    1.83     saveLowTimeStampCountInto( masterPr->startMasterTSCLow );
    1.84     #endif
    1.85     //========================================================================
    1.86  
    1.87 -   masterEnv        = (MasterEnv*)_VMSMasterEnv;
    1.88 -   
    1.89 -      //GCC may optimize so doesn't always re-define from frame-storage
    1.90 -   masterPr         = (VirtProcr*)volatileMasterPr;  //just to make sure after jmp
    1.91 +   //GCC may optimize so doesn't always re-define from frame-storage
    1.92     thisCoresIdx     = masterPr->coreAnimatedBy;
    1.93 +   masterEnv->currentMasterProcrID = thisCoresIdx;
    1.94     readyToAnimateQ  = masterEnv->readyToAnimateQs[thisCoresIdx];
    1.95     schedSlots       = masterEnv->allSchedSlots[thisCoresIdx];
    1.96  
    1.97 @@ -129,8 +127,28 @@
    1.98     slaveScheduler   = masterEnv->slaveScheduler;
    1.99     semanticEnv      = masterEnv->semanticEnv;
   1.100  
   1.101 -
   1.102 -      //Poll each slot's Done flag
   1.103 +      //First, check for requests from other MasterVPs, and handle them
   1.104 +   InterMasterReqst* currReqst = masterEnv->interMasterRequestsFor[thisCoresIdx];
   1.105 +   while(currReqst)
   1.106 +   {
   1.107 +       handleInterMasterReq( currReqst, semanticEnv, masterPr );
   1.108 +       currReqst = currReqst->nextReqst;
   1.109 +   }
   1.110 +   masterEnv->interMasterRequestsFor[thisCoresIdx] = NULL;
   1.111 +   
   1.112 +   //Second, check for own request that were handled for other MasterVPs
   1.113 +   currReqst = masterEnv->interMasterRequestsSentBy[thisCoresIdx];
   1.114 +   while(currReqst && currReqst->obsolete)
   1.115 +   {
   1.116 +       InterMasterReqst *nextReqst = currReqst->nextSentReqst;
   1.117 +       VMS__free(currReqst);
   1.118 +       currReqst = nextReqst;
   1.119 +   }
   1.120 +   masterEnv->interMasterRequestsSentBy[thisCoresIdx] = currReqst;
   1.121 +   
   1.122 +      //Now, take care of the SlaveVPs
   1.123 +      //Go through the slots -- if Slave there newly suspended, handle its request
   1.124 +      // then, either way, ask assigner to fill each slot
   1.125     numSlotsFilled = 0;
   1.126     for( slotIdx = 0; slotIdx < NUM_SCHED_SLOTS; slotIdx++)
   1.127      {
   1.128 @@ -167,6 +185,7 @@
   1.129           if( schedVirtPr != NULL )
   1.130            { currSlot->procrAssignedToSlot = schedVirtPr;
   1.131              schedVirtPr->schedSlot        = currSlot;
   1.132 +            schedVirtPr->coreAnimatedBy   = thisCoresIdx;
   1.133              currSlot->needsProcrAssigned  = FALSE;
   1.134              numSlotsFilled               += 1;
   1.135              
   1.136 @@ -190,15 +209,56 @@
   1.137  
   1.138     masterSwitchToCoreLoop(animatingPr);
   1.139     flushRegisters();
   1.140 -   }//MasterLoop
   1.141 -
   1.142 -
   1.143 +   }//while(1)   MasterLoop
   1.144   }
   1.145  
   1.146 +/*This is for inter-master communication.  Either the master itself or
   1.147 + * the plugin sends one of these requests.  Some are handled here, by the
   1.148 + * master_loop, others are handed off to the plugin.
   1.149 + */
   1.150 +void inline
   1.151 +handleInterMasterReq( InterMasterReqst *currReq, void *_semEnv,
   1.152 +                                                    VirtProcr *masterPr )
   1.153 + { 
   1.154 +    
   1.155 +    switch( currReq->reqType )
   1.156 +    {  
   1.157 +      case destVMSCore:
   1.158 +         handleInterVMSCoreReq( (InterVMSCoreReqst *)currReq, masterPr);
   1.159 +         break;
   1.160 +      case destPlugin:
   1.161 +         _VMSMasterEnv->interPluginReqHdlr( ((InterPluginReqst  *)currReq)->pluginReq,
   1.162 +                                                                    _semEnv );
   1.163 +          break;
   1.164 +      default:
   1.165 +         break;
   1.166 +    }
   1.167 + }
   1.168  
   1.169 +void inline
   1.170 +handleInterVMSCoreReq( InterVMSCoreReqst *currReq, VirtProcr *masterPr )
   1.171 + { 
   1.172 +   switch( currReq->secondReqType )
   1.173 +    {
   1.174 +      case transfer_free_ptr:
   1.175 +          handleTransferFree( currReq, masterPr );
   1.176 +          currReq->obsolete = 1; //now the sender can free the structure
   1.177 +          break;
   1.178 +       case shutdownVP:
   1.179 +           currReq->obsolete = 1;
   1.180 +           handleShutdown(currReq, masterPr); 
   1.181 +           //The Execution of the MasterLoop ends here
   1.182 +           break;
   1.183 +      default:
   1.184 +          break;
   1.185 +    }
   1.186 +}
   1.187  
   1.188 -/*This has a race condition -- the coreloops are accessing their own queues
   1.189 - * at the same time that this work-stealer on a different core is trying to
   1.190 +/*Work Stealing Alg -- racy one
   1.191 + *This algorithm has a race condition -- the coreloops are accessing their
   1.192 + * own queues at the same time that this work-stealer on a different core
   1.193 + * is trying to.
   1.194 + *The second stealing alg, below, protects against this.
   1.195   */
   1.196  void inline
   1.197  stealWorkInto( SchedSlot *currSlot, VMSQueueStruc *readyToAnimateQ,
   1.198 @@ -234,7 +294,8 @@
   1.199      }
   1.200   }
   1.201  
   1.202 -/*This algorithm makes the common case fast.  Make the coreloop passive,
   1.203 +/*Work Stealing alg -- protected one
   1.204 + *This algorithm makes the common case fast.  Make the coreloop passive,
   1.205   * and show its progress.  Make the stealer control a gate that coreloop
   1.206   * has to pass.
   1.207   *To avoid interference, only one stealer at a time.  Use a global
   1.208 @@ -360,7 +421,7 @@
   1.209     //======= End Gate-protection  =======
   1.210  
   1.211  
   1.212 -   if( stolenPr != NULL )  //victim could have been in protected and taken
   1.213 +   if( stolenPr != NULL )  //victim could have been in protected and took it
   1.214      { currSlot->procrAssignedToSlot = stolenPr;
   1.215        stolenPr->schedSlot           = currSlot;
   1.216        currSlot->needsProcrAssigned  = FALSE;