1.1 --- a/CoreLoop.c	Tue Aug 30 21:55:04 2011 -0700
     1.2 +++ b/CoreLoop.c	Sat Sep 03 20:41:51 2011 -0700
     1.3 @@ -91,8 +91,8 @@
     1.4     readyToAnimateQ  = _VMSMasterEnv->readyToAnimateQs[thisCoresIdx];
     1.5  
     1.6     #ifdef USE_WORK_STEALING
     1.7 -      //Alg for work-stealing designed to make common case fast.  Comment
     1.8 -      // in stealer code explains.
     1.9 +      //protect access to readyToAnimateQ -- other cores also want access!
    1.10 +      //Alg makes common case fast.  Comment in stealer code explains.
    1.11     gate.preGateProgress++;
    1.12     if( gate.gateClosed )
    1.13      {    //now, set coreloop's progress, so stealer can see that core loop
    1.14 @@ -103,7 +103,7 @@
    1.15  
    1.16     currPr = (VirtProcr *) readVMSQ( readyToAnimateQ );
    1.17  
    1.18 -      //Set the coreloop's progress, so stealer can see it has made it out
    1.19 +      //Set coreloop's progress, so stealer can see this has made it out
    1.20        // of the protected area
    1.21     gate.exitProgress = gate.preGateProgress;
    1.22     #else
    1.23 @@ -112,7 +112,8 @@
    1.24  
    1.25     if( currPr != NULL ) _VMSMasterEnv->numMasterInARow[thisCoresIdx] = 0;
    1.26     else
    1.27 -    {
    1.28 +    { //no more Slaves, get master lock and switch to master Pr
    1.29 +	
    1.30        //============================= MEASUREMENT STUFF =====================
    1.31        #ifdef MEAS__TIME_MASTER_LOCK
    1.32        int32 startStamp, endStamp;
    1.33 @@ -120,17 +121,17 @@
    1.34        #endif
    1.35        //=====================================================================
    1.36        int tries = 0; int gotLock = 0;
    1.37 -      while( currPr == NULL ) //if queue was empty, enter get masterLock loop
    1.38 -       {    //queue was empty, so get master lock
    1.39 +      while( currPr == NULL ) 
    1.40 +       {    //didn't get lock, so keep trying
    1.41  
    1.42           gotLock = __sync_bool_compare_and_swap(&(_VMSMasterEnv->masterLock),
    1.43                                                            UNLOCKED, LOCKED );
    1.44           if( gotLock )
    1.45 -          {    //run own MasterVP -- jmps to coreLoops startPt when done
    1.46 +          {    //run own MasterVP
    1.47              currPr = _VMSMasterEnv->masterVPs[thisCoresIdx];
    1.48              if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 1000 )
    1.49               {       DEBUG( dbgB2BMaster,"Many back to back MasterVPs\n");
    1.50 -               pthread_yield();
    1.51 +               pthread_yield();   //this core has no slaves to schedule..
    1.52               }
    1.53              _VMSMasterEnv->numMasterInARow[thisCoresIdx] += 1;
    1.54              break;  //end while -- have a VP to animate now
    1.55 @@ -152,7 +153,7 @@
    1.56      }
    1.57  
    1.58     
    1.59 -   switchToVP(currPr); //The VPs return in here
    1.60 +   switchToVP(currPr); //The VPs all return back to here
    1.61     flushRegisters();
    1.62     }//CoreLoop      
    1.63   }
    1.64 @@ -160,7 +161,7 @@
    1.65  
    1.66  void *
    1.67  terminateCoreLoop(VirtProcr *currPr){
    1.68 -   //first free shutdown VP that jumped here -- it first restores the
    1.69 +   //first free the shutdown VP that jumped here -- it first restores the
    1.70     // coreloop's stack, so addr of currPr in stack frame is still correct
    1.71     VMS__dissipate_procr( currPr );
    1.72     pthread_exit( NULL );

     2.1 --- a/MasterLoop.c	Tue Aug 30 21:55:04 2011 -0700
     2.2 +++ b/MasterLoop.c	Sat Sep 03 20:41:51 2011 -0700
     2.3 @@ -49,9 +49,7 @@
     2.4   *So VMS__init just births the master virtual processor same way it births
     2.5   * all the others -- then does any extra setup needed and puts it into the
     2.6   * work queue.
     2.7 - *However means have to make masterEnv a global static volatile the same way
     2.8 - * did with readyToAnimateQ in core loop.  -- for performance, put the
     2.9 - * jump to the core loop directly in here, and have it directly jump back.
    2.10 + *However means have to make masterEnv a global static volatile.
    2.11   *
    2.12   *
    2.13   *Aug 18, 2010 -- Going to a separate MasterVP for each core, to see if this
    2.14 @@ -60,7 +58,7 @@
    2.15   *
    2.16   *So, this function is coupled to each of the MasterVPs, -- meaning this
    2.17   * function can't rely on a particular stack and frame -- each MasterVP that
    2.18 - * animates this function has a different one.
    2.19 + * animates this function has a different stack.
    2.20   *
    2.21   *At this point, the masterLoop does not write itself into the queue anymore,
    2.22   * instead, the coreLoop acquires the masterLock when it has nothing to
    2.23 @@ -96,24 +94,17 @@
    2.24        //So, setup values about stack ptr, jmp pt and all that
    2.25     //masterPr->nextInstrPt = &&masterLoopStartPt;
    2.26  
    2.27 -
    2.28 -      //Note, got rid of writing the stack and frame ptr up here, because
    2.29 -      // only one
    2.30 -      // core can ever animate a given MasterVP, so don't need to communicate
    2.31 -      // new frame and stack ptr to the MasterVP storage before a second
    2.32 -      // version of that MasterVP can get animated on a different core.
    2.33 -      //Also got rid of the busy-wait.
    2.34 -
    2.35 -   
    2.36 -   //masterLoopStartPt:
    2.37 -   //The animating materVP suspends at end of this loop, then later resumes and
    2.38 -   // comes back here 
    2.39 -   while(1){
    2.40 +      //Sept 2011
    2.41 +      //Old code jumped directly to this point, but doesn't work on x64
    2.42 +	  // So, just make this an endless loop, and do assembly function at end
    2.43 +	  // that saves its own return addr, then jumps to core_loop.
    2.44 +   while(1)
    2.45 +   {
    2.46         
    2.47     //============================= MEASUREMENT STUFF ========================
    2.48     #ifdef MEAS__TIME_MASTER
    2.49        //Total Master time includes one coreloop time -- just assume the core
    2.50 -      // loop time is same for Master as for AppVPs, even though it may be
    2.51 +      // loop time is same for Master as is for AppVPs, even though it may be
    2.52        // smaller due to higher predictability of the fixed jmp.
    2.53     saveLowTimeStampCountInto( masterPr->startMasterTSCLow );
    2.54     #endif
    2.55 @@ -122,7 +113,7 @@
    2.56     masterEnv        = (MasterEnv*)_VMSMasterEnv;
    2.57     
    2.58        //GCC may optimize so doesn't always re-define from frame-storage
    2.59 -   masterPr         = (VirtProcr*)volatileMasterPr; //on stack, to be sure after jmp
    2.60 +   masterPr         = (VirtProcr*)volatileMasterPr; //on stack, reload after jmp
    2.61     thisCoresIdx     = masterPr->coreAnimatedBy;
    2.62     readyToAnimateQ  = masterEnv->readyToAnimateQs[thisCoresIdx];
    2.63     schedSlots       = masterEnv->allSchedSlots[thisCoresIdx];
    2.64 @@ -132,15 +123,15 @@
    2.65     semanticEnv      = masterEnv->semanticEnv;
    2.66  
    2.67        //First, check for requests from other MasterVPs, and handle them
    2.68 -   if( masterEnv->requestsWaitingFor[thisCoresIdx] )
    2.69 -    { masterReqQ = masterEnv->masterReqQs[thisCoresIdx];
    2.70 -	  while( currReq = readVMSQ(masterReqQ) )
    2.71 -       { handleMasterReq( currReq, semanticEnv, masterPr );
    2.72 -       }	   
    2.73 -	}
    2.74 +   if( currReq = masterEnv->interMasterRequestsFor[thisCoresIdx] )
    2.75 +    { do
    2.76 +       { handleInterMasterReq( currReq, semanticEnv, masterPr );
    2.77 +       }
    2.78 +      while( currReq = currReq->nextReqst );
    2.79 +    }
    2.80        //Now, take care of the SlaveVPs
    2.81 -	  //Go through the slots -- if Slave there newly suspended, handle its request
    2.82 -	  // then, either way, ask assigner to fill each slot
    2.83 +      //Go through the slots -- if Slave there newly suspended, handle its request
    2.84 +      // then, either way, ask assigner to fill each slot
    2.85     numSlotsFilled = 0;
    2.86     for( slotIdx = 0; slotIdx < NUM_SCHED_SLOTS; slotIdx++)
    2.87      {
    2.88 @@ -208,14 +199,15 @@
    2.89   * master_loop, others are handed off to the plugin.
    2.90   */
    2.91  void inline
    2.92 -handleMasterReq( MasterReq *currReq, void *_semEnv, VirtProcr *masterPr )
    2.93 - {
    2.94 -   switch( currReq->reqType )
    2.95 -    { case interVMSReq:
    2.96 -            handleInterVMSReq(     (InterVMSCoreReq *)currReq, masterPr);
    2.97 +handleInterMasterReq( InterMasterReqst *currReq, void *_semEnv,
    2.98 +                                                    VirtProcr *masterPr )
    2.99 + { switch( currReq->reqType )
   2.100 +    { case destVMSCore:
   2.101 +            handleInterVMSCoreReq( (InterVMSCoreReqst *)currReq, masterPr);
   2.102           break;
   2.103 -      case interPluginReq:
   2.104 -	       (*interPluginReqHdlr)(  (InterPluginReq  *)currReq, _semEnv );
   2.105 +      case destPlugin:
   2.106 +	       (*interPluginReqHdlr)( ((InterPluginReqst  *)currReq)->pluginReq,
   2.107 +                                                                _semEnv );
   2.108           break;
   2.109        default:
   2.110           break;
   2.111 @@ -223,15 +215,16 @@
   2.112   }
   2.113  
   2.114  void inline
   2.115 -handleInterVMSReq( InterVMSCoreReq *currReq, VirtProcr *masterPr )
   2.116 +handleInterVMSReq( InterVMSCoreReqst *currReq, VirtProcr *masterPr )
   2.117   { 
   2.118     switch( currReq->reqType )
   2.119      {
   2.120 -      case transfer_free:        handleTransferFree( currReq, masterPr );
   2.121 +      case transfer_free_ptr:    handleTransferFree( currReq, masterPr );
   2.122           break;
   2.123      }
   2.124   }
   2.125   
   2.126 + 
   2.127  
   2.128  /*Work Stealing Alg -- racy one
   2.129   *This algorithm has a race condition -- the coreloops are accessing their

     3.1 --- a/VMS.c	Tue Aug 30 21:55:04 2011 -0700
     3.2 +++ b/VMS.c	Sat Sep 03 20:41:51 2011 -0700
     3.3 @@ -497,6 +497,19 @@
     3.4     VMS__suspend_procr( callingPr );
     3.5   }
     3.6  
     3.7 +void inline
     3.8 +VMS__send_inter_plugin_req( void *reqData, int32 targetMaster, 
     3.9 +                                            VirtProcr *requestingMaster )
    3.10 + { _VMSMasterEnv->interMasterRequestsFor[targetMaster] = 
    3.11 +                                            (InterMasterReqst *) reqData;
    3.12 + }
    3.13 +
    3.14 +void inline
    3.15 +VMS__send_inter_VMSCore_req( InterVMSCoreReqst *reqData,
    3.16 +                        int32 targetMaster, VirtProcr *requestingMaster )
    3.17 + { _VMSMasterEnv->interMasterRequestsFor[targetMaster] = 
    3.18 +                                            (InterMasterReqst *) reqData;
    3.19 + }
    3.20  
    3.21  /*
    3.22   */

     4.1 --- a/VMS.h	Tue Aug 30 21:55:04 2011 -0700
     4.2 +++ b/VMS.h	Sat Sep 03 20:41:51 2011 -0700
     4.3 @@ -113,7 +113,7 @@
     4.4  
     4.5  typedef struct _SchedSlot     SchedSlot;
     4.6  typedef struct _VMSReqst      VMSReqst;
     4.7 -typedef struct _VirtProcr     VirtProcr;
     4.8 +typedef struct _InterMasterReqst InterMasterReqst;
     4.9  typedef struct _IntervalProbe IntervalProbe;
    4.10  typedef struct _GateStruc     GateStruc;
    4.11  
    4.12 @@ -128,9 +128,12 @@
    4.13  //============= Requests ===========
    4.14  //
    4.15  
    4.16 -enum VMSReqstType   //avoid starting enums at 0, for debug reasons
    4.17 - {
    4.18 -   semantic = 1,
    4.19 +//VMS Request is the carrier for Slave to Master requests
    4.20 +// it has an embedded sub-type request that is pulled out
    4.21 +// inside the plugin's request handler
    4.22 +enum VMSReqstType   //For Slave->Master requests
    4.23 + { 
    4.24 +   semantic = 1,    //avoid starting enums at 0, for debug reasons
    4.25     createReq,
    4.26     dissipate,
    4.27     VMSSemantic      //goes with VMSSemReqst below
    4.28 @@ -145,6 +148,9 @@
    4.29   };
    4.30  //VMSReqst
    4.31  
    4.32 +//This is a sub-type of Slave->Master requests.
    4.33 +// It's for Slaves to invoke built-in VMS-core functions that have language-like
    4.34 +// behavior.
    4.35  enum VMSSemReqstType   //These are equivalent to semantic requests, but for
    4.36   {                     // VMS's services available directly to app, like OS
    4.37     createProbe = 1,    // and probe services -- like a VMS-wide built-in lang
    4.38 @@ -157,7 +163,53 @@
    4.39     VirtProcr           *requestingPr;
    4.40     char                *nameStr;  //for create probe
    4.41   }
    4.42 - VMSSemReq;
    4.43 +VMSSemReq;
    4.44 +
    4.45 +//These are for Master to Master requests
    4.46 +// They get re-cast to the appropriate sub-type of request
    4.47 +enum InterMasterReqstType    //For Master->Master
    4.48 + {
    4.49 +   destVMSCore = 1,          //avoid starting enums at 0, for debug reasons
    4.50 +   destPlugin
    4.51 + };
    4.52 +
    4.53 +struct _InterMasterReqst //Doing a trick to save space & time -- allocate
    4.54 + {  // space for a sub-type then cast first as InterMaster then as sub-type
    4.55 +   enum InterMasterReqstType  reqType;
    4.56 +   InterMasterReqst *nextReqst;
    4.57 + };
    4.58 +//InterMasterReqst  (defined above in typedef block)
    4.59 +
    4.60 +
    4.61 +//These are a sub-type of InterMaster requests.  The inter-master req gets
    4.62 +// re-cast to be of this type, after checking
    4.63 +//This ones for requests between internals of VMS-core.. such as malloc
    4.64 +enum InterVMSCoreReqType   
    4.65 + {
    4.66 +   transfer_free_ptr = 1     //avoid starting enums at 0, for debug reasons
    4.67 + };
    4.68 +
    4.69 +typedef struct  //Doing a trick to save space & time -- allocate space
    4.70 + {              // for this, cast first as InterMaster then as this
    4.71 +   enum InterMasterReqstType  reqType;  //duplicate InterMasterReqst at top
    4.72 +   InterMasterReqst *nextReqst;
    4.73 +   
    4.74 +   enum InterVMSCoreReqType  secondReqType;
    4.75 +   void                     *freePtr;  //pile up fields, add as needed
    4.76 + }
    4.77 +InterVMSCoreReqst;
    4.78 +
    4.79 +
    4.80 +//This is for requests between plugins on different cores
    4.81 +// Here, after casting, the pluginReq is extracted and handed to plugin
    4.82 +typedef struct  //Doing a trick to save space & time -- allocate space
    4.83 + {              // for this, cast first as InterMaster then as this
    4.84 +   enum InterMasterReqstType  reqType;  //copy InterMasterReqst at top
    4.85 +   InterMasterReqst          *nextReqst;
    4.86 +   
    4.87 +   void                      *pluginReq; //plugin will cast to approp type
    4.88 + }
    4.89 +InterPluginReqst;
    4.90  
    4.91  
    4.92  //====================  Core data structures  ===================
    4.93 @@ -209,9 +261,8 @@
    4.94  //VirtProcr
    4.95  
    4.96  
    4.97 -/*WARNING: re-arranging this data structure could cause VP-switching
    4.98 - *         assembly code to fail -- hard-codes offsets of fields
    4.99 - *         (because -O3 messes with things otherwise)
   4.100 +/*Master Env is the only global variable -- has entry points for any other
   4.101 + * data needed.  
   4.102   */
   4.103  typedef struct
   4.104   {
   4.105 @@ -368,6 +419,14 @@
   4.106  inline void
   4.107  VMS__send_VMSSem_request( void *semReqData, VirtProcr *callingPr );
   4.108  
   4.109 +
   4.110 +void inline
   4.111 +VMS__send_inter_plugin_req( void *reqData, int32 targetMaster, 
   4.112 +                                            VirtProcr *requestingMaster );
   4.113 +void inline
   4.114 +VMS__send_inter_VMSCore_req( InterVMSCoreReqst *reqData, int32 targetMaster,
   4.115 +                                           VirtProcr *requestingMaster );
   4.116 +
   4.117  VMSReqst *
   4.118  VMS__take_next_request_out_of( VirtProcr *procrWithReq );
   4.119  

     5.1 --- a/inter_VMS_request_handlers.h	Tue Aug 30 21:55:04 2011 -0700
     5.2 +++ b/inter_VMS_request_handlers.h	Sat Sep 03 20:41:51 2011 -0700
     5.3 @@ -16,7 +16,7 @@
     5.4   */
     5.5  
     5.6  inline void
     5.7 -handleMakeMutex(  VPThdSemReq *semReq, VPThdSemEnv *semEnv);
     5.8 +handleTransferFree( MasterReq *masterReq, VirtProcr *masterPr );
     5.9  
    5.10  
    5.11  #endif	/* _MASTER_REQ_H */