changeset 169:d1dd9e6ee72c malloc2 version_that_goes_into_paper

VMS preprocessor definitions moved to VMS_defs.h and changes to measurement
author Merten Sach <msach@mailbox.tu-berlin.de>
date Fri, 16 Dec 2011 20:00:21 +0100
parents cb7277bac147
children 92d97c6c09d6
files CoreLoop.c MasterLoop.c ProcrContext.h VMS.c VMS.h VMS_defs.h
diffstat 6 files changed, 447 insertions(+), 380 deletions(-) [+]
line diff
     1.1 --- a/CoreLoop.c	Thu Oct 06 16:25:59 2011 +0200
     1.2 +++ b/CoreLoop.c	Fri Dec 16 20:00:21 2011 +0100
     1.3 @@ -16,7 +16,7 @@
     1.4  #include <pthread.h>
     1.5  #include <sched.h>
     1.6  
     1.7 -void *terminateCoreLoop(VirtProcr *currPr);
     1.8 +void *terminateCoreLoop(VirtProcr *currVP);
     1.9  
    1.10  /*This is the loop that runs in the OS Thread pinned to each core
    1.11   *Get virt procr from queue,
    1.12 @@ -35,10 +35,12 @@
    1.13   { 
    1.14     ThdParams      *coreLoopThdParams;
    1.15     int             thisCoresIdx;
    1.16 -   VirtProcr      *currPr;
    1.17 -   VMSQueueStruc *readyToAnimateQ;
    1.18 -   cpu_set_t   coreMask;  //has 1 in bit positions of allowed cores
    1.19 +   VirtProcr      *currVP;
    1.20 +   VMSQueueStruc  *readyToAnimateQ;
    1.21 +   cpu_set_t       coreMask;  //has 1 in bit positions of allowed cores
    1.22     int             errorCode;
    1.23 +   TSCountLowHigh  endSusp;
    1.24 +   uint64          numCycles;
    1.25  
    1.26        //work-stealing struc on stack to prevent false-sharing in cache-line
    1.27     volatile GateStruc gate;
    1.28 @@ -101,16 +103,16 @@
    1.29        while( gate.gateClosed ) /*busy wait*/;
    1.30      }
    1.31  
    1.32 -   currPr = (VirtProcr *) readVMSQ( readyToAnimateQ );
    1.33 +   currVP = (VirtProcr *) readVMSQ( readyToAnimateQ );
    1.34  
    1.35        //Set the coreloop's progress, so stealer can see it has made it out
    1.36        // of the protected area
    1.37     gate.exitProgress = gate.preGateProgress;
    1.38     #else
    1.39 -   currPr = (VirtProcr *) readVMSQ( readyToAnimateQ );
    1.40 +   currVP = (VirtProcr *) readVMSQ( readyToAnimateQ );
    1.41     #endif
    1.42  
    1.43 -   if( currPr != NULL ) _VMSMasterEnv->numMasterInARow[thisCoresIdx] = 0;
    1.44 +   if( currVP != NULL ) _VMSMasterEnv->numMasterInARow[thisCoresIdx] = 0;
    1.45     else
    1.46      {
    1.47        //============================= MEASUREMENT STUFF =====================
    1.48 @@ -120,14 +122,14 @@
    1.49        #endif
    1.50        //=====================================================================
    1.51        int tries = 0; int gotLock = 0;
    1.52 -      while( currPr == NULL ) //if queue was empty, enter get masterLock loop
    1.53 +      while( currVP == NULL ) //if queue was empty, enter get masterLock loop
    1.54         {    //queue was empty, so get master lock
    1.55  
    1.56           gotLock = __sync_bool_compare_and_swap(&(_VMSMasterEnv->masterLock),
    1.57                                                            UNLOCKED, LOCKED );
    1.58           if( gotLock )
    1.59            {    //run own MasterVP -- jmps to coreLoops startPt when done
    1.60 -            currPr = _VMSMasterEnv->masterVPs[thisCoresIdx];
    1.61 +            currVP = _VMSMasterEnv->masterVPs[thisCoresIdx];
    1.62              if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 1000 )
    1.63               {       DEBUG( dbgB2BMaster,"Many back to back MasterVPs\n");
    1.64                 pthread_yield();
    1.65 @@ -152,17 +154,33 @@
    1.66      }
    1.67  
    1.68     
    1.69 -   switchToVP(currPr); //The VPs return in here
    1.70 +   switchToVP(currVP); //enter "Fn" to resume a VP -- comes back at VP suspend
    1.71     flushRegisters();
    1.72 +          //===================  Meas  =======================
    1.73 +          //A TSC is stored in VP first thing inside wrapper-lib
    1.74 +          //Now, measures cycles from there to here
    1.75 +          //Master and Plugin will add this value to other trace-seg measures
    1.76 +          #ifdef MEAS__TIME_2011_SYS
    1.77 +          saveTSCLowHigh(endSusp);
    1.78 +          numCycles = endSusp.longVal - currVP->startSusp.longVal;
    1.79 +          //sanity check (400K is about 20K iters)
    1.80 +          if( numCycles < 400000 ) 
    1.81 +           { currVP->totalSuspCycles += numCycles;
    1.82 +             currVP->numGoodSusp++;
    1.83 +           }
    1.84 +             //only used if currVP == MasterVP 
    1.85 +          _VMSMasterEnv->startMaster.longVal = endSusp.longVal;
    1.86 +          #endif
    1.87 +          //==================================================
    1.88     }//CoreLoop      
    1.89   }
    1.90  
    1.91  
    1.92  void *
    1.93 -terminateCoreLoop(VirtProcr *currPr){
    1.94 +terminateCoreLoop(VirtProcr *currVP){
    1.95     //first free shutdown VP that jumped here -- it first restores the
    1.96 -   // coreloop's stack, so addr of currPr in stack frame is still correct
    1.97 -   VMS__dissipate_procr( currPr );
    1.98 +   // coreloop's stack, so addr of currVP in stack frame is still correct
    1.99 +   VMS__dissipate_procr( currVP );
   1.100     pthread_exit( NULL );
   1.101  }
   1.102  
   1.103 @@ -177,7 +195,7 @@
   1.104  void *
   1.105  coreLoop_Seq( void *paramsIn )
   1.106   {
   1.107 -   VirtProcr      *currPr;
   1.108 +   VirtProcr      *currVP;
   1.109     VMSQueueStruc *readyToAnimateQ;
   1.110     
   1.111     ThdParams      *coreLoopThdParams;
   1.112 @@ -196,19 +214,19 @@
   1.113        //_VMSWorkQ must be a global, static volatile var, so not kept in reg,
   1.114        // which forces reloading the pointer after each jmp to this point
   1.115     readyToAnimateQ  = _VMSMasterEnv->readyToAnimateQs[thisCoresIdx];
   1.116 -   currPr = (VirtProcr *) readVMSQ( readyToAnimateQ );
   1.117 -   if( currPr == NULL )
   1.118 +   currVP = (VirtProcr *) readVMSQ( readyToAnimateQ );
   1.119 +   if( currVP == NULL )
   1.120      { if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 1000 )
   1.121         { printf("too many back to back MasterVP\n"); exit(1); }
   1.122        _VMSMasterEnv->numMasterInARow[thisCoresIdx] += 1;
   1.123        
   1.124 -      currPr = _VMSMasterEnv->masterVPs[thisCoresIdx];
   1.125 +      currVP = _VMSMasterEnv->masterVPs[thisCoresIdx];
   1.126      }
   1.127     else
   1.128        _VMSMasterEnv->numMasterInARow[thisCoresIdx] = 0;
   1.129  
   1.130  
   1.131 -   switchToVP( currPr );
   1.132 +   switchToVP( currVP );
   1.133     flushRegisters();
   1.134     }
   1.135   }
     2.1 --- a/MasterLoop.c	Thu Oct 06 16:25:59 2011 +0200
     2.2 +++ b/MasterLoop.c	Fri Dec 16 20:00:21 2011 +0100
     2.3 @@ -16,7 +16,7 @@
     2.4  //===========================================================================
     2.5  void inline
     2.6  stealWorkInto( SchedSlot *currSlot, VMSQueueStruc *readyToAnimateQ,
     2.7 -               VirtProcr *masterPr );
     2.8 +               VirtProcr *masterVP );
     2.9  
    2.10  //===========================================================================
    2.11  
    2.12 @@ -84,44 +84,34 @@
    2.13     void           *semanticEnv;
    2.14  
    2.15     int32           thisCoresIdx;
    2.16 -   VirtProcr      *masterPr;
    2.17 -   volatile        VirtProcr *volatileMasterPr;
    2.18 +   VirtProcr      *masterVP;
    2.19 +   volatile        VirtProcr *volatilemasterVP;
    2.20     
    2.21 -   volatileMasterPr = animatingPr;
    2.22 -   masterPr         = (VirtProcr*)volatileMasterPr; //used to force re-define after jmp
    2.23 -
    2.24 -      //First animation of each MasterVP will in turn animate this part
    2.25 -      // of setup code.. (VP creator sets up the stack as if this function
    2.26 -      // was called normally, but actually get here by jmp)
    2.27 -      //So, setup values about stack ptr, jmp pt and all that
    2.28 -   //masterPr->nextInstrPt = &&masterLoopStartPt;
    2.29 -
    2.30 -
    2.31 -      //Note, got rid of writing the stack and frame ptr up here, because
    2.32 -      // only one
    2.33 -      // core can ever animate a given MasterVP, so don't need to communicate
    2.34 -      // new frame and stack ptr to the MasterVP storage before a second
    2.35 -      // version of that MasterVP can get animated on a different core.
    2.36 -      //Also got rid of the busy-wait.
    2.37 -
    2.38 +   volatilemasterVP = animatingPr;
    2.39 +   masterVP         = (VirtProcr*)volatilemasterVP; //used to force re-define after jmp
    2.40 +   
    2.41 +      //======================  Measurement  =====================
    2.42 +       TSCountLowHigh endMaster;
    2.43 +       uint64 numCycles;
    2.44 +      //==========================================================
    2.45     
    2.46     //masterLoopStartPt:
    2.47 -   while(1){
    2.48 +   while(1){    //switch to core_loop and back to here is at end of loop
    2.49         
    2.50 -   //============================= MEASUREMENT STUFF ========================
    2.51 -   #ifdef MEAS__TIME_MASTER
    2.52 -      //Total Master time includes one coreloop time -- just assume the core
    2.53 -      // loop time is same for Master as for AppVPs, even though it may be
    2.54 -      // smaller due to higher predictability of the fixed jmp.
    2.55 -   saveLowTimeStampCountInto( masterPr->startMasterTSCLow );
    2.56 -   #endif
    2.57 -   //========================================================================
    2.58 +       //============================= MEASUREMENT STUFF =======================
    2.59 +       #ifdef MEAS__TIME_MASTER
    2.60 +          //Total Master time includes one coreloop time -- just assume the core
    2.61 +          // loop time is same for Master as for AppVPs, even though it may be
    2.62 +          // smaller due to higher predictability of the fixed jmp.
    2.63 +       saveLowTimeStampCountInto( masterVP->startMasterTSCLow );
    2.64 +       #endif
    2.65 +       //=======================================================================
    2.66  
    2.67     masterEnv        = (MasterEnv*)_VMSMasterEnv;
    2.68     
    2.69        //GCC may optimize so doesn't always re-define from frame-storage
    2.70 -   masterPr         = (VirtProcr*)volatileMasterPr;  //just to make sure after jmp
    2.71 -   thisCoresIdx     = masterPr->coreAnimatedBy;
    2.72 +   masterVP         = (VirtProcr*)volatilemasterVP;  //just to make sure after jmp
    2.73 +   thisCoresIdx     = masterVP->coreAnimatedBy;
    2.74     readyToAnimateQ  = masterEnv->readyToAnimateQs[thisCoresIdx];
    2.75     schedSlots       = masterEnv->allSchedSlots[thisCoresIdx];
    2.76  
    2.77 @@ -132,6 +122,7 @@
    2.78  
    2.79        //Poll each slot's Done flag
    2.80     numSlotsFilled = 0;
    2.81 +         Meas_startMasterLoop
    2.82     for( slotIdx = 0; slotIdx < NUM_SCHED_SLOTS; slotIdx++)
    2.83      {
    2.84        currSlot = schedSlots[ slotIdx ];
    2.85 @@ -142,22 +133,9 @@
    2.86           currSlot->needsProcrAssigned = TRUE;
    2.87  
    2.88              //process requests from slave to master
    2.89 -               //====================== MEASUREMENT STUFF ===================
    2.90 -               #ifdef MEAS__TIME_PLUGIN
    2.91 -               int32 startStamp1, endStamp1;
    2.92 -               saveLowTimeStampCountInto( startStamp1 );
    2.93 -               #endif
    2.94 -               //============================================================
    2.95 +               Meas_startReqHdlr
    2.96           (*requestHandler)( currSlot->procrAssignedToSlot, semanticEnv );
    2.97 -               //====================== MEASUREMENT STUFF ===================
    2.98 -               #ifdef MEAS__TIME_PLUGIN
    2.99 -               saveLowTimeStampCountInto( endStamp1 );
   2.100 -               addIntervalToHist( startStamp1, endStamp1,
   2.101 -                                        _VMSMasterEnv->reqHdlrLowTimeHist );
   2.102 -               addIntervalToHist( startStamp1, endStamp1,
   2.103 -                                        _VMSMasterEnv->reqHdlrHighTimeHist );
   2.104 -               #endif
   2.105 -               //============================================================
   2.106 +               Meas_endReqHdlr
   2.107         }
   2.108        if( currSlot->needsProcrAssigned )
   2.109         {    //give slot a new virt procr
   2.110 @@ -174,25 +152,36 @@
   2.111            }
   2.112         }
   2.113      }
   2.114 -
   2.115 +         Meas_endMasterLoop
   2.116     
   2.117     #ifdef USE_WORK_STEALING
   2.118        //If no slots filled, means no more work, look for work to steal.
   2.119     if( numSlotsFilled == 0 )
   2.120 -    { gateProtected_stealWorkInto( currSlot, readyToAnimateQ, masterPr );
   2.121 +    { gateProtected_stealWorkInto( currSlot, readyToAnimateQ, masterVP );
   2.122      }
   2.123     #endif
   2.124  
   2.125     
   2.126 -   #ifdef MEAS__TIME_MASTER
   2.127 -   saveLowTimeStampCountInto( masterPr->endMasterTSCLow );
   2.128 -   #endif
   2.129  
   2.130 -   masterSwitchToCoreLoop(animatingPr);
   2.131 +          //===================  Meas  =======================
   2.132 +          #ifdef MEAS__TIME_MASTER
   2.133 +          saveLowTimeStampCountInto( masterVP->endMasterTSCLow );
   2.134 +          #endif
   2.135 +          #ifdef MEAS__TIME_2011_SYS
   2.136 +         //Take meas here, to get cycles since entered Master
   2.137 +          saveTSCLowHigh(endMaster);
   2.138 +          numCycles = endMaster.longVal - _VMSMasterEnv->startMaster.longVal;
   2.139 +          
   2.140 +          if( numCycles < 200000 ) //sanity check against swap thd out)
   2.141 +           { masterEnv->totalMasterCycles += numCycles;
   2.142 +             masterEnv->numMasterAnimations++;
   2.143 +           }
   2.144 +          #endif
   2.145 +          //==================================================
   2.146 +   masterSwitchToCoreLoop(animatingPr); //"finishes" when switch back to Master
   2.147     flushRegisters();
   2.148     }//MasterLoop
   2.149  
   2.150 -
   2.151   }
   2.152  
   2.153  
   2.154 @@ -202,14 +191,14 @@
   2.155   */
   2.156  void inline
   2.157  stealWorkInto( SchedSlot *currSlot, VMSQueueStruc *readyToAnimateQ,
   2.158 -               VirtProcr *masterPr )
   2.159 +               VirtProcr *masterVP )
   2.160   { 
   2.161     VirtProcr   *stolenPr;
   2.162     int32        coreIdx, i;
   2.163     VMSQueueStruc *currQ;
   2.164  
   2.165     stolenPr = NULL;
   2.166 -   coreIdx = masterPr->coreAnimatedBy;
   2.167 +   coreIdx = masterVP->coreAnimatedBy;
   2.168     for( i = 0; i < NUM_CORES -1; i++ )
   2.169      {
   2.170        if( coreIdx >= NUM_CORES -1 )
   2.171 @@ -306,7 +295,7 @@
   2.172  void inline
   2.173  gateProtected_stealWorkInto( SchedSlot *currSlot,
   2.174                               VMSQueueStruc *myReadyToAnimateQ,
   2.175 -                             VirtProcr *masterPr )
   2.176 +                             VirtProcr *masterVP )
   2.177   {
   2.178     VirtProcr     *stolenPr;
   2.179     int32          coreIdx, i, haveAVictim, gotLock;
   2.180 @@ -319,7 +308,7 @@
   2.181  
   2.182        //see if any other cores have work available to steal
   2.183     haveAVictim = FALSE;
   2.184 -   coreIdx = masterPr->coreAnimatedBy;
   2.185 +   coreIdx = masterVP->coreAnimatedBy;
   2.186     for( i = 0; i < NUM_CORES -1; i++ )
   2.187      {
   2.188        if( coreIdx >= NUM_CORES -1 )
     3.1 --- a/ProcrContext.h	Thu Oct 06 16:25:59 2011 +0200
     3.2 +++ b/ProcrContext.h	Fri Dec 16 20:00:21 2011 +0100
     3.3 @@ -20,7 +20,7 @@
     3.4  
     3.5  void startVirtProcrFn();
     3.6  
     3.7 -void *asmTerminateCoreLoop(VirtProcr *currPr);
     3.8 +void *asmTerminateCoreLoop(VirtProcr *currVP);
     3.9  
    3.10  #define flushRegisters() \
    3.11          asm volatile ("":::"%rbx", "%r12", "%r13","%r14","%r15")
     4.1 --- a/VMS.c	Thu Oct 06 16:25:59 2011 +0200
     4.2 +++ b/VMS.c	Fri Dec 16 20:00:21 2011 +0100
     4.3 @@ -100,7 +100,8 @@
     4.4  
     4.5  
     4.6        //Make the master env, which holds everything else
     4.7 -   _VMSMasterEnv = malloc( sizeof(MasterEnv) );
     4.8 +   _VMSMasterEnv = malloc(   sizeof(MasterEnv) );
     4.9 +   memset( _VMSMasterEnv, 0, sizeof(MasterEnv) );
    4.10  
    4.11          //Very first thing put into the master env is the free-list, seeded
    4.12          // with a massive initial chunk of memory.
    4.13 @@ -681,6 +682,7 @@
    4.14     //forAllInDynArrayDo( _VMSMasterEnv->measHistsInfo, (DynArrayFnPtr)&printHist );
    4.15     //forAllInDynArrayDo( _VMSMasterEnv->measHistsInfo, (DynArrayFnPtr)&saveHistToFile);
    4.16     //forAllInDynArrayDo( _VMSMasterEnv->measHistsInfo, &freeHistExt );
    4.17 +/*
    4.18     #ifdef MEAS__TIME_PLUGIN
    4.19     printHist( _VMSMasterEnv->reqHdlrLowTimeHist );
    4.20     saveHistToFile( _VMSMasterEnv->reqHdlrLowTimeHist );
    4.21 @@ -712,6 +714,7 @@
    4.22        freeSchedSlots( allSchedSlots[ coreIdx ] );
    4.23      }
    4.24     #endif
    4.25 + */
    4.26     #ifdef MEAS__TIME_STAMP_SUSP
    4.27     printHist( _VMSMasterEnv->pluginTimeHist );
    4.28     for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    4.29 @@ -770,3 +773,15 @@
    4.30     exit(1);
    4.31   }
    4.32  
    4.33 +//=======================  Measurement  =======================
    4.34 +#ifdef MEAS__TIME_2011_SYS
    4.35 +uint64
    4.36 +VMS__give_num_plugin_cycles()
    4.37 + { return _VMSMasterEnv->totalPluginCycles;
    4.38 + }
    4.39 +
    4.40 +uint32
    4.41 +VMS__give_num_plugin_animations()
    4.42 + { return _VMSMasterEnv->numPluginAnimations;
    4.43 + }
    4.44 +#endif
     5.1 --- a/VMS.h	Thu Oct 06 16:25:59 2011 +0200
     5.2 +++ b/VMS.h	Fri Dec 16 20:00:21 2011 +0100
     5.3 @@ -20,96 +20,19 @@
     5.4  #include <pthread.h>
     5.5  #include <sys/time.h>
     5.6  
     5.7 -
     5.8 -//===============================  Debug  ===================================
     5.9 +//=============  #defines  ===========
    5.10  //
    5.11 -//When SEQUENTIAL is defined, VMS does sequential exe in the main thread
    5.12 -// It still does co-routines and all the mechanisms are the same, it just
    5.13 -// has only a single thread and animates VPs one at a time
    5.14 -//#define SEQUENTIAL
    5.15 -
    5.16 -//#define USE_WORK_STEALING
    5.17 -
    5.18 -//turns on the probe-instrumentation in the application -- when not
    5.19 -// defined, the calls to the probe functions turn into comments
    5.20 -#define STATS__ENABLE_PROBES
    5.21 -//#define TURN_ON_DEBUG_PROBES
    5.22 -
    5.23 -//These defines turn types of bug messages on and off
    5.24 -// be sure debug messages are un-commented (next block of defines)
    5.25 -#define dbgAppFlow   TRUE /* Top level flow of application code -- general*/
    5.26 -#define dbgProbes    FALSE /* for issues inside probes themselves*/
    5.27 -#define dbgB2BMaster FALSE /* in coreloop, back to back master VPs*/
    5.28 -#define dbgRqstHdlr  FALSE /* in request handler code*/
    5.29 -
    5.30 -//Comment or un- the substitute half to turn on/off types of debug message
    5.31 -#define DEBUG(  bool, msg)         \
    5.32 -//  if( bool){ printf(msg); fflush(stdin);}
    5.33 -#define DEBUG1( bool, msg, param)  \
    5.34 -//   if(bool){printf(msg, param); fflush(stdin);}
    5.35 -#define DEBUG2( bool, msg, p1, p2) \
    5.36 -//   if(bool) {printf(msg, p1, p2); fflush(stdin);}
    5.37 -
    5.38 -#define ERROR(msg) printf(msg);
    5.39 -#define ERROR1(msg, param) printf(msg, param); 
    5.40 -#define ERROR2(msg, p1, p2) printf(msg, p1, p2);
    5.41 -
    5.42 -//===========================  STATS =======================
    5.43 -
    5.44 -   //when MEAS__TIME_STAMP_SUSP is defined, causes code to be inserted and
    5.45 -   // compiled-in that saves the low part of the time stamp count just before
    5.46 -   // suspending a processor and just after resuming that processorsrc/VPThread_lib/VMS/VMS.h:322: warning: previous declaration of ‘VMS__create_procr’ was here.  It is
    5.47 -   // saved into a field added to VirtProcr.  Have to sanity-check for
    5.48 -   // rollover of low portion into high portion.
    5.49 -//#define MEAS__TIME_STAMP_SUSP
    5.50 -//#define MEAS__TIME_MASTER
    5.51 -#define MEAS__TIME_PLUGIN
    5.52 -#define MEAS__TIME_MALLOC
    5.53 -//#define MEAS__TIME_MASTER_LOCK
    5.54 -#define MEAS__NUM_TIMES_TO_RUN 100000
    5.55 -
    5.56 -   //For code that calculates normalization-offset between TSC counts of
    5.57 -   // different cores.
    5.58 -#define NUM_TSC_ROUND_TRIPS 10
    5.59 -
    5.60 -
    5.61 -//=========================  Hardware related Constants =====================
    5.62 -   //This value is the number of hardware threads in the shared memory
    5.63 -   // machine
    5.64 -//#define NUM_CORES        8
    5.65 -
    5.66 -   // tradeoff amortizing master fixed overhead vs imbalance potential
    5.67 -   // when work-stealing, can make bigger, at risk of losing cache affinity
    5.68 -#define NUM_SCHED_SLOTS  5
    5.69 -
    5.70 -#define MIN_WORK_UNIT_CYCLES 20000
    5.71 -
    5.72 -#define MASTERLOCK_RETRIES 10000
    5.73 -
    5.74 -   // stack size in virtual processors created
    5.75 -#define VIRT_PROCR_STACK_SIZE 0x8000 /* 32K */
    5.76 -
    5.77 -   // memory for VMS__malloc
    5.78 -#define MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE 0x10000000 /* 256M */
    5.79 -
    5.80 -#define CACHE_LINE 64
    5.81 -#define PAGE_SIZE 4096
    5.82 -
    5.83 -
    5.84 -//==============================
    5.85 -
    5.86 -#define SUCCESS 0
    5.87 -
    5.88 -#define writeVMSQ     writePrivQ
    5.89 -#define readVMSQ      readPrivQ
    5.90 -#define makeVMSQ      makeVMSPrivQ
    5.91 -#define numInVMSQ     numInPrivQ
    5.92 -#define VMSQueueStruc PrivQueueStruc
    5.93 +#include "VMS_defs.h"
    5.94  
    5.95  
    5.96  
    5.97  //===========================================================================
    5.98  typedef unsigned long long TSCount;
    5.99 +typedef union
   5.100 + { uint32 lowHigh[2];
   5.101 +   uint64 longVal;
   5.102 + }
   5.103 +TSCountLowHigh;
   5.104  
   5.105  typedef struct _SchedSlot     SchedSlot;
   5.106  typedef struct _VMSReqst      VMSReqst;
   5.107 @@ -194,14 +117,19 @@
   5.108     void       *dataRetFromReq;//values returned from plugin to VP go here
   5.109  
   5.110        //=========== MEASUREMENT STUFF ==========
   5.111 -   #ifdef MEAS__TIME_STAMP_SUSP
   5.112 -   unsigned int preSuspTSCLow;
   5.113 -   unsigned int postSuspTSCLow;
   5.114 -   #endif
   5.115 -   #ifdef MEAS__TIME_MASTER /* in VirtProcr because multiple masterVPs*/
   5.116 -   unsigned int startMasterTSCLow;USE_GNU
   5.117 -   unsigned int endMasterTSCLow;
   5.118 -   #endif
   5.119 +       #ifdef MEAS__TIME_STAMP_SUSP
   5.120 +       uint32  preSuspTSCLow;
   5.121 +       uint32  postSuspTSCLow;
   5.122 +       #endif
   5.123 +       #ifdef MEAS__TIME_MASTER /* in VirtProcr because multiple masterVPs*/
   5.124 +       uint32  startMasterTSCLow;USE_GNU
   5.125 +       uint32  endMasterTSCLow;
   5.126 +       #endif
   5.127 +       #ifdef MEAS__TIME_2011_SYS
   5.128 +       TSCountLowHigh  startSusp;
   5.129 +       uint64  totalSuspCycles;
   5.130 +       uint32  numGoodSusp;
   5.131 +       #endif
   5.132        //========================================
   5.133     
   5.134     float64      createPtInSecs;  //have space but don't use on some configs
   5.135 @@ -239,25 +167,36 @@
   5.136     int32            numProcrsCreated; //gives ordering to processor creation
   5.137  
   5.138        //=========== MEASUREMENT STUFF =============
   5.139 -   IntervalProbe  **intervalProbes;
   5.140 -   PrivDynArrayInfo    *dynIntervalProbesInfo;
   5.141 -   HashTable       *probeNameHashTbl;
   5.142 -   int32            masterCreateProbeID;
   5.143 -   float64          createPtInSecs;
   5.144 -   Histogram      **measHists;
   5.145 -   PrivDynArrayInfo *measHistsInfo;
   5.146 -   #ifdef MEAS__TIME_PLUGIN
   5.147 -   Histogram       *reqHdlrLowTimeHist;
   5.148 -   Histogram       *reqHdlrHighTimeHist;
   5.149 -   #endif
   5.150 -   #ifdef MEAS__TIME_MALLOC
   5.151 -   Histogram       *mallocTimeHist;
   5.152 -   Histogram       *freeTimeHist;
   5.153 -   #endif
   5.154 -   #ifdef MEAS__TIME_MASTER_LOCK
   5.155 -   Histogram       *masterLockLowTimeHist;
   5.156 -   Histogram       *masterLockHighTimeHist;
   5.157 -   #endif
   5.158 +       IntervalProbe  **intervalProbes;
   5.159 +       PrivDynArrayInfo    *dynIntervalProbesInfo;
   5.160 +       HashTable       *probeNameHashTbl;
   5.161 +       int32            masterCreateProbeID;
   5.162 +       float64          createPtInSecs;
   5.163 +       Histogram      **measHists;
   5.164 +       PrivDynArrayInfo *measHistsInfo;
   5.165 +       #ifdef MEAS__TIME_PLUGIN
   5.166 +       Histogram       *reqHdlrLowTimeHist;
   5.167 +       Histogram       *reqHdlrHighTimeHist;
   5.168 +       #endif
   5.169 +       #ifdef MEAS__TIME_MALLOC
   5.170 +       Histogram       *mallocTimeHist;
   5.171 +       Histogram       *freeTimeHist;
   5.172 +       #endif
   5.173 +       #ifdef MEAS__TIME_MASTER_LOCK
   5.174 +       Histogram       *masterLockLowTimeHist;
   5.175 +       Histogram       *masterLockHighTimeHist;
   5.176 +       #endif
   5.177 +       #ifdef MEAS__TIME_2011_SYS
   5.178 +       TSCountLowHigh   startMaster;
   5.179 +       uint64           totalMasterCycles;
   5.180 +       uint32           numMasterAnimations;
   5.181 +       TSCountLowHigh   startReqHdlr;
   5.182 +       uint64           totalPluginCycles;
   5.183 +       uint32           numPluginAnimations;
   5.184 +       uint64           cyclesTillStartMasterLoop;
   5.185 +       TSCountLowHigh   endMasterLoop;
   5.186 +       #endif
   5.187 +      //==========================================
   5.188   }
   5.189  MasterEnv;
   5.190  
   5.191 @@ -281,7 +220,7 @@
   5.192  
   5.193  void * coreLoop( void *paramsIn );  //standard PThreads fn prototype
   5.194  void * coreLoop_Seq( void *paramsIn );  //standard PThreads fn prototype
   5.195 -void masterLoop( void *initData, VirtProcr *masterPr );
   5.196 +void masterLoop( void *initData, VirtProcr *masterVP );
   5.197  
   5.198  
   5.199  typedef struct
   5.200 @@ -378,202 +317,17 @@
   5.201  VMS__handle_VMSSemReq( VMSReqst *req, VirtProcr *requestingPr, void *semEnv,
   5.202                         ResumePrFnPtr resumePrFnPtr );
   5.203  
   5.204 -//======================== STATS ======================
   5.205 +//======================== MEASUREMENT ======================
   5.206 +uint64
   5.207 +VMS__give_num_plugin_cycles();
   5.208 +uint32
   5.209 +VMS__give_num_plugin_animations();
   5.210  
   5.211 -//===== RDTSC wrapper ===== //Also runs with x86_64 code
   5.212  
   5.213 -#define saveTimeStampCountInto(low, high) \
   5.214 -   asm volatile("RDTSC;                   \
   5.215 -                 movl %%eax, %0;          \
   5.216 -                 movl %%edx, %1;"         \
   5.217 -   /* outputs */ : "=m" (low), "=m" (high)\
   5.218 -   /* inputs  */ :                        \
   5.219 -   /* clobber */ : "%eax", "%edx"         \
   5.220 -                );
   5.221 -
   5.222 -#define saveLowTimeStampCountInto(low)    \
   5.223 -   asm volatile("RDTSC;                   \
   5.224 -                 movl %%eax, %0;"         \
   5.225 -   /* outputs */ : "=m" (low)             \
   5.226 -   /* inputs  */ :                        \
   5.227 -   /* clobber */ : "%eax", "%edx"         \
   5.228 -                );
   5.229 -
   5.230 -//====================
   5.231 -#define makeAMeasHist( idx, name, numBins, startVal, binWidth ) \
   5.232 -   makeHighestDynArrayIndexBeAtLeast( _VMSMasterEnv->measHistsInfo, idx ); \
   5.233 -   _VMSMasterEnv->measHists[idx] =  \
   5.234 -                       makeFixedBinHist( numBins, startVal, binWidth, name );
   5.235 -
   5.236 -
   5.237 -#define MEAS__SUB_CREATE  /*turn on/off subtraction of create from plugin*/
   5.238 -
   5.239 -#ifdef VPTHREAD
   5.240 -
   5.241 -//VPThread
   5.242 -#define createHistIdx      1
   5.243 -#define mutexLockHistIdx   2
   5.244 -#define mutexUnlockHistIdx 3
   5.245 -#define condWaitHistIdx    4
   5.246 -#define condSignalHistIdx  5
   5.247 -
   5.248 -#define MakeTheMeasHists() \
   5.249 -   _VMSMasterEnv->measHistsInfo = \
   5.250 -              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
   5.251 -   makeAMeasHist( createHistIdx,      "create",        250, 0, 100 ) \
   5.252 -   makeAMeasHist( mutexLockHistIdx,   "mutex_lock",    50, 0, 100 ) \
   5.253 -   makeAMeasHist( mutexUnlockHistIdx, "mutex_unlock",  50, 0, 100 ) \
   5.254 -   makeAMeasHist( condWaitHistIdx,    "cond_wait",     50, 0, 100 ) \
   5.255 -   makeAMeasHist( condSignalHistIdx,  "cond_signal",   50, 0, 100 )
   5.256 -
   5.257 -#endif
   5.258 -
   5.259 -
   5.260 -#ifdef VCILK
   5.261 -
   5.262 -//VCilk
   5.263 -#define spawnHistIdx      1
   5.264 -#define syncHistIdx       2
   5.265 -
   5.266 -#define MakeTheMeasHists() \
   5.267 -   _VMSMasterEnv->measHistsInfo = \
   5.268 -              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
   5.269 -    makeAMeasHist( spawnHistIdx,      "Spawn",        50, 0, 200 ) \
   5.270 -    makeAMeasHist( syncHistIdx,       "Sync",         50, 0, 200 )
   5.271 -
   5.272 -
   5.273 -#endif
   5.274 -
   5.275 -#ifdef SSR
   5.276 -
   5.277 -//SSR
   5.278 -#define SendFromToHistIdx      1
   5.279 -#define SendOfTypeHistIdx      2
   5.280 -#define ReceiveFromToHistIdx   3
   5.281 -#define ReceiveOfTypeHistIdx   4
   5.282 -
   5.283 -#define MakeTheMeasHists() \
   5.284 -   _VMSMasterEnv->measHistsInfo = \
   5.285 -              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
   5.286 -    makeAMeasHist( SendFromToHistIdx,   "SendFromTo",    50, 0, 100 ) \
   5.287 -    makeAMeasHist( SendOfTypeHistIdx,   "SendOfType",    50, 0, 100 ) \
   5.288 -    makeAMeasHist( ReceiveFromToHistIdx,"ReceiveFromTo", 50, 0, 100 ) \
   5.289 -    makeAMeasHist( ReceiveOfTypeHistIdx,"ReceiveOfType", 50, 0, 100 )
   5.290 -
   5.291 -#endif
   5.292 -
   5.293 -//===========================================================================
   5.294 -//VPThread
   5.295 -
   5.296 -
   5.297 -#define Meas_startCreate \
   5.298 -    int32 startStamp, endStamp; \
   5.299 -    saveLowTimeStampCountInto( startStamp ); \
   5.300 -
   5.301 -#define Meas_endCreate \
   5.302 -    saveLowTimeStampCountInto( endStamp ); \
   5.303 -    addIntervalToHist( startStamp, endStamp, \
   5.304 -                                 _VMSMasterEnv->measHists[ createHistIdx ] );
   5.305 -
   5.306 -#define Meas_startMutexLock \
   5.307 -    int32 startStamp, endStamp; \
   5.308 -    saveLowTimeStampCountInto( startStamp ); \
   5.309 -
   5.310 -#define Meas_endMutexLock \
   5.311 -    saveLowTimeStampCountInto( endStamp ); \
   5.312 -    addIntervalToHist( startStamp, endStamp, \
   5.313 -                              _VMSMasterEnv->measHists[ mutexLockHistIdx ] );
   5.314 -
   5.315 -#define Meas_startMutexUnlock \
   5.316 -    int32 startStamp, endStamp; \
   5.317 -    saveLowTimeStampCountInto( startStamp ); \
   5.318 -
   5.319 -#define Meas_endMutexUnlock \
   5.320 -    saveLowTimeStampCountInto( endStamp ); \
   5.321 -    addIntervalToHist( startStamp, endStamp, \
   5.322 -                            _VMSMasterEnv->measHists[ mutexUnlockHistIdx ] );
   5.323 -
   5.324 -#define Meas_startCondWait \
   5.325 -    int32 startStamp, endStamp; \
   5.326 -    saveLowTimeStampCountInto( startStamp ); \
   5.327 -
   5.328 -#define Meas_endCondWait \
   5.329 -    saveLowTimeStampCountInto( endStamp ); \
   5.330 -    addIntervalToHist( startStamp, endStamp, \
   5.331 -                               _VMSMasterEnv->measHists[ condWaitHistIdx ] );
   5.332 -
   5.333 -#define Meas_startCondSignal \
   5.334 -    int32 startStamp, endStamp; \
   5.335 -    saveLowTimeStampCountInto( startStamp ); \
   5.336 -
   5.337 -#define Meas_endCondSignal \
   5.338 -    saveLowTimeStampCountInto( endStamp ); \
   5.339 -    addIntervalToHist( startStamp, endStamp, \
   5.340 -                             _VMSMasterEnv->measHists[ condSignalHistIdx ] );
   5.341 -
   5.342 -//===========================================================================
   5.343 -// VCilk
   5.344 -#define Meas_startSpawn \
   5.345 -    int32 startStamp, endStamp; \
   5.346 -    saveLowTimeStampCountInto( startStamp ); \
   5.347 -
   5.348 -#define Meas_endSpawn \
   5.349 -    saveLowTimeStampCountInto( endStamp ); \
   5.350 -    addIntervalToHist( startStamp, endStamp, \
   5.351 -                             _VMSMasterEnv->measHists[ spawnHistIdx ] );
   5.352 -
   5.353 -#define Meas_startSync \
   5.354 -    int32 startStamp, endStamp; \
   5.355 -    saveLowTimeStampCountInto( startStamp ); \
   5.356 -
   5.357 -#define Meas_endSync \
   5.358 -    saveLowTimeStampCountInto( endStamp ); \
   5.359 -    addIntervalToHist( startStamp, endStamp, \
   5.360 -                             _VMSMasterEnv->measHists[ syncHistIdx ] );
   5.361 -
   5.362 -//===========================================================================
   5.363 -// SSR
   5.364 -#define Meas_startSendFromTo \
   5.365 -    int32 startStamp, endStamp; \
   5.366 -    saveLowTimeStampCountInto( startStamp ); \
   5.367 -
   5.368 -#define Meas_endSendFromTo \
   5.369 -    saveLowTimeStampCountInto( endStamp ); \
   5.370 -    addIntervalToHist( startStamp, endStamp, \
   5.371 -                             _VMSMasterEnv->measHists[ SendFromToHistIdx ] );
   5.372 -
   5.373 -#define Meas_startSendOfType \
   5.374 -    int32 startStamp, endStamp; \
   5.375 -    saveLowTimeStampCountInto( startStamp ); \
   5.376 -
   5.377 -#define Meas_endSendOfType \
   5.378 -    saveLowTimeStampCountInto( endStamp ); \
   5.379 -    addIntervalToHist( startStamp, endStamp, \
   5.380 -                             _VMSMasterEnv->measHists[ SendOfTypeHistIdx ] );
   5.381 -
   5.382 -#define Meas_startReceiveFromTo \
   5.383 -    int32 startStamp, endStamp; \
   5.384 -    saveLowTimeStampCountInto( startStamp ); \
   5.385 -
   5.386 -#define Meas_endReceiveFromTo \
   5.387 -    saveLowTimeStampCountInto( endStamp ); \
   5.388 -    addIntervalToHist( startStamp, endStamp, \
   5.389 -                             _VMSMasterEnv->measHists[ ReceiveFromToHistIdx ] );
   5.390 -
   5.391 -#define Meas_startReceiveOfType \
   5.392 -    int32 startStamp, endStamp; \
   5.393 -    saveLowTimeStampCountInto( startStamp ); \
   5.394 -
   5.395 -#define Meas_endReceiveOfType \
   5.396 -    saveLowTimeStampCountInto( endStamp ); \
   5.397 -    addIntervalToHist( startStamp, endStamp, \
   5.398 -                             _VMSMasterEnv->measHists[ReceiveOfTypeHistIdx ] );
   5.399 -
   5.400 -//=====
   5.401  
   5.402  #include "ProcrContext.h"
   5.403  #include "probes.h"
   5.404  #include "vutilities.h"
   5.405 -
   5.406 +#include "../VMS_lang_specific_defines.h"
   5.407  #endif	/* _VMS_H */
   5.408  
     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/VMS_defs.h	Fri Dec 16 20:00:21 2011 +0100
     6.3 @@ -0,0 +1,291 @@
     6.4 +/*
     6.5 + *  Copyright 2009 OpenSourceStewardshipFoundation.org
     6.6 + *  Licensed under GNU General Public License version 2
     6.7 + *
     6.8 + * Author: seanhalle@yahoo.com
     6.9 + * 
    6.10 + */
    6.11 +
    6.12 +#ifndef _VMS_DEFS_H
    6.13 +#define	_VMS_DEFS_H
    6.14 +#define _GNU_SOURCE
    6.15 +
    6.16 +
    6.17 +//===============================  Debug  ===================================
    6.18 +//
    6.19 +//When SEQUENTIAL is defined, VMS does sequential exe in the main thread
    6.20 +// It still does co-routines and all the mechanisms are the same, it just
    6.21 +// has only a single thread and animates VPs one at a time
    6.22 +//#define SEQUENTIAL
    6.23 +
    6.24 +//#define USE_WORK_STEALING
    6.25 +
    6.26 +//turns on the probe-instrumentation in the application -- when not
    6.27 +// defined, the calls to the probe functions turn into comments
    6.28 +#define STATS__ENABLE_PROBES
    6.29 +//#define TURN_ON_DEBUG_PROBES
    6.30 +
    6.31 +//These defines turn types of bug messages on and off
    6.32 +// be sure debug messages are un-commented (next block of defines)
    6.33 +#define dbgAppFlow   TRUE /* Top level flow of application code -- general*/
    6.34 +#define dbgProbes    FALSE /* for issues inside probes themselves*/
    6.35 +#define dbgB2BMaster FALSE /* in coreloop, back to back master VPs*/
    6.36 +#define dbgRqstHdlr  FALSE /* in request handler code*/
    6.37 +
    6.38 +//Comment or un- the substitute half to turn on/off types of debug message
    6.39 +#define DEBUG(  bool, msg)         \
    6.40 +//  if( bool){ printf(msg); fflush(stdin);}
    6.41 +#define DEBUG1( bool, msg, param)  \
    6.42 +//   if(bool){printf(msg, param); fflush(stdin);}
    6.43 +#define DEBUG2( bool, msg, p1, p2) \
    6.44 +//   if(bool) {printf(msg, p1, p2); fflush(stdin);}
    6.45 +
    6.46 +#define ERROR(msg) printf(msg);
    6.47 +#define ERROR1(msg, param) printf(msg, param); 
    6.48 +#define ERROR2(msg, p1, p2) printf(msg, p1, p2);
    6.49 +
    6.50 +//===========================  MEASUREMENT =======================
    6.51 +
    6.52 +#define MEAS__TIME_2011_SYS
    6.53 +//define this if any MEAS__... below are
    6.54 +//#define MAKE_HISTS_FOR_MEASUREMENTS
    6.55 +   //when MEAS__TIME_STAMP_SUSP is defined, causes code to be inserted and
    6.56 +   // compiled-in that saves the low part of the time stamp count just before
    6.57 +   // suspending a processor and just after resuming that processor.  It is
    6.58 +   // saved into a field added to VirtProcr.  Have to sanity-check for
    6.59 +   // rollover of low portion into high portion.
    6.60 +//#define MEAS__TIME_STAMP_SUSP
    6.61 +//#define MEAS__TIME_MASTER
    6.62 +//#define MEAS__TIME_PLUGIN
    6.63 +//#define MEAS__TIME_MALLOC
    6.64 +//#define MEAS__TIME_MASTER_LOCK
    6.65 +
    6.66 +   //For code that calculates normalization-offset between TSC counts of
    6.67 +   // different cores.
    6.68 +//#define NUM_TSC_ROUND_TRIPS 10
    6.69 +
    6.70 +
    6.71 +//=========================  Hardware related Constants =====================
    6.72 +   //This value is the number of hardware threads in the shared memory
    6.73 +   // machine
    6.74 +//#define NUM_CORES        8
    6.75 +
    6.76 +   // tradeoff amortizing master fixed overhead vs imbalance potential
    6.77 +   // when work-stealing, can make bigger, at risk of losing cache affinity
    6.78 +#define NUM_SCHED_SLOTS  5
    6.79 +
    6.80 +#define MIN_WORK_UNIT_CYCLES 20000
    6.81 +
    6.82 +#define MASTERLOCK_RETRIES 10000
    6.83 +
    6.84 +   // stack size in virtual processors created
    6.85 +#define VIRT_PROCR_STACK_SIZE 0x8000 /* 32K */
    6.86 +
    6.87 +   // memory for VMS__malloc
    6.88 +#define MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE 0x10000000 /* 256M */
    6.89 +
    6.90 +#define CACHE_LINE 64
    6.91 +#define PAGE_SIZE 4096
    6.92 +
    6.93 +
    6.94 +//==============================
    6.95 +
    6.96 +#define SUCCESS 0
    6.97 +
    6.98 +#define writeVMSQ     writePrivQ
    6.99 +#define readVMSQ      readPrivQ
   6.100 +#define makeVMSQ      makeVMSPrivQ
   6.101 +#define numInVMSQ     numInPrivQ
   6.102 +#define VMSQueueStruc PrivQueueStruc
   6.103 +
   6.104 +
   6.105 +//============================  MEASUREMENT ==========================
   6.106 +//
   6.107 +//===== RDTSC wrapper ===== //Also runs with x86_64 code
   6.108 +#define saveTSCLowHigh(lowHighIn) \
   6.109 +   asm volatile("RDTSC;                   \
   6.110 +                 movl %%eax, %0;          \
   6.111 +                 movl %%edx, %1;"         \
   6.112 +   /* outputs */ : "=m" (lowHighIn.lowHigh[0]), "=m" (lowHighIn.lowHigh[1])\
   6.113 +   /* inputs  */ :                        \
   6.114 +   /* clobber */ : "%eax", "%edx"         \
   6.115 +                );
   6.116 +
   6.117 +#define saveTimeStampCountInto(low, high) \
   6.118 +   asm volatile("RDTSC;                   \
   6.119 +                 movl %%eax, %0;          \
   6.120 +                 movl %%edx, %1;"         \
   6.121 +   /* outputs */ : "=m" (low), "=m" (high)\
   6.122 +   /* inputs  */ :                        \
   6.123 +   /* clobber */ : "%eax", "%edx"         \
   6.124 +                );
   6.125 +
   6.126 +#define saveLowTimeStampCountInto(low)    \
   6.127 +   asm volatile("RDTSC;                   \
   6.128 +                 movl %%eax, %0;"         \
   6.129 +   /* outputs */ : "=m" (low)             \
   6.130 +   /* inputs  */ :                        \
   6.131 +   /* clobber */ : "%eax", "%edx"         \
   6.132 +                );
   6.133 +//=====
   6.134 +#ifdef MEAS__TIME_PLUGIN
   6.135 +
   6.136 +#define Meas_startReqHdlr \
   6.137 +        int32 startStamp1, endStamp1; \
   6.138 +        saveLowTimeStampCountInto( startStamp1 );
   6.139 +
   6.140 +#define Meas_endReqHdlr \
   6.141 +        saveLowTimeStampCountInto( endStamp1 ); \
   6.142 +        addIntervalToHist( startStamp1, endStamp1, \
   6.143 +                           _VMSMasterEnv->reqHdlrLowTimeHist ); \
   6.144 +        addIntervalToHist( startStamp1, endStamp1, \
   6.145 +                           _VMSMasterEnv->reqHdlrHighTimeHist );
   6.146 +               
   6.147 +#elif defined MEAS__TIME_2011_SYS
   6.148 +#define Meas_startMasterLoop \
   6.149 +        TSCountLowHigh startStamp1, endStamp1; \
   6.150 +        saveTSCLowHigh( endStamp1 ); \
   6.151 +        _VMSMasterEnv->cyclesTillStartMasterLoop = \
   6.152 +        endStamp1.longVal - masterVP->startSusp.longVal;
   6.153 +
   6.154 +#define Meas_startReqHdlr \
   6.155 +        saveTSCLowHigh( startStamp1 ); \
   6.156 +        _VMSMasterEnv->startReqHdlr.longVal = startStamp1.longVal;
   6.157 +
   6.158 +#define Meas_endReqHdlr 
   6.159 +
   6.160 +#define Meas_endMasterLoop \
   6.161 +        saveTSCLowHigh( startStamp1 ); \
   6.162 +        _VMSMasterEnv->endMasterLoop.longVal = startStamp1.longVal;
   6.163 +
   6.164 +#else
   6.165 +#define Meas_startMasterLoop 
   6.166 +#define Meas_startReqHdlr 
   6.167 +#define Meas_endReqHdlr 
   6.168 +#define Meas_endMasterLoop
   6.169 +#endif
   6.170 +
   6.171 +//==============  Histogram stuff -- used in measurements ==========
   6.172 +//
   6.173 +//
   6.174 +#ifdef MAKE_HISTS_FOR_MEASUREMENTS
   6.175 +#define makeAMeasHist( idx, name, numBins, startVal, binWidth ) \
   6.176 +   makeHighestDynArrayIndexBeAtLeast( _VMSMasterEnv->measHistsInfo, idx ); \
   6.177 +   _VMSMasterEnv->measHists[idx] =  \
   6.178 +                       makeFixedBinHist( numBins, startVal, binWidth, name );
   6.179 +#else
   6.180 +#define makeAMeasHist( idx, name, numBins, startVal, binWidth )
   6.181 +#endif
   6.182 +
   6.183 +
   6.184 +#define MEAS__SUB_CREATE  /*turn on/off subtraction of create from plugin*/
   6.185 +
   6.186 +
   6.187 +//====================  Language-specific Measurement Stuff ===================
   6.188 +//
   6.189 +//  MOVE THESE into ../VMS_lang_specific_defines.h
   6.190 +//
   6.191 +//
   6.192 +
   6.193 +
   6.194 +//TODO:  Move these to ../VMS_language_specific_defines for that language
   6.195 +//  and wrap them in #ifdef MEAS__...  the way have for Vthread
   6.196 +
   6.197 +#ifdef VCILK
   6.198 +
   6.199 +//VCilk
   6.200 +#define spawnHistIdx      1
   6.201 +#define syncHistIdx       2
   6.202 +
   6.203 +#define MakeTheMeasHists() \
   6.204 +   _VMSMasterEnv->measHistsInfo = \
   6.205 +              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
   6.206 +    makeAMeasHist( spawnHistIdx,      "Spawn",        50, 0, 200 ) \
   6.207 +    makeAMeasHist( syncHistIdx,       "Sync",         50, 0, 200 )
   6.208 +
   6.209 +
   6.210 +#endif
   6.211 +
   6.212 +#ifdef SSR
   6.213 +
   6.214 +//SSR
   6.215 +#define SendFromToHistIdx      1
   6.216 +#define SendOfTypeHistIdx      2
   6.217 +#define ReceiveFromToHistIdx   3
   6.218 +#define ReceiveOfTypeHistIdx   4
   6.219 +
   6.220 +#define MakeTheMeasHists() \
   6.221 +   _VMSMasterEnv->measHistsInfo = \
   6.222 +              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
   6.223 +    makeAMeasHist( SendFromToHistIdx,   "SendFromTo",    50, 0, 100 ) \
   6.224 +    makeAMeasHist( SendOfTypeHistIdx,   "SendOfType",    50, 0, 100 ) \
   6.225 +    makeAMeasHist( ReceiveFromToHistIdx,"ReceiveFromTo", 50, 0, 100 ) \
   6.226 +    makeAMeasHist( ReceiveOfTypeHistIdx,"ReceiveOfType", 50, 0, 100 )
   6.227 +
   6.228 +#endif
   6.229 +
   6.230 +//===========================================================================
   6.231 +//VPThread
   6.232 +
   6.233 +// already moved
   6.234 +
   6.235 +//===========================================================================
   6.236 +// VCilk
   6.237 +#define Meas_startSpawn \
   6.238 +    int32 startStamp, endStamp; \
   6.239 +    saveLowTimeStampCountInto( startStamp ); \
   6.240 +
   6.241 +#define Meas_endSpawn \
   6.242 +    saveLowTimeStampCountInto( endStamp ); \
   6.243 +    addIntervalToHist( startStamp, endStamp, \
   6.244 +                             _VMSMasterEnv->measHists[ spawnHistIdx ] );
   6.245 +
   6.246 +#define Meas_startSync \
   6.247 +    int32 startStamp, endStamp; \
   6.248 +    saveLowTimeStampCountInto( startStamp ); \
   6.249 +
   6.250 +#define Meas_endSync \
   6.251 +    saveLowTimeStampCountInto( endStamp ); \
   6.252 +    addIntervalToHist( startStamp, endStamp, \
   6.253 +                             _VMSMasterEnv->measHists[ syncHistIdx ] );
   6.254 +
   6.255 +//===========================================================================
   6.256 +// SSR
   6.257 +#define Meas_startSendFromTo \
   6.258 +    int32 startStamp, endStamp; \
   6.259 +    saveLowTimeStampCountInto( startStamp ); \
   6.260 +
   6.261 +#define Meas_endSendFromTo \
   6.262 +    saveLowTimeStampCountInto( endStamp ); \
   6.263 +    addIntervalToHist( startStamp, endStamp, \
   6.264 +                             _VMSMasterEnv->measHists[ SendFromToHistIdx ] );
   6.265 +
   6.266 +#define Meas_startSendOfType \
   6.267 +    int32 startStamp, endStamp; \
   6.268 +    saveLowTimeStampCountInto( startStamp ); \
   6.269 +
   6.270 +#define Meas_endSendOfType \
   6.271 +    saveLowTimeStampCountInto( endStamp ); \
   6.272 +    addIntervalToHist( startStamp, endStamp, \
   6.273 +                             _VMSMasterEnv->measHists[ SendOfTypeHistIdx ] );
   6.274 +
   6.275 +#define Meas_startReceiveFromTo \
   6.276 +    int32 startStamp, endStamp; \
   6.277 +    saveLowTimeStampCountInto( startStamp ); \
   6.278 +
   6.279 +#define Meas_endReceiveFromTo \
   6.280 +    saveLowTimeStampCountInto( endStamp ); \
   6.281 +    addIntervalToHist( startStamp, endStamp, \
   6.282 +                             _VMSMasterEnv->measHists[ ReceiveFromToHistIdx ] );
   6.283 +
   6.284 +#define Meas_startReceiveOfType \
   6.285 +    int32 startStamp, endStamp; \
   6.286 +    saveLowTimeStampCountInto( startStamp ); \
   6.287 +
   6.288 +#define Meas_endReceiveOfType \
   6.289 +    saveLowTimeStampCountInto( endStamp ); \
   6.290 +    addIntervalToHist( startStamp, endStamp, \
   6.291 +                             _VMSMasterEnv->measHists[ReceiveOfTypeHistIdx ] );
   6.292 +
   6.293 +#endif	/* _VMS_DEFS_H */
   6.294 +