1.1 --- a/VMS.h	Thu Oct 06 16:25:59 2011 +0200
     1.2 +++ b/VMS.h	Fri Dec 16 20:00:21 2011 +0100
     1.3 @@ -20,96 +20,19 @@
     1.4  #include <pthread.h>
     1.5  #include <sys/time.h>
     1.6  
     1.7 -
     1.8 -//===============================  Debug  ===================================
     1.9 +//=============  #defines  ===========
    1.10  //
    1.11 -//When SEQUENTIAL is defined, VMS does sequential exe in the main thread
    1.12 -// It still does co-routines and all the mechanisms are the same, it just
    1.13 -// has only a single thread and animates VPs one at a time
    1.14 -//#define SEQUENTIAL
    1.15 -
    1.16 -//#define USE_WORK_STEALING
    1.17 -
    1.18 -//turns on the probe-instrumentation in the application -- when not
    1.19 -// defined, the calls to the probe functions turn into comments
    1.20 -#define STATS__ENABLE_PROBES
    1.21 -//#define TURN_ON_DEBUG_PROBES
    1.22 -
    1.23 -//These defines turn types of bug messages on and off
    1.24 -// be sure debug messages are un-commented (next block of defines)
    1.25 -#define dbgAppFlow   TRUE /* Top level flow of application code -- general*/
    1.26 -#define dbgProbes    FALSE /* for issues inside probes themselves*/
    1.27 -#define dbgB2BMaster FALSE /* in coreloop, back to back master VPs*/
    1.28 -#define dbgRqstHdlr  FALSE /* in request handler code*/
    1.29 -
    1.30 -//Comment or un- the substitute half to turn on/off types of debug message
    1.31 -#define DEBUG(  bool, msg)         \
    1.32 -//  if( bool){ printf(msg); fflush(stdin);}
    1.33 -#define DEBUG1( bool, msg, param)  \
    1.34 -//   if(bool){printf(msg, param); fflush(stdin);}
    1.35 -#define DEBUG2( bool, msg, p1, p2) \
    1.36 -//   if(bool) {printf(msg, p1, p2); fflush(stdin);}
    1.37 -
    1.38 -#define ERROR(msg) printf(msg);
    1.39 -#define ERROR1(msg, param) printf(msg, param); 
    1.40 -#define ERROR2(msg, p1, p2) printf(msg, p1, p2);
    1.41 -
    1.42 -//===========================  STATS =======================
    1.43 -
    1.44 -   //when MEAS__TIME_STAMP_SUSP is defined, causes code to be inserted and
    1.45 -   // compiled-in that saves the low part of the time stamp count just before
    1.46 -   // suspending a processor and just after resuming that processorsrc/VPThread_lib/VMS/VMS.h:322: warning: previous declaration of ‘VMS__create_procr’ was here.  It is
    1.47 -   // saved into a field added to VirtProcr.  Have to sanity-check for
    1.48 -   // rollover of low portion into high portion.
    1.49 -//#define MEAS__TIME_STAMP_SUSP
    1.50 -//#define MEAS__TIME_MASTER
    1.51 -#define MEAS__TIME_PLUGIN
    1.52 -#define MEAS__TIME_MALLOC
    1.53 -//#define MEAS__TIME_MASTER_LOCK
    1.54 -#define MEAS__NUM_TIMES_TO_RUN 100000
    1.55 -
    1.56 -   //For code that calculates normalization-offset between TSC counts of
    1.57 -   // different cores.
    1.58 -#define NUM_TSC_ROUND_TRIPS 10
    1.59 -
    1.60 -
    1.61 -//=========================  Hardware related Constants =====================
    1.62 -   //This value is the number of hardware threads in the shared memory
    1.63 -   // machine
    1.64 -//#define NUM_CORES        8
    1.65 -
    1.66 -   // tradeoff amortizing master fixed overhead vs imbalance potential
    1.67 -   // when work-stealing, can make bigger, at risk of losing cache affinity
    1.68 -#define NUM_SCHED_SLOTS  5
    1.69 -
    1.70 -#define MIN_WORK_UNIT_CYCLES 20000
    1.71 -
    1.72 -#define MASTERLOCK_RETRIES 10000
    1.73 -
    1.74 -   // stack size in virtual processors created
    1.75 -#define VIRT_PROCR_STACK_SIZE 0x8000 /* 32K */
    1.76 -
    1.77 -   // memory for VMS__malloc
    1.78 -#define MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE 0x10000000 /* 256M */
    1.79 -
    1.80 -#define CACHE_LINE 64
    1.81 -#define PAGE_SIZE 4096
    1.82 -
    1.83 -
    1.84 -//==============================
    1.85 -
    1.86 -#define SUCCESS 0
    1.87 -
    1.88 -#define writeVMSQ     writePrivQ
    1.89 -#define readVMSQ      readPrivQ
    1.90 -#define makeVMSQ      makeVMSPrivQ
    1.91 -#define numInVMSQ     numInPrivQ
    1.92 -#define VMSQueueStruc PrivQueueStruc
    1.93 +#include "VMS_defs.h"
    1.94  
    1.95  
    1.96  
    1.97  //===========================================================================
    1.98  typedef unsigned long long TSCount;
    1.99 +typedef union
   1.100 + { uint32 lowHigh[2];
   1.101 +   uint64 longVal;
   1.102 + }
   1.103 +TSCountLowHigh;
   1.104  
   1.105  typedef struct _SchedSlot     SchedSlot;
   1.106  typedef struct _VMSReqst      VMSReqst;
   1.107 @@ -194,14 +117,19 @@
   1.108     void       *dataRetFromReq;//values returned from plugin to VP go here
   1.109  
   1.110        //=========== MEASUREMENT STUFF ==========
   1.111 -   #ifdef MEAS__TIME_STAMP_SUSP
   1.112 -   unsigned int preSuspTSCLow;
   1.113 -   unsigned int postSuspTSCLow;
   1.114 -   #endif
   1.115 -   #ifdef MEAS__TIME_MASTER /* in VirtProcr because multiple masterVPs*/
   1.116 -   unsigned int startMasterTSCLow;USE_GNU
   1.117 -   unsigned int endMasterTSCLow;
   1.118 -   #endif
   1.119 +       #ifdef MEAS__TIME_STAMP_SUSP
   1.120 +       uint32  preSuspTSCLow;
   1.121 +       uint32  postSuspTSCLow;
   1.122 +       #endif
   1.123 +       #ifdef MEAS__TIME_MASTER /* in VirtProcr because multiple masterVPs*/
   1.124 +       uint32  startMasterTSCLow;USE_GNU
   1.125 +       uint32  endMasterTSCLow;
   1.126 +       #endif
   1.127 +       #ifdef MEAS__TIME_2011_SYS
   1.128 +       TSCountLowHigh  startSusp;
   1.129 +       uint64  totalSuspCycles;
   1.130 +       uint32  numGoodSusp;
   1.131 +       #endif
   1.132        //========================================
   1.133     
   1.134     float64      createPtInSecs;  //have space but don't use on some configs
   1.135 @@ -239,25 +167,36 @@
   1.136     int32            numProcrsCreated; //gives ordering to processor creation
   1.137  
   1.138        //=========== MEASUREMENT STUFF =============
   1.139 -   IntervalProbe  **intervalProbes;
   1.140 -   PrivDynArrayInfo    *dynIntervalProbesInfo;
   1.141 -   HashTable       *probeNameHashTbl;
   1.142 -   int32            masterCreateProbeID;
   1.143 -   float64          createPtInSecs;
   1.144 -   Histogram      **measHists;
   1.145 -   PrivDynArrayInfo *measHistsInfo;
   1.146 -   #ifdef MEAS__TIME_PLUGIN
   1.147 -   Histogram       *reqHdlrLowTimeHist;
   1.148 -   Histogram       *reqHdlrHighTimeHist;
   1.149 -   #endif
   1.150 -   #ifdef MEAS__TIME_MALLOC
   1.151 -   Histogram       *mallocTimeHist;
   1.152 -   Histogram       *freeTimeHist;
   1.153 -   #endif
   1.154 -   #ifdef MEAS__TIME_MASTER_LOCK
   1.155 -   Histogram       *masterLockLowTimeHist;
   1.156 -   Histogram       *masterLockHighTimeHist;
   1.157 -   #endif
   1.158 +       IntervalProbe  **intervalProbes;
   1.159 +       PrivDynArrayInfo    *dynIntervalProbesInfo;
   1.160 +       HashTable       *probeNameHashTbl;
   1.161 +       int32            masterCreateProbeID;
   1.162 +       float64          createPtInSecs;
   1.163 +       Histogram      **measHists;
   1.164 +       PrivDynArrayInfo *measHistsInfo;
   1.165 +       #ifdef MEAS__TIME_PLUGIN
   1.166 +       Histogram       *reqHdlrLowTimeHist;
   1.167 +       Histogram       *reqHdlrHighTimeHist;
   1.168 +       #endif
   1.169 +       #ifdef MEAS__TIME_MALLOC
   1.170 +       Histogram       *mallocTimeHist;
   1.171 +       Histogram       *freeTimeHist;
   1.172 +       #endif
   1.173 +       #ifdef MEAS__TIME_MASTER_LOCK
   1.174 +       Histogram       *masterLockLowTimeHist;
   1.175 +       Histogram       *masterLockHighTimeHist;
   1.176 +       #endif
   1.177 +       #ifdef MEAS__TIME_2011_SYS
   1.178 +       TSCountLowHigh   startMaster;
   1.179 +       uint64           totalMasterCycles;
   1.180 +       uint32           numMasterAnimations;
   1.181 +       TSCountLowHigh   startReqHdlr;
   1.182 +       uint64           totalPluginCycles;
   1.183 +       uint32           numPluginAnimations;
   1.184 +       uint64           cyclesTillStartMasterLoop;
   1.185 +       TSCountLowHigh   endMasterLoop;
   1.186 +       #endif
   1.187 +      //==========================================
   1.188   }
   1.189  MasterEnv;
   1.190  
   1.191 @@ -281,7 +220,7 @@
   1.192  
   1.193  void * coreLoop( void *paramsIn );  //standard PThreads fn prototype
   1.194  void * coreLoop_Seq( void *paramsIn );  //standard PThreads fn prototype
   1.195 -void masterLoop( void *initData, VirtProcr *masterPr );
   1.196 +void masterLoop( void *initData, VirtProcr *masterVP );
   1.197  
   1.198  
   1.199  typedef struct
   1.200 @@ -378,202 +317,17 @@
   1.201  VMS__handle_VMSSemReq( VMSReqst *req, VirtProcr *requestingPr, void *semEnv,
   1.202                         ResumePrFnPtr resumePrFnPtr );
   1.203  
   1.204 -//======================== STATS ======================
   1.205 +//======================== MEASUREMENT ======================
   1.206 +uint64
   1.207 +VMS__give_num_plugin_cycles();
   1.208 +uint32
   1.209 +VMS__give_num_plugin_animations();
   1.210  
   1.211 -//===== RDTSC wrapper ===== //Also runs with x86_64 code
   1.212  
   1.213 -#define saveTimeStampCountInto(low, high) \
   1.214 -   asm volatile("RDTSC;                   \
   1.215 -                 movl %%eax, %0;          \
   1.216 -                 movl %%edx, %1;"         \
   1.217 -   /* outputs */ : "=m" (low), "=m" (high)\
   1.218 -   /* inputs  */ :                        \
   1.219 -   /* clobber */ : "%eax", "%edx"         \
   1.220 -                );
   1.221 -
   1.222 -#define saveLowTimeStampCountInto(low)    \
   1.223 -   asm volatile("RDTSC;                   \
   1.224 -                 movl %%eax, %0;"         \
   1.225 -   /* outputs */ : "=m" (low)             \
   1.226 -   /* inputs  */ :                        \
   1.227 -   /* clobber */ : "%eax", "%edx"         \
   1.228 -                );
   1.229 -
   1.230 -//====================
   1.231 -#define makeAMeasHist( idx, name, numBins, startVal, binWidth ) \
   1.232 -   makeHighestDynArrayIndexBeAtLeast( _VMSMasterEnv->measHistsInfo, idx ); \
   1.233 -   _VMSMasterEnv->measHists[idx] =  \
   1.234 -                       makeFixedBinHist( numBins, startVal, binWidth, name );
   1.235 -
   1.236 -
   1.237 -#define MEAS__SUB_CREATE  /*turn on/off subtraction of create from plugin*/
   1.238 -
   1.239 -#ifdef VPTHREAD
   1.240 -
   1.241 -//VPThread
   1.242 -#define createHistIdx      1
   1.243 -#define mutexLockHistIdx   2
   1.244 -#define mutexUnlockHistIdx 3
   1.245 -#define condWaitHistIdx    4
   1.246 -#define condSignalHistIdx  5
   1.247 -
   1.248 -#define MakeTheMeasHists() \
   1.249 -   _VMSMasterEnv->measHistsInfo = \
   1.250 -              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
   1.251 -   makeAMeasHist( createHistIdx,      "create",        250, 0, 100 ) \
   1.252 -   makeAMeasHist( mutexLockHistIdx,   "mutex_lock",    50, 0, 100 ) \
   1.253 -   makeAMeasHist( mutexUnlockHistIdx, "mutex_unlock",  50, 0, 100 ) \
   1.254 -   makeAMeasHist( condWaitHistIdx,    "cond_wait",     50, 0, 100 ) \
   1.255 -   makeAMeasHist( condSignalHistIdx,  "cond_signal",   50, 0, 100 )
   1.256 -
   1.257 -#endif
   1.258 -
   1.259 -
   1.260 -#ifdef VCILK
   1.261 -
   1.262 -//VCilk
   1.263 -#define spawnHistIdx      1
   1.264 -#define syncHistIdx       2
   1.265 -
   1.266 -#define MakeTheMeasHists() \
   1.267 -   _VMSMasterEnv->measHistsInfo = \
   1.268 -              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
   1.269 -    makeAMeasHist( spawnHistIdx,      "Spawn",        50, 0, 200 ) \
   1.270 -    makeAMeasHist( syncHistIdx,       "Sync",         50, 0, 200 )
   1.271 -
   1.272 -
   1.273 -#endif
   1.274 -
   1.275 -#ifdef SSR
   1.276 -
   1.277 -//SSR
   1.278 -#define SendFromToHistIdx      1
   1.279 -#define SendOfTypeHistIdx      2
   1.280 -#define ReceiveFromToHistIdx   3
   1.281 -#define ReceiveOfTypeHistIdx   4
   1.282 -
   1.283 -#define MakeTheMeasHists() \
   1.284 -   _VMSMasterEnv->measHistsInfo = \
   1.285 -              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
   1.286 -    makeAMeasHist( SendFromToHistIdx,   "SendFromTo",    50, 0, 100 ) \
   1.287 -    makeAMeasHist( SendOfTypeHistIdx,   "SendOfType",    50, 0, 100 ) \
   1.288 -    makeAMeasHist( ReceiveFromToHistIdx,"ReceiveFromTo", 50, 0, 100 ) \
   1.289 -    makeAMeasHist( ReceiveOfTypeHistIdx,"ReceiveOfType", 50, 0, 100 )
   1.290 -
   1.291 -#endif
   1.292 -
   1.293 -//===========================================================================
   1.294 -//VPThread
   1.295 -
   1.296 -
   1.297 -#define Meas_startCreate \
   1.298 -    int32 startStamp, endStamp; \
   1.299 -    saveLowTimeStampCountInto( startStamp ); \
   1.300 -
   1.301 -#define Meas_endCreate \
   1.302 -    saveLowTimeStampCountInto( endStamp ); \
   1.303 -    addIntervalToHist( startStamp, endStamp, \
   1.304 -                                 _VMSMasterEnv->measHists[ createHistIdx ] );
   1.305 -
   1.306 -#define Meas_startMutexLock \
   1.307 -    int32 startStamp, endStamp; \
   1.308 -    saveLowTimeStampCountInto( startStamp ); \
   1.309 -
   1.310 -#define Meas_endMutexLock \
   1.311 -    saveLowTimeStampCountInto( endStamp ); \
   1.312 -    addIntervalToHist( startStamp, endStamp, \
   1.313 -                              _VMSMasterEnv->measHists[ mutexLockHistIdx ] );
   1.314 -
   1.315 -#define Meas_startMutexUnlock \
   1.316 -    int32 startStamp, endStamp; \
   1.317 -    saveLowTimeStampCountInto( startStamp ); \
   1.318 -
   1.319 -#define Meas_endMutexUnlock \
   1.320 -    saveLowTimeStampCountInto( endStamp ); \
   1.321 -    addIntervalToHist( startStamp, endStamp, \
   1.322 -                            _VMSMasterEnv->measHists[ mutexUnlockHistIdx ] );
   1.323 -
   1.324 -#define Meas_startCondWait \
   1.325 -    int32 startStamp, endStamp; \
   1.326 -    saveLowTimeStampCountInto( startStamp ); \
   1.327 -
   1.328 -#define Meas_endCondWait \
   1.329 -    saveLowTimeStampCountInto( endStamp ); \
   1.330 -    addIntervalToHist( startStamp, endStamp, \
   1.331 -                               _VMSMasterEnv->measHists[ condWaitHistIdx ] );
   1.332 -
   1.333 -#define Meas_startCondSignal \
   1.334 -    int32 startStamp, endStamp; \
   1.335 -    saveLowTimeStampCountInto( startStamp ); \
   1.336 -
   1.337 -#define Meas_endCondSignal \
   1.338 -    saveLowTimeStampCountInto( endStamp ); \
   1.339 -    addIntervalToHist( startStamp, endStamp, \
   1.340 -                             _VMSMasterEnv->measHists[ condSignalHistIdx ] );
   1.341 -
   1.342 -//===========================================================================
   1.343 -// VCilk
   1.344 -#define Meas_startSpawn \
   1.345 -    int32 startStamp, endStamp; \
   1.346 -    saveLowTimeStampCountInto( startStamp ); \
   1.347 -
   1.348 -#define Meas_endSpawn \
   1.349 -    saveLowTimeStampCountInto( endStamp ); \
   1.350 -    addIntervalToHist( startStamp, endStamp, \
   1.351 -                             _VMSMasterEnv->measHists[ spawnHistIdx ] );
   1.352 -
   1.353 -#define Meas_startSync \
   1.354 -    int32 startStamp, endStamp; \
   1.355 -    saveLowTimeStampCountInto( startStamp ); \
   1.356 -
   1.357 -#define Meas_endSync \
   1.358 -    saveLowTimeStampCountInto( endStamp ); \
   1.359 -    addIntervalToHist( startStamp, endStamp, \
   1.360 -                             _VMSMasterEnv->measHists[ syncHistIdx ] );
   1.361 -
   1.362 -//===========================================================================
   1.363 -// SSR
   1.364 -#define Meas_startSendFromTo \
   1.365 -    int32 startStamp, endStamp; \
   1.366 -    saveLowTimeStampCountInto( startStamp ); \
   1.367 -
   1.368 -#define Meas_endSendFromTo \
   1.369 -    saveLowTimeStampCountInto( endStamp ); \
   1.370 -    addIntervalToHist( startStamp, endStamp, \
   1.371 -                             _VMSMasterEnv->measHists[ SendFromToHistIdx ] );
   1.372 -
   1.373 -#define Meas_startSendOfType \
   1.374 -    int32 startStamp, endStamp; \
   1.375 -    saveLowTimeStampCountInto( startStamp ); \
   1.376 -
   1.377 -#define Meas_endSendOfType \
   1.378 -    saveLowTimeStampCountInto( endStamp ); \
   1.379 -    addIntervalToHist( startStamp, endStamp, \
   1.380 -                             _VMSMasterEnv->measHists[ SendOfTypeHistIdx ] );
   1.381 -
   1.382 -#define Meas_startReceiveFromTo \
   1.383 -    int32 startStamp, endStamp; \
   1.384 -    saveLowTimeStampCountInto( startStamp ); \
   1.385 -
   1.386 -#define Meas_endReceiveFromTo \
   1.387 -    saveLowTimeStampCountInto( endStamp ); \
   1.388 -    addIntervalToHist( startStamp, endStamp, \
   1.389 -                             _VMSMasterEnv->measHists[ ReceiveFromToHistIdx ] );
   1.390 -
   1.391 -#define Meas_startReceiveOfType \
   1.392 -    int32 startStamp, endStamp; \
   1.393 -    saveLowTimeStampCountInto( startStamp ); \
   1.394 -
   1.395 -#define Meas_endReceiveOfType \
   1.396 -    saveLowTimeStampCountInto( endStamp ); \
   1.397 -    addIntervalToHist( startStamp, endStamp, \
   1.398 -                             _VMSMasterEnv->measHists[ReceiveOfTypeHistIdx ] );
   1.399 -
   1.400 -//=====
   1.401  
   1.402  #include "ProcrContext.h"
   1.403  #include "probes.h"
   1.404  #include "vutilities.h"
   1.405 -
   1.406 +#include "../VMS_lang_specific_defines.h"
   1.407  #endif	/* _VMS_H */
   1.408