1.1 --- a/CoreLoop.c	Wed Mar 07 22:53:50 2012 -0800
     1.2 +++ b/CoreLoop.c	Fri Mar 09 10:46:36 2012 -0800
     1.3 @@ -37,6 +37,7 @@
     1.4     SchedSlot      *currSlot, **schedSlots;
     1.5     cpu_set_t       coreMask;  //has 1 in bit positions of allowed cores
     1.6     int32           errorCode;
     1.7 +   TSCountLowHigh  endSusp;
     1.8  
     1.9        //work-stealing struc on stack to prevent false-sharing in cache-line
    1.10     volatile GateStruc gate;
    1.11 @@ -128,6 +129,9 @@
    1.12     
    1.13     switchToSlv(currVP); //Slave suspend makes core "return" from this call
    1.14     flushRegisters();
    1.15 +   
    1.16 +          MEAS__endCoreLoop_for_Sys;
    1.17 +          
    1.18     }//CoreLoop      
    1.19   }
    1.20  

     2.1 --- a/Defines/VMS_defs__MEAS.h	Wed Mar 07 22:53:50 2012 -0800
     2.2 +++ b/Defines/VMS_defs__MEAS.h	Fri Mar 09 10:46:36 2012 -0800
     2.3 @@ -195,27 +195,6 @@
     2.4     #define MEAS__Print_Hists_for_Malloc_Meas 
     2.5  #endif
     2.6  
     2.7 -      
     2.8 -#ifdef MEAS__TURN_ON_SYSTEM_MEAS
     2.9 -   #define MEAS__Insert_System_Meas_Fields_into_Slave \
    2.10 -      TSCountLowHigh  startSusp; \
    2.11 -      uint64  totalSuspCycles; \
    2.12 -      uint32  numGoodSusp;
    2.13 -
    2.14 -   #define MEAS__Insert_System_Meas_Fields_into_MasterEnv \
    2.15 -       TSCountLowHigh   startMaster; \
    2.16 -       uint64           totalMasterCycles; \
    2.17 -       uint32           numMasterAnimations; \
    2.18 -       TSCountLowHigh   startReqHdlr; \
    2.19 -       uint64           totalPluginCycles; \
    2.20 -       uint32           numPluginAnimations; \
    2.21 -       uint64           cyclesTillStartMasterLoop; \
    2.22 -       TSCountLowHigh   endMasterLoop; 
    2.23 -
    2.24 -#else
    2.25 -   #define MEAS__Insert_System_Meas_Fields_into_Slave 
    2.26 -   #define MEAS__Insert_System_Meas_Fields_into_MasterEnv 
    2.27 -#endif
    2.28  
    2.29  
    2.30  #ifdef MEAS__TURN_ON_PLUGIN_MEAS 
    2.31 @@ -256,27 +235,60 @@
    2.32  
    2.33  #endif
    2.34  
    2.35 -               
    2.36 +      
    2.37  #ifdef MEAS__TURN_ON_SYSTEM_MEAS
    2.38 -   #define MEAS__Insert_Plugin_Meas_Fields_into_MasterEnv 
    2.39 +   #define MEAS__Insert_System_Meas_Fields_into_Slave \
    2.40 +      TSCountLowHigh  startSusp; \
    2.41 +      uint64  totalSuspCycles; \
    2.42 +      uint32  numGoodSusp;
    2.43  
    2.44 -   #define MEAS__startMasterLoop \
    2.45 +   #define MEAS__Insert_System_Meas_Fields_into_MasterEnv \
    2.46 +       TSCountLowHigh   startMaster; \
    2.47 +       uint64           totalMasterCycles; \
    2.48 +       uint32           numMasterAnimations; \
    2.49 +       TSCountLowHigh   startReqHdlr; \
    2.50 +       uint64           totalPluginCycles; \
    2.51 +       uint32           numPluginAnimations; \
    2.52 +       uint64           cyclesTillStartMasterLoop; \
    2.53 +       TSCountLowHigh   endMasterLoop;
    2.54 +
    2.55 +   #define MEAS__startMasterLoop_forSys \
    2.56        TSCountLowHigh startStamp1, endStamp1; \
    2.57        saveTSCLowHigh( endStamp1 ); \
    2.58        _VMSMasterEnv->cyclesTillStartMasterLoop = \
    2.59        endStamp1.longVal - masterVP->startSusp.longVal;
    2.60  
    2.61 -   #define MEAS__endMasterLoop \
    2.62 +   #define Meas_startReqHdlr_forSys \
    2.63 +        saveTSCLowHigh( startStamp1 ); \
    2.64 +        _VMSMasterEnv->startReqHdlr.longVal = startStamp1.longVal;
    2.65 + 
    2.66 +   #define MEAS__endMasterLoop_forSys \
    2.67        saveTSCLowHigh( startStamp1 ); \
    2.68        _VMSMasterEnv->endMasterLoop.longVal = startStamp1.longVal;
    2.69  
    2.70 +   /*A TSC is stored in VP first thing inside wrapper-lib
    2.71 +    * Now, measures cycles from there to here
    2.72 +    * Master and Plugin will add this value to other trace-seg measures
    2.73 +    */
    2.74 +   #define MEAS__capture_end_susp_in_CoreLoop_forSys\
    2.75 +          saveTSCLowHigh(endSusp); \
    2.76 +          numCycles = endSusp.longVal - currVP->startSusp.longVal; \
    2.77 +          /*sanity check (400K is about 20K iters)*/ \
    2.78 +          if( numCycles < 400000 ) \
    2.79 +           { currVP->totalSuspCycles += numCycles; \
    2.80 +             currVP->numGoodSusp++; \
    2.81 +           } \
    2.82 +             /*recorded every time, but only read if currVP == MasterVP*/ \ 
    2.83 +          _VMSMasterEnv->startMaster.longVal = endSusp.longVal;
    2.84 +
    2.85  #else
    2.86 -   #define MEAS__Insert_System_Meas_Fields_into_MasterEnv
    2.87 +   #define MEAS__Insert_System_Meas_Fields_into_Slave 
    2.88 +   #define MEAS__Insert_System_Meas_Fields_into_MasterEnv 
    2.89     #define MEAS__Make_Meas_Hists_for_System_Meas
    2.90     #define MEAS__startMasterLoop_forSys 
    2.91 +   #define MEAS__startReqHdlr_forSys
    2.92     #define MEAS__endMasterLoop_forSys
    2.93 -   #define MEAS__startReqHdlr_forSys
    2.94 -   #define MEAS__endReqHdlr_forSys
    2.95 +   #define MEAS__capture_end_susp_in_CoreLoop_forSys
    2.96     #define MEAS__Print_Hists_for_System_Meas 
    2.97  #endif
    2.98  

     3.1 --- a/VMS.h	Wed Mar 07 22:53:50 2012 -0800
     3.2 +++ b/VMS.h	Fri Mar 09 10:46:36 2012 -0800
     3.3 @@ -219,6 +219,8 @@
     3.4  
     3.5  volatile MasterEnv      *_VMSMasterEnv __align_to_cacheline__;
     3.6  
     3.7 +pthread_mutex_t suspendLock = PTHREAD_MUTEX_INITIALIZER;
     3.8 +pthread_cond_t  suspend_cond  = PTHREAD_COND_INITIALIZER;
     3.9  
    3.10  //=========================  Function Prototypes  ===========================
    3.11  

     4.1 --- a/VMS__startup_and_shutdown.c	Wed Mar 07 22:53:50 2012 -0800
     4.2 +++ b/VMS__startup_and_shutdown.c	Fri Mar 09 10:46:36 2012 -0800
     4.3 @@ -35,8 +35,6 @@
     4.4  void
     4.5  endOSThreadFn( void *initData, SlaveVP *animatingSlv );
     4.6  
     4.7 -pthread_mutex_t suspendLock = PTHREAD_MUTEX_INITIALIZER;
     4.8 -pthread_cond_t  suspend_cond  = PTHREAD_COND_INITIALIZER;
     4.9  
    4.10  //===========================================================================
    4.11  
    4.12 @@ -80,23 +78,127 @@
    4.13     #endif
    4.14   }
    4.15  
    4.16 -   VMS__start_VMS_running( );
    4.17 +/*This structure holds all the information VMS needs to manage a program.  VMS
    4.18 + * stores information about what percent of CPU time the program is getting, what
    4.19 + * language it uses, the request handlers to call for its slaves, and so on.
    4.20 + */
    4.21 +typedef struct
    4.22 + { void               *semEnv;
    4.23 +   RequestHdlrFnPtr    requestHandler;
    4.24 +   SlaveAssignerFnPtr  slaveAssigner;
    4.25 +   int32               numSlavesLive;
    4.26 +   void               *resultToReturn;
    4.27 +  
    4.28 +   TopLevelFnPtr   seedFnPtr;
    4.29 +   void           *dataForSeed;
    4.30 +   bool32          executionIsComplete;
    4.31 +   pthread_mutex_t doneLock;
    4.32 +   pthread_cond_t  doneCond;
    4.33 + }
    4.34 +VMSProcess;
    4.35 +
    4.36 +         
    4.37 +void
    4.38 +VMS__start_VMS_running()
    4.39 + {
    4.40 +   create_masterEnv();
    4.41     
    4.42 -   VMSProcessID matrixMultProcessID;
    4.43 +   #ifdef DEBUG__TURN_ON_SEQUENTIAL_MODE
    4.44 +      /*Nothing else to create for sequential mode*/
    4.45 +   #else
    4.46 +      create_the_coreLoop_OS_threads();
    4.47 +   #endif    
    4.48 + }
    4.49 +
    4.50 +/*A pointer to the startup-function for the language is given as the last
    4.51 + * argument to the call.  Use this to initialize a program in the language.
    4.52 + * This creates a data structure that encapsulates the bookkeeping info
    4.53 + * VMS uses to track and schedule a program run.
    4.54 + */
    4.55 +VMSProcess *
    4.56 +VMS__spawn_program_on_data_in_Lang( TopLevelFnPtr prog_seed_fn, void *data,
    4.57 +                                    LangInitFnPtr langInitFnPtr )
    4.58 + { VMSProcess *newProcess;
    4.59 +   newProcess = malloc( sizeof(VMSProcess) );
    4.60 +   newProcess->doneLock = PTHREAD_MUTEX_INITIALIZER;
    4.61 +   newProcess->doneCond = PTHREAD_COND_INITIALIZER;
    4.62 +   newProcess->executionIsComplete = FALSE;
    4.63 +   newProcess->numSlavesLive = 0;
    4.64     
    4.65 -   matrixMultProcessID =
    4.66 +   newProcess->dataForSeed = data;
    4.67 +   newProcess->seedFnPtr   = prog_seed_fn;
    4.68 +   
    4.69 +   /*The language's spawn-process function fills in the plugin function-ptrs in
    4.70 +    * the VMSProcess struct, gives the struct to VMS, which then makes and
    4.71 +    * queues the seed SlaveVP, which starts processors made from the code being
    4.72 +    * animated.
    4.73 +    */
    4.74 +   (*langInitFnPtr)( newProcess );  
    4.75 +   
    4.76 +   return newProcess;
    4.77 + }
    4.78 +
    4.79 +/*When all SlaveVPs owned by the program-run associated to the process have
    4.80 + * dissipated, then return from this call.  There is no language to cleanup,
    4.81 + * and VMS does not shutdown..  but the process bookkeeping structure,
    4.82 + * which is used by VMS to track and schedule the program, is freed.
    4.83 + *The VMSProcess structure is kept until this call collects the results from it,
    4.84 + * then freed.  If the process is not done yet when VMS gets this
    4.85 + * call, then this call waits..  the challenge here is that this call comes from
    4.86 + * a live OS thread that's outside VMS..  so, inside here, it waits on a 
    4.87 + * condition..  then it's a VMS thread that signals this to wake up..
    4.88 + *First checks whether the process is done, if yes, calls the clean-up fn then
    4.89 + * returns the result extracted from the VMSProcess struct.
    4.90 + *If process not done yet, then performs a wait (in a loop to be sure the
    4.91 + * wakeup is not spurious, which can happen).  VMS registers the wait, and upon
    4.92 + * the process ending (last SlaveVP owned by it dissipates), then VMS signals
    4.93 + * this to wakeup.  This then calls the cleanup fn and returns the result.
    4.94 + */
    4.95 +void *
    4.96 +VMS__give_results_when_done_for( VMSProcess *process )
    4.97 + { void *result;
    4.98 +   
    4.99 +   pthread_mutex_lock( process->doneLock );
   4.100 +   while( !(process->executionIsComplete) )
   4.101 +    {
   4.102 +      pthread_cond_wait( process->doneCond,
   4.103 +                         process->doneLock );
   4.104 +    }
   4.105 +   pthread_mutex_unlock( process->doneLock );
   4.106 +   
   4.107 +   result = process->resultToReturn;
   4.108 +   
   4.109 +   VMS_int__cleanup_process_after_done( process );
   4.110 +   free( process );  //was malloc'd above, so free it here
   4.111 +   
   4.112 +   return result;
   4.113 + }
   4.114 +
   4.115 +/*Turns off the VMS system, and frees all data associated with it.  Does this
   4.116 + * by creating shutdown SlaveVPs and inserting them into scheduling slots.
   4.117 + * Will probably have to wake up sleeping cores as part of this -- the fn that
   4.118 + * inserts the new SlaveVPs should handle the wakeup..
   4.119 + */
   4.120 +void
   4.121 +VMS__shutdown()
   4.122 + {
   4.123 +   for( cores )
   4.124 +    { slave = VMS_int__create_new_SlaveVP( shutdownFN, NULL );
   4.125 +      VMS_int__insert_slave_onto_core( SlaveVP *slave, coreNum );
   4.126 +    }
   4.127 + }
   4.128 +
   4.129 +/* VMS__start_VMS_running();
   4.130 +
   4.131 +   VMSProcess matrixMultProcess;
   4.132 +   
   4.133 +   matrixMultProcess =
   4.134      VMS__spawn_program_on_data_in_Lang( &prog_seed_fn, data, Vthread_lang );
   4.135     
   4.136 -   resMatrix = VMS__give_results_when_done_for( matrixMultProcessID );
   4.137 -   
   4.138 -//    Vthread__give_results_when_done_for( matrixMultProcessID );
   4.139 -
   4.140 -   //VMS__shutdown_lang( Vthread );
   4.141 -   
   4.142 -   //Vthread__shutdown_lang();
   4.143 +   resMatrix = VMS__give_results_when_done_for( matrixMultProcess );
   4.144     
   4.145     VMS__shutdown();
   4.146 -
   4.147 + */
   4.148  
   4.149  void
   4.150  create_masterEnv()