1.1 --- a/CoreLoop.c	Thu Nov 04 17:57:39 2010 -0700
     1.2 +++ b/CoreLoop.c	Thu Nov 04 18:27:27 2010 -0700
     1.3 @@ -108,16 +108,19 @@
     1.4         {    //run own MasterVP -- when its done, unlocks MasterLock and
     1.5              // jumps back to coreLoops's startPt
     1.6           currPr = _VMSMasterEnv->masterVPs[thisCoresIdx];
     1.7 -         if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 10000 )
     1.8 -          { //printf("10000 back to back MasterVP\n");
     1.9 +         if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 100 )
    1.10 +          { //printf("1000 back to back MasterVP\n");
    1.11 +            //TODO: turn this into work-stealing from another core
    1.12 +            //only yield if no work to steal -- and count consecutive yields
    1.13 +            // if too many of those, then sleep for 10ms or whatever
    1.14              pthread_yield();
    1.15            }
    1.16           _VMSMasterEnv->numMasterInARow[thisCoresIdx] += 1;
    1.17           break;  //end while -- have a VP to animate now
    1.18         }
    1.19        
    1.20 -      tries++;
    1.21 -      if( tries > READYTOANIMATE_RETRIES ) { tries = 0; pthread_yield(); }
    1.22 +      tries++;      //if too many, means master on other core taking too long
    1.23 +      if( tries > MASTERLOCK_RETRIES ) { tries = 0; pthread_yield(); }
    1.24      }
    1.25     
    1.26  
    1.27 @@ -206,17 +209,15 @@
    1.28     readyToAnimateQ  = _VMSMasterEnv->readyToAnimateQs[thisCoresIdx];
    1.29     currPr = (VirtProcr *) readSRSWQ_NonBlocking( readyToAnimateQ );
    1.30     if( currPr == NULL )
    1.31 -    { if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] )
    1.32 -         printf("back to back MasterVP\n");
    1.33 -      _VMSMasterEnv->numMasterInARow[thisCoresIdx] = TRUE;
    1.34 +    { if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 1000 )
    1.35 +       { printf("too many back to back MasterVP\n"); exit(1); }
    1.36 +      _VMSMasterEnv->numMasterInARow[thisCoresIdx] += 1;
    1.37 +      
    1.38        currPr = _VMSMasterEnv->masterVPs[thisCoresIdx];
    1.39      }
    1.40     else
    1.41 -    _VMSMasterEnv->numMasterInARow[thisCoresIdx] = FALSE;
    1.42 +      _VMSMasterEnv->numMasterInARow[thisCoresIdx] = 0;
    1.43  
    1.44 -         PRINT2_DEBUG("core %d loop procr addr: %d\n",\
    1.45 -                       coreLoopThdParams->coreNum,    \
    1.46 -                       (int)currPr )
    1.47  
    1.48        //switch to virt procr's stack and frame ptr then jump to virt procr
    1.49     void *stackPtr, *framePtr, *jmpPt, *coreLoopFramePtrAddr, \

     2.1 --- a/MasterLoop.c	Thu Nov 04 17:57:39 2010 -0700
     2.2 +++ b/MasterLoop.c	Thu Nov 04 18:27:27 2010 -0700
     2.3 @@ -7,7 +7,6 @@
     2.4  
     2.5  
     2.6  #include <stdio.h>
     2.7 -#include <malloc.h>
     2.8  #include <stddef.h>
     2.9  
    2.10  #include "VMS.h"

     3.1 --- a/VMS.c	Thu Nov 04 17:57:39 2010 -0700
     3.2 +++ b/VMS.c	Thu Nov 04 18:27:27 2010 -0700
     3.3 @@ -33,6 +33,8 @@
     3.4  MallocProlog *
     3.5  create_free_list();
     3.6  
     3.7 +void
     3.8 +endOSThreadFn( void *initData, VirtProcr *animatingPr );
     3.9  
    3.10  pthread_mutex_t suspendLock = PTHREAD_MUTEX_INITIALIZER;
    3.11  pthread_cond_t  suspend_cond  = PTHREAD_COND_INITIALIZER;
    3.12 @@ -89,55 +91,55 @@
    3.13     int              coreIdx;
    3.14     VirtProcr      **masterVPs;
    3.15     SchedSlot     ***allSchedSlots; //ptr to array of ptrs
    3.16 -   
    3.17 +
    3.18 +
    3.19        //Make the master env, which holds everything else
    3.20     _VMSMasterEnv = malloc( sizeof(MasterEnv) );
    3.21 +
    3.22 +        //Very first thing put into the master env is the free-list, seeded
    3.23 +        // with a massive initial chunk of memory.
    3.24 +        //After this, all other mallocs are VMS__malloc.
    3.25 +   _VMSMasterEnv->freeListHead        = VMS_ext__create_free_list();
    3.26 +
    3.27 +   //===================== Only VMS__malloc after this ====================
    3.28     masterEnv     = _VMSMasterEnv;
    3.29 -      //Need to set start pt here 'cause used by seed procr, which is created
    3.30 -      // before the first core loop starts up. -- not sure how yet..
    3.31 -//   masterEnv->coreLoopStartPt = ;
    3.32 -//   masterEnv->coreLoopEndPt   = ;
    3.33     
    3.34        //Make a readyToAnimateQ for each core loop
    3.35 -   readyToAnimateQs = malloc( NUM_CORES * sizeof(SRSWQueueStruc *) );
    3.36 -   masterVPs        = malloc( NUM_CORES * sizeof(VirtProcr *) );
    3.37 +   readyToAnimateQs = VMS__malloc( NUM_CORES * sizeof(SRSWQueueStruc *) );
    3.38 +   masterVPs        = VMS__malloc( NUM_CORES * sizeof(VirtProcr *) );
    3.39  
    3.40        //One array for each core, 3 in array, core's masterVP scheds all
    3.41 -   allSchedSlots    = malloc( NUM_CORES * sizeof(SchedSlot *) );
    3.42 +   allSchedSlots    = VMS__malloc( NUM_CORES * sizeof(SchedSlot *) );
    3.43  
    3.44 +   _VMSMasterEnv->numProcrsCreated = 0;  //used by create procr
    3.45     for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    3.46 -    {    //running in main thread -- normal malloc inside makeSRSWQ
    3.47 +    {    
    3.48        readyToAnimateQs[ coreIdx ] = makeSRSWQ();
    3.49        
    3.50           //Q: should give masterVP core-specific info as its init data?
    3.51 -      masterVPs[ coreIdx ] = VMS_ext__create_procr( &masterLoop, masterEnv );
    3.52 +      masterVPs[ coreIdx ] = VMS__create_procr( &masterLoop, masterEnv );
    3.53        masterVPs[ coreIdx ]->coreAnimatedBy = coreIdx;
    3.54        allSchedSlots[ coreIdx ] = create_sched_slots(); //makes for one core
    3.55 -      _VMSMasterEnv->numMasterInARow[ coreIdx ] = FALSE;
    3.56 +      _VMSMasterEnv->numMasterInARow[ coreIdx ] = 0;
    3.57      }
    3.58     _VMSMasterEnv->readyToAnimateQs = readyToAnimateQs;
    3.59     _VMSMasterEnv->masterVPs        = masterVPs;
    3.60     _VMSMasterEnv->masterLock       = UNLOCKED;
    3.61     _VMSMasterEnv->allSchedSlots    = allSchedSlots;
    3.62 -   _VMSMasterEnv->numProcrsCreated = 0;
    3.63  
    3.64  
    3.65        //Aug 19, 2010:  no longer need to place initial masterVP into queue
    3.66        // because coreLoop now controls -- animates its masterVP when no work
    3.67  
    3.68 -   _VMSMasterEnv->freeListHead        = VMS__create_free_list();
    3.69 -   _VMSMasterEnv->amtOfOutstandingMem = 0; //none allocated yet
    3.70  
    3.71     //============================= MEASUREMENT STUFF ========================
    3.72     #ifdef STATS__TURN_ON_PROBES
    3.73 -      //creates intervalProbes array and sets pointer to it in masterEnv too
    3.74     _VMSMasterEnv->dynIntervalProbesInfo =
    3.75 -                  makeDynArrayOfSize( &(_VMSMasterEnv->intervalProbes), 20 );
    3.76 +              makePrivDynArrayOfSize( &(_VMSMasterEnv->intervalProbes), 200);
    3.77  
    3.78 -   _VMSMasterEnv->probeNameHashTbl = makeHashTable( 1000, NULL );
    3.79 -   _VMSMasterEnv->masterCreateProbeID =
    3.80 -       VMS_ext__record_time_point_into_new_probe( "masterCreateProbe" );
    3.81 -      //Also put creation time directly into master env, for fast retrieval
    3.82 +   _VMSMasterEnv->probeNameHashTbl = makeHashTable( 1000, &VMS__free );
    3.83 +   
    3.84 +      //put creation time directly into master env, for fast retrieval
    3.85     struct timeval timeStamp;
    3.86     gettimeofday( &(timeStamp), NULL);
    3.87     _VMSMasterEnv->createPtInSecs =
    3.88 @@ -152,11 +154,11 @@
    3.89   { SchedSlot  **schedSlots;
    3.90     int i;
    3.91  
    3.92 -   schedSlots  = malloc( NUM_SCHED_SLOTS * sizeof(SchedSlot *) );
    3.93 +   schedSlots  = VMS__malloc( NUM_SCHED_SLOTS * sizeof(SchedSlot *) );
    3.94  
    3.95     for( i = 0; i < NUM_SCHED_SLOTS; i++ )
    3.96      {
    3.97 -      schedSlots[i] = malloc( sizeof(SchedSlot) );
    3.98 +      schedSlots[i] = VMS__malloc( sizeof(SchedSlot) );
    3.99  
   3.100           //Set state to mean "handling requests done, slot needs filling"
   3.101        schedSlots[i]->workIsDone         = FALSE;
   3.102 @@ -171,9 +173,9 @@
   3.103   { int i;
   3.104     for( i = 0; i < NUM_SCHED_SLOTS; i++ )
   3.105      {
   3.106 -      free( schedSlots[i] );
   3.107 +      VMS__free( schedSlots[i] );
   3.108      }
   3.109 -   free( schedSlots );
   3.110 +   VMS__free( schedSlots );
   3.111   }
   3.112  
   3.113  
   3.114 @@ -191,7 +193,7 @@
   3.115  
   3.116        //Make the threads that animate the core loops
   3.117     for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
   3.118 -    { coreLoopThdParams[coreIdx]          = malloc( sizeof(ThdParams) );
   3.119 +    { coreLoopThdParams[coreIdx]          = VMS__malloc( sizeof(ThdParams) );
   3.120        coreLoopThdParams[coreIdx]->coreNum = coreIdx;
   3.121  
   3.122        retCode =
   3.123 @@ -263,11 +265,12 @@
   3.124   {
   3.125     char  *stackPtr;
   3.126  
   3.127 -   newPr->procrID     = _VMSMasterEnv->numProcrsCreated++;
   3.128 -   newPr->nextInstrPt = fnPtr;
   3.129 -   newPr->initialData = initialData;
   3.130 -   newPr->requests    = NULL;
   3.131 -   newPr->schedSlot   = NULL;
   3.132 +   newPr->startOfStack = stackLocs;
   3.133 +   newPr->procrID      = _VMSMasterEnv->numProcrsCreated++;
   3.134 +   newPr->nextInstrPt  = fnPtr;
   3.135 +   newPr->initialData  = initialData;
   3.136 +   newPr->requests     = NULL;
   3.137 +   newPr->schedSlot    = NULL;
   3.138  
   3.139        //fnPtr takes two params -- void *initData & void *animProcr
   3.140        //alloc stack locations, make stackPtr be the highest addr minus room
   3.141 @@ -285,7 +288,8 @@
   3.142     #ifdef STATS__TURN_ON_PROBES
   3.143     struct timeval timeStamp;
   3.144     gettimeofday( &(timeStamp), NULL);
   3.145 -   newPr->createPtInSecs = timeStamp.tv_sec +(timeStamp.tv_usec/1000000.0);
   3.146 +   newPr->createPtInSecs = timeStamp.tv_sec +(timeStamp.tv_usec/1000000.0) -
   3.147 +                                               _VMSMasterEnv->createPtInSecs;
   3.148     #endif
   3.149     //========================================================================
   3.150  
   3.151 @@ -301,7 +305,6 @@
   3.152     stackLocs  = VMS__malloc( VIRT_PROCR_STACK_SIZE );
   3.153     if( stackLocs == 0 )
   3.154      { perror("VMS__malloc stack"); exit(1); }
   3.155 -   newPr->startOfStack = stackLocs;
   3.156  
   3.157     return create_procr_helper( newPr, fnPtr, initialData, stackLocs );
   3.158   }
   3.159 @@ -319,7 +322,6 @@
   3.160     stackLocs  = malloc( VIRT_PROCR_STACK_SIZE );
   3.161     if( stackLocs == 0 )
   3.162      { perror("malloc stack"); exit(1); }
   3.163 -   newPr->startOfStack = stackLocs;
   3.164  
   3.165     return create_procr_helper( newPr, fnPtr, initialData, stackLocs );
   3.166   }
   3.167 @@ -477,18 +479,26 @@
   3.168  
   3.169  
   3.170  
   3.171 -/*This inserts the semantic-layer's request data into standard VMS carrier
   3.172 - * request data-struct is allocated on stack of this call & ptr to it sent
   3.173 - * to plugin
   3.174 +/*This call's name indicates that request is malloc'd -- so req handler
   3.175 + * has to free any extra requests tacked on before a send, using this.
   3.176 + *
   3.177 + * This inserts the semantic-layer's request data into standard VMS carrier
   3.178 + * request data-struct that is mallocd.  The sem request doesn't need to
   3.179 + * be malloc'd if this is called inside the same call chain before the
   3.180 + * send of the last request is called.
   3.181 + *
   3.182 + *The request handler has to call VMS__free_VMSReq for any of these
   3.183   */
   3.184  inline void
   3.185 -VMS__add_sem_request( void *semReqData, VirtProcr *callingPr )
   3.186 - { VMSReqst req;
   3.187 +VMS__add_sem_request_in_mallocd_VMSReqst( void *semReqData,
   3.188 +                                          VirtProcr *callingPr )
   3.189 + { VMSReqst *req;
   3.190  
   3.191 -   req.reqType         = semantic;
   3.192 -   req.semReqData      = semReqData;
   3.193 -   req.nextReqst       = callingPr->requests;
   3.194 -   callingPr->requests = &req;
   3.195 +   req = VMS__malloc( sizeof(VMSReqst) );
   3.196 +   req->reqType         = semantic;
   3.197 +   req->semReqData      = semReqData;
   3.198 +   req->nextReqst       = callingPr->requests;
   3.199 +   callingPr->requests = req;
   3.200   }
   3.201  
   3.202  /*This inserts the semantic-layer's request data into standard VMS carrier
   3.203 @@ -573,10 +583,12 @@
   3.204     memcpy( newProbe->nameStr, semReq->nameStr, nameLen );
   3.205     newProbe->hist    = NULL;
   3.206     newProbe->schedChoiceWasRecorded = FALSE;
   3.207 +
   3.208 +      //This runs in masterVP, so no race-condition worries
   3.209     newProbe->probeID =
   3.210               addToDynArray( newProbe, _VMSMasterEnv->dynIntervalProbesInfo );
   3.211  
   3.212 -   requestingPr->dataReturnedFromReq = newProbe;
   3.213 +   requestingPr->dataRetFromReq = newProbe;
   3.214  
   3.215     (*resumePrFnPtr)( requestingPr, semEnv );
   3.216   }
   3.217 @@ -619,15 +631,13 @@
   3.218   }
   3.219  
   3.220  
   3.221 -//TODO: re-architect so that have clean separation between request handler
   3.222 +//TODO: look at architecting cleanest separation between request handler
   3.223  // and master loop, for dissipate, create, shutdown, and other non-semantic
   3.224  // requests.  Issue is chain: one removes requests from AppVP, one dispatches
   3.225  // on type of request, and one handles each type..  but some types require
   3.226  // action from both request handler and master loop -- maybe just give the
   3.227  // request handler calls like:  VMS__handle_X_request_type
   3.228  
   3.229 -void
   3.230 -endOSThreadFn( void *initData, VirtProcr *animatingPr );
   3.231  
   3.232  /*This is called by the semantic layer's request handler when it decides its
   3.233   * time to shut down the VMS system.  Calling this causes the core loop OS
   3.234 @@ -641,10 +651,9 @@
   3.235   * masterVP any AppVPs that might still be allocated and sitting in the
   3.236   * semantic environment, or have been orphaned in the _VMSWorkQ.
   3.237   * 
   3.238 - *NOTE: the semantic plug-in is expected to use VMS__malloc_to to get all the
   3.239 + *NOTE: the semantic plug-in is expected to use VMS__malloc to get all the
   3.240   * locations it needs, and give ownership to masterVP.  Then, they will be
   3.241 - * automatically freed when the masterVP is dissipated.  (This happens after
   3.242 - * the core loop threads have all exited)
   3.243 + * automatically freed.
   3.244   *
   3.245   *In here,create one core-loop shut-down processor for each core loop and put
   3.246   * them all directly into the readyToAnimateQ.
   3.247 @@ -655,7 +664,7 @@
   3.248   * point is it sure that all results have completed.
   3.249   */
   3.250  void
   3.251 -VMS__handle_shutdown_reqst( void *dummy, VirtProcr *animatingPr )
   3.252 +VMS__shutdown()
   3.253   { int coreIdx;
   3.254     VirtProcr *shutDownPr;
   3.255  
   3.256 @@ -703,19 +712,19 @@
   3.257   }
   3.258  
   3.259  
   3.260 -/*This is called after the threads have shut down and control has returned
   3.261 - * to the semantic layer, in the entry point function in the main thread.
   3.262 - * It has to free anything allocated during VMS_init, and any other alloc'd
   3.263 - * locations that might be left over.
   3.264 +/*This is called from the startup & shutdown
   3.265   */
   3.266  void
   3.267 -VMS__cleanup_after_shutdown()
   3.268 +VMS__cleanup_at_end_of_shutdown()
   3.269   { 
   3.270     SRSWQueueStruc **readyToAnimateQs;
   3.271     int              coreIdx;
   3.272     VirtProcr      **masterVPs;
   3.273     SchedSlot     ***allSchedSlots; //ptr to array of ptrs
   3.274  
   3.275 +      //All the environment data has been allocated with VMS__malloc, so just
   3.276 +      // free its internal big-chunk and all inside it disappear.
   3.277 +/*
   3.278     readyToAnimateQs = _VMSMasterEnv->readyToAnimateQs;
   3.279     masterVPs        = _VMSMasterEnv->masterVPs;
   3.280     allSchedSlots    = _VMSMasterEnv->allSchedSlots;
   3.281 @@ -724,23 +733,40 @@
   3.282      {
   3.283        freeSRSWQ( readyToAnimateQs[ coreIdx ] );
   3.284           //master VPs were created external to VMS, so use external free
   3.285 -      VMS_ext__dissipate_procr( masterVPs[ coreIdx ] );
   3.286 +      VMS__dissipate_procr( masterVPs[ coreIdx ] );
   3.287        
   3.288        freeSchedSlots( allSchedSlots[ coreIdx ] );
   3.289      }
   3.290     
   3.291 -   free( _VMSMasterEnv->readyToAnimateQs );
   3.292 -   free( _VMSMasterEnv->masterVPs );
   3.293 -   free( _VMSMasterEnv->allSchedSlots );
   3.294 +   VMS__free( _VMSMasterEnv->readyToAnimateQs );
   3.295 +   VMS__free( _VMSMasterEnv->masterVPs );
   3.296 +   VMS__free( _VMSMasterEnv->allSchedSlots );
   3.297     
   3.298 -   VMS_ext__free_free_list( _VMSMasterEnv->freeListHead );
   3.299 -
   3.300     //============================= MEASUREMENT STUFF ========================
   3.301     #ifdef STATS__TURN_ON_PROBES
   3.302 -   freeDynArrayDeep( _VMSMasterEnv->dynIntervalProbesInfo, &free );
   3.303 +   freeDynArrayDeep( _VMSMasterEnv->dynIntervalProbesInfo, &VMS__free_probe);
   3.304     #endif
   3.305     //========================================================================
   3.306 -
   3.307 -   free( _VMSMasterEnv );
   3.308 +*/
   3.309 +      //These are the only two that use system free 
   3.310 +   VMS_ext__free_free_list( _VMSMasterEnv->freeListHead );
   3.311 +   free( (void *)_VMSMasterEnv );
   3.312   }
   3.313  
   3.314 +
   3.315 +//================================
   3.316 +
   3.317 +
   3.318 +/*Later, improve this -- for now, just exits the application after printing
   3.319 + * the error message.
   3.320 + */
   3.321 +void
   3.322 +VMS__throw_exception( char *msgStr, VirtProcr *reqstPr, VMSExcp *excpData )
   3.323 + {
   3.324 +   printf(msgStr);
   3.325 +   fflush(stdin);
   3.326 +   exit(1);
   3.327 + }
   3.328 +
   3.329 +
   3.330 +

     4.1 --- a/VMS.h	Thu Nov 04 17:57:39 2010 -0700
     4.2 +++ b/VMS.h	Thu Nov 04 18:27:27 2010 -0700
     4.3 @@ -22,6 +22,10 @@
     4.4  
     4.5  
     4.6  //===============================  Debug  ===================================
     4.7 +   //These defines turn types of bug messages on and off
     4.8 +#define dbgProbes FALSE
     4.9 +#define dbgAppFlow FALSE
    4.10 +
    4.11     //When SEQUENTIAL is defined, VMS does sequential exe in the main thread
    4.12     // It still does co-routines and all the mechanisms are the same, it just
    4.13     // has only a single thread and animates VPs one at a time
    4.14 @@ -32,7 +36,8 @@
    4.15  #define STATS__ENABLE_PROBES
    4.16  
    4.17  
    4.18 -#define PRINT_DEBUG(msg)// printf(msg); fflush(stdin);
    4.19 +#define DEBUG(msg)// printf(msg); fflush(stdin);
    4.20 +#define DEBUG_MSG( bool, msg) //if( bool){ printf(msg); fflush(stdin);}
    4.21  #define PRINT1_DEBUG(msg, param) //printf(msg, param); fflush(stdin);
    4.22  #define PRINT2_DEBUG(msg, p1, p2) //printf(msg, p1, p2); fflush(stdin);
    4.23  
    4.24 @@ -40,7 +45,6 @@
    4.25  #define PRINT1_ERROR(msg, param) printf(msg, param); fflush(stdin);
    4.26  #define PRINT2_ERROR(msg, p1, p2) printf(msg, p1, p2); fflush(stdin);
    4.27  
    4.28 -
    4.29  //===========================  STATS =======================
    4.30  
    4.31     //when MEAS__TIME_STAMP_SUSP is defined, causes code to be inserted and
    4.32 @@ -65,13 +69,13 @@
    4.33  
    4.34  #define MIN_WORK_UNIT_CYCLES 20000
    4.35  
    4.36 -#define READYTOANIMATE_RETRIES 10000
    4.37 +#define MASTERLOCK_RETRIES 10000
    4.38  
    4.39 -   // stack
    4.40 -#define VIRT_PROCR_STACK_SIZE 0x4000
    4.41 +   // stack size in virtual processors created
    4.42 +#define VIRT_PROCR_STACK_SIZE 0x4000 /* 16K */
    4.43  
    4.44 -   // memory for VMS__malloc -- 256M
    4.45 -#define MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE 0x10000000
    4.46 +   // memory for VMS__malloc
    4.47 +#define MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE 0x10000000 /* 256M */
    4.48  
    4.49  
    4.50  //==============================
    4.51 @@ -163,7 +167,7 @@
    4.52     VMSReqst   *requests;
    4.53  
    4.54     void       *semanticData; //this lives here for the life of VP
    4.55 -   void       *dataReturnedFromReq;//values returned from plugin to VP go here
    4.56 +   void       *dataRetFromReq;//values returned from plugin to VP go here
    4.57  
    4.58        //=========== MEASUREMENT STUFF ==========
    4.59     #ifdef MEAS__TIME_STAMP_SUSP
    4.60 @@ -205,14 +209,19 @@
    4.61  
    4.62        //=========== MEASUREMENT STUFF =============
    4.63     IntervalProbe  **intervalProbes;
    4.64 -   DynArrayInfo    *dynIntervalProbesInfo;
    4.65 +   PrivDynArrayInfo    *dynIntervalProbesInfo;
    4.66     HashTable       *probeNameHashTbl;
    4.67     int32            masterCreateProbeID;
    4.68     float64          createPtInSecs;
    4.69   }
    4.70  MasterEnv;
    4.71  
    4.72 +//=============================
    4.73 +typedef struct
    4.74 + {
    4.75  
    4.76 + }
    4.77 +VMSExcp;
    4.78  
    4.79  
    4.80  //=======================  OS Thread related  ===============================
    4.81 @@ -245,7 +254,8 @@
    4.82  
    4.83  //===========================  Function Prototypes  =========================
    4.84  
    4.85 -//============== Setup and shutdown =============
    4.86 +
    4.87 +//========== Setup and shutdown ==========
    4.88  void
    4.89  VMS__init();
    4.90  
    4.91 @@ -261,16 +271,25 @@
    4.92  VirtProcr *
    4.93  VMS__create_procr( VirtProcrFnPtr fnPtr, void *initialData );
    4.94  
    4.95 +void
    4.96 +VMS__dissipate_procr( VirtProcr *procrToDissipate );
    4.97 +
    4.98     //Use this to create processor inside entry point & other places outside
    4.99     // the VMS system boundary (IE, not run in slave nor Master)
   4.100  VirtProcr *
   4.101  VMS_ext__create_procr( VirtProcrFnPtr fnPtr, void *initialData );
   4.102  
   4.103 -VirtProcr *
   4.104 -VMS__create_the_shutdown_procr();
   4.105 +void
   4.106 +VMS_ext__dissipate_procr( VirtProcr *procrToDissipate );
   4.107  
   4.108  void
   4.109 -VMS__cleanup_after_shutdown();
   4.110 +VMS__throw_exception( char *msgStr, VirtProcr *reqstPr, VMSExcp *excpData );
   4.111 +
   4.112 +void
   4.113 +VMS__shutdown();
   4.114 +
   4.115 +void
   4.116 +VMS__cleanup_at_end_of_shutdown();
   4.117  
   4.118  
   4.119  //==============  Request Related  ===============
   4.120 @@ -279,49 +298,29 @@
   4.121  VMS__suspend_procr( VirtProcr *callingPr );
   4.122  
   4.123  inline void
   4.124 -VMS__add_sem_request( void *semReqData, VirtProcr *callingPr );
   4.125 +VMS__add_sem_request_in_mallocd_VMSReqst( void *semReqData, VirtProcr *callingPr );
   4.126 +
   4.127 +inline void
   4.128 +VMS__send_sem_request( void *semReqData, VirtProcr *callingPr );
   4.129  
   4.130  void
   4.131  VMS__send_create_procr_req( void *semReqData, VirtProcr *reqstingPr );
   4.132  
   4.133 +void inline
   4.134 +VMS__send_dissipate_req( VirtProcr *prToDissipate );
   4.135 +
   4.136  inline void
   4.137  VMS__send_VMSSem_request( void *semReqData, VirtProcr *callingPr );
   4.138  
   4.139 -void
   4.140 -VMS__free_request( VMSReqst *req );
   4.141 -
   4.142 -void
   4.143 -VMS__remove_and_free_top_request( VirtProcr *reqstingPr );
   4.144 -
   4.145  VMSReqst *
   4.146  VMS__take_next_request_out_of( VirtProcr *procrWithReq );
   4.147  
   4.148  inline void *
   4.149  VMS__take_sem_reqst_from( VMSReqst *req );
   4.150 -//
   4.151 -//VMSReqst *
   4.152 -//VMS__take_top_request_from( VirtProcr *reqstingPr );
   4.153 -//
   4.154 -//inline int
   4.155 -//VMS__isSemanticReqst( VMSReqst *req );
   4.156 -//
   4.157 -//inline int
   4.158 -//VMS__isDissipateReqst( VMSReqst *req );
   4.159 -//
   4.160 -//inline int
   4.161 -//VMS__isCreateReqst( VMSReqst *req );
   4.162  
   4.163 -//==========================
   4.164 +//======================== STATS ======================
   4.165  
   4.166 -void inline
   4.167 -VMS__send_dissipate_req( VirtProcr *prToDissipate );
   4.168 -
   4.169 -void
   4.170 -VMS__dissipate_procr( VirtProcr *procrToDissipate );
   4.171 -
   4.172 -
   4.173 -
   4.174 -//===================== RDTSC wrapper ==================
   4.175 +//===== RDTSC wrapper =====
   4.176  
   4.177  #define saveTimeStampCountInto(low, high) \
   4.178     asm volatile("RDTSC;                   \
   4.179 @@ -339,8 +338,7 @@
   4.180     /* inputs  */ :                        \
   4.181     /* clobber */ : "%eax", "%edx"         \
   4.182                  );
   4.183 -
   4.184 -//======================== STATS ======================
   4.185 +//=====
   4.186  
   4.187  #include "probes.h"
   4.188  

     5.1 --- a/probes.c	Thu Nov 04 17:57:39 2010 -0700
     5.2 +++ b/probes.c	Thu Nov 04 18:27:27 2010 -0700
     5.3 @@ -31,17 +31,6 @@
     5.4  
     5.5  //====================  Probes =================
     5.6  #ifdef STATS__USE_TSC_PROBES
     5.7 -int32
     5.8 -VMS__create_single_interval_probe( char *nameStr )
     5.9 - { IntervalProbe *newProbe;
    5.10 -   int32 idx;
    5.11 -
    5.12 -   newProbe = malloc( sizeof(IntervalProbe) );
    5.13 -   newProbe->nameStr = nameStr;  //caller frees if not constant on stack
    5.14 -   newProbe->hist = NULL;
    5.15 -   idx = addToDynArray( newProbe, _VMSMasterEnv->dynIntervalProbesInfo );
    5.16 -   return idx;
    5.17 - }
    5.18  
    5.19  int32
    5.20  VMS__create_histogram_probe( int32 numBins, float32 startValue,
    5.21 @@ -131,7 +120,7 @@
    5.22  
    5.23     VMS__send_VMSSem_request( &reqData, animPr );
    5.24  
    5.25 -   return animPr->dataReturnedFromReq;
    5.26 +   return animPr->dataRetFromReq;
    5.27   }
    5.28  
    5.29  /*Use this version from outside VMS -- it uses external malloc, and modifies
    5.30 @@ -154,8 +143,19 @@
    5.31     return newProbe;
    5.32   }
    5.33  
    5.34 +
    5.35 +/*Only call from inside master or main startup/shutdown thread
    5.36 + */
    5.37 +void
    5.38 +VMS_impl__free_probe( IntervalProbe *probe )
    5.39 + { if( probe->hist != NULL )   freeDblHist( probe->hist );
    5.40 +   if( probe->nameStr != NULL) VMS__free( probe->nameStr );
    5.41 +   VMS__free( probe );
    5.42 + }
    5.43 +
    5.44 +
    5.45  int32
    5.46 -VMS_impl__record_time_point_into_new_probe( char *nameStr, VirtProcr *animPr )
    5.47 +VMS_impl__record_time_point_into_new_probe( char *nameStr, VirtProcr *animPr)
    5.48   { IntervalProbe *newProbe;
    5.49     struct timeval *startStamp;
    5.50     float64 startSecs;
    5.51 @@ -243,7 +243,7 @@
    5.52     probe->schedChoiceWasRecorded = TRUE;
    5.53     probe->coreNum = animatingPr->coreAnimatedBy;
    5.54     probe->procrID = animatingPr->procrID;
    5.55 -   probe->procrCreateSecs = 0;
    5.56 +   probe->procrCreateSecs = animatingPr->createPtInSecs;
    5.57   }
    5.58  
    5.59  /*Everything is local to the animating procr, so no need for request, do
    5.60 @@ -253,6 +253,7 @@
    5.61  VMS_impl__record_interval_start_in_probe( int32 probeID )
    5.62   { IntervalProbe *probe;
    5.63  
    5.64 +         DEBUG_MSG( dbgProbes, "record start of interval\n" )
    5.65     probe = _VMSMasterEnv->intervalProbes[ probeID ];
    5.66     gettimeofday( &(probe->startStamp), NULL );
    5.67   }
    5.68 @@ -265,8 +266,9 @@
    5.69  VMS_impl__record_interval_end_in_probe( int32 probeID )
    5.70   { IntervalProbe *probe;
    5.71     struct timeval *endStamp, *startStamp;
    5.72 -   double startSecs, endSecs;
    5.73 +   float64 startSecs, endSecs;
    5.74  
    5.75 +         DEBUG_MSG( dbgProbes, "record end of interval\n" )
    5.76        //possible seg-fault if array resized by diff core right after this
    5.77        // one gets probe..?  Something like that?  Might be safe.. don't care
    5.78     probe = _VMSMasterEnv->intervalProbes[ probeID ];

     6.1 --- a/probes.h	Thu Nov 04 17:57:39 2010 -0700
     6.2 +++ b/probes.h	Thu Nov 04 18:27:27 2010 -0700
     6.3 @@ -87,6 +87,10 @@
     6.4                                            binWidth, nameStr, animPr )       \
     6.5          VMS_impl__create_histogram_probe( numBins, startValue,              \
     6.6                                            binWidth, nameStr, animPr )
     6.7 +void
     6.8 +VMS_impl__free_probe( IntervalProbe *probe );
     6.9 +#define VMS__free_probe( probe ) \
    6.10 +        VMS_impl__free_probe( probe )
    6.11  
    6.12  void
    6.13  VMS_impl__index_probe_by_its_name( int32 probeID, VirtProcr *animPr );

     7.1 --- a/vmalloc.c	Thu Nov 04 17:57:39 2010 -0700
     7.2 +++ b/vmalloc.c	Thu Nov 04 18:27:27 2010 -0700
     7.3 @@ -8,6 +8,7 @@
     7.4   */
     7.5  
     7.6  #include <malloc.h>
     7.7 +#include <stdlib.h>
     7.8  
     7.9  #include "VMS.h"
    7.10  
    7.11 @@ -123,14 +124,19 @@
    7.12   { MallocProlog *elemToFree, *nextLowerElem, *nextHigherElem;
    7.13     int32         lowerExistsAndIsFree, higherExistsAndIsFree, sizeOfElem;
    7.14  
    7.15 -   if( ptrToFree < _VMSMasterEnv->freeListHead->nextLowerInMem ||
    7.16 -       ptrToFree > _VMSMasterEnv->freeListHead->nextHigherInMem )
    7.17 +   if( ptrToFree < (void*)_VMSMasterEnv->freeListHead->nextLowerInMem ||
    7.18 +       ptrToFree > (void*)_VMSMasterEnv->freeListHead->nextHigherInMem )
    7.19      {    //outside the range of data owned by VMS's malloc, so do nothing
    7.20        return;
    7.21      }
    7.22        //subtract size of prolog to get pointer to prolog, then cast
    7.23     elemToFree = (MallocProlog *)((char *)ptrToFree - sizeof(MallocProlog));
    7.24     sizeOfElem =(int32)((char*)elemToFree->nextHigherInMem-(char*)elemToFree);
    7.25 +
    7.26 +   if( elemToFree->prevChunkInFreeList != NULL )
    7.27 +    { printf( "error: freeing same element twice!" ); exit(1);
    7.28 +    }
    7.29 +
    7.30     _VMSMasterEnv->amtOfOutstandingMem -= sizeOfElem;
    7.31  
    7.32     nextLowerElem  = elemToFree->nextLowerInMem;
    7.33 @@ -209,10 +215,73 @@
    7.34   }
    7.35  
    7.36  
    7.37 +/*Allocates memory from the external system -- higher overhead
    7.38 + *
    7.39 + *Because of Linux's malloc throwing bizarre random faults when malloc is
    7.40 + * used inside a VMS virtual processor, have to pass this as a request and
    7.41 + * have the core loop do it when it gets around to it -- will look for these
    7.42 + * chores leftover from the previous animation of masterVP the next time it
    7.43 + * goes to animate the masterVP -- so it takes two separate masterVP
    7.44 + * animations, separated by work, to complete an external malloc or
    7.45 + * external free request.
    7.46 + *
    7.47 + *Thinking core loop accepts signals -- just looks if signal-location is
    7.48 + * empty or not --
    7.49 + */
    7.50 +void *
    7.51 +VMS__malloc_in_ext( int32 sizeRequested )
    7.52 + {
    7.53 + /*
    7.54 +      //This is running in the master, so no chance for multiple cores to be
    7.55 +      // competing for the core's flag.
    7.56 +   if(  *(_VMSMasterEnv->coreLoopSignalAddr[ 0 ]) != 0 )
    7.57 +    {    //something has already signalled to core loop, so save the signal
    7.58 +         // and look, next time master animated, to see if can send it.
    7.59 +         //Note, the addr to put a signal is in the coreloop's frame, so just
    7.60 +         // checks it each time through -- make it volatile to avoid GCC
    7.61 +         // optimizations -- it's a coreloop local var that only changes
    7.62 +         // after jumping away.  The signal includes the addr to send the
    7.63 +         //return to -- even if just empty return completion-signal
    7.64 +         //
    7.65 +         //save the signal in some queue that the master looks at each time
    7.66 +         // it starts up -- one loc says if empty for fast common case --
    7.67 +         //something like that -- want to hide this inside this call -- but
    7.68 +         // think this has to come as a request -- req handler gives procr
    7.69 +         // back to master loop, which gives it back to req handler at point
    7.70 +         // it sees that core loop has sent return signal.  Something like
    7.71 +         // that.
    7.72 +      saveTheSignal
    7.73 +
    7.74 +    }
    7.75 +  coreSigData->type = malloc;
    7.76 +  coreSigData->sizeToMalloc = sizeRequested;
    7.77 +  coreSigData->locToSignalCompletion = &figureOut;
    7.78 +   _VMSMasterEnv->coreLoopSignals[ 0 ] = coreSigData;
    7.79 +  */
    7.80 +      //just risk system-stack faults until get this figured out
    7.81 +   return malloc( sizeRequested );
    7.82 + }
    7.83 +
    7.84 +
    7.85 +/*Frees memory that was allocated in the external system -- higher overhead
    7.86 + *
    7.87 + *As noted in external malloc comment, this is clunky 'cause the free has
    7.88 + * to be called in the core loop.
    7.89 + */
    7.90 +void
    7.91 +VMS__free_in_ext( void *ptrToFree )
    7.92 + {
    7.93 +      //just risk system-stack faults until get this figured out
    7.94 +   free( ptrToFree );
    7.95 +
    7.96 +      //TODO: fix this -- so 
    7.97 + }
    7.98 +
    7.99 +
   7.100  /*Designed to be called from the main thread outside of VMS, during init
   7.101   */
   7.102  MallocProlog *
   7.103 -VMS__create_free_list()
   7.104 +VMS_ext__create_free_list()
   7.105   { MallocProlog *freeListHead, *firstChunk;
   7.106  
   7.107        //Note, this is running in the main thread -- all increases in malloc
   7.108 @@ -226,16 +295,18 @@
   7.109        //Use this addr to free the heap when cleanup
   7.110     freeListHead->nextLowerInMem      = firstChunk;
   7.111        //to identify top-of-heap elem, compare this addr to elem's next higher
   7.112 -   freeListHead->nextHigherInMem     = (char *)firstChunk +
   7.113 -                                          MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE;
   7.114 +   freeListHead->nextHigherInMem     = (void*)( (char*)firstChunk +
   7.115 +                                         MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE);
   7.116     freeListHead->nextChunkInFreeList = firstChunk;
   7.117  
   7.118     firstChunk->nextChunkInFreeList   = NULL;
   7.119     firstChunk->prevChunkInFreeList   = freeListHead;
   7.120        //next Higher has to be set to top of chunk, so can calc size in malloc
   7.121 -   firstChunk->nextHigherInMem       = (char *)firstChunk +
   7.122 -                                          MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE;
   7.123 +   firstChunk->nextHigherInMem       = (void*)( (char*)firstChunk +
   7.124 +                                         MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE);
   7.125     firstChunk->nextLowerInMem        = NULL; //identifies as bott of heap
   7.126 +   
   7.127 +   _VMSMasterEnv->amtOfOutstandingMem = 0; //none allocated yet
   7.128  
   7.129     return freeListHead;
   7.130   }

     8.1 --- a/vmalloc.h	Thu Nov 04 17:57:39 2010 -0700
     8.2 +++ b/vmalloc.h	Thu Nov 04 18:27:27 2010 -0700
     8.3 @@ -34,8 +34,19 @@
     8.4  void
     8.5  VMS__free( void *ptrToFree );
     8.6  
     8.7 +/*Allocates memory from the external system -- higher overhead
     8.8 + */
     8.9 +void *
    8.10 +VMS__malloc_in_ext( int32 sizeRequested );
    8.11 +
    8.12 +/*Frees memory that was allocated in the external system -- higher overhead
    8.13 + */
    8.14 +void
    8.15 +VMS__free_in_ext( void *ptrToFree );
    8.16 +
    8.17 +
    8.18  MallocProlog *
    8.19 -VMS__create_free_list();
    8.20 +VMS_ext__create_free_list();
    8.21  
    8.22  void
    8.23  VMS_ext__free_free_list( MallocProlog *freeListHead );