1.1 --- a/CoreLoop.c	Sat Oct 30 21:53:55 2010 -0700
     1.2 +++ b/CoreLoop.c	Mon Nov 01 21:21:32 2010 -0700
     1.3 @@ -108,16 +108,19 @@
     1.4         {    //run own MasterVP -- when its done, unlocks MasterLock and
     1.5              // jumps back to coreLoops's startPt
     1.6           currPr = _VMSMasterEnv->masterVPs[thisCoresIdx];
     1.7 -         if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 10000 )
     1.8 -          { //printf("10000 back to back MasterVP\n");
     1.9 +         if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 100 )
    1.10 +          { //printf("1000 back to back MasterVP\n");
    1.11 +            //TODO: turn this into work-stealing from another core
    1.12 +            //only yield if no work to steal -- and count consecutive yields
    1.13 +            // if too many of those, then sleep for 10ms or whatever
    1.14              pthread_yield();
    1.15            }
    1.16           _VMSMasterEnv->numMasterInARow[thisCoresIdx] += 1;
    1.17           break;  //end while -- have a VP to animate now
    1.18         }
    1.19        
    1.20 -      tries++;
    1.21 -      if( tries > READYTOANIMATE_RETRIES ) { tries = 0; pthread_yield(); }
    1.22 +      tries++;      //if too many, means master on other core taking too long
    1.23 +      if( tries > MASTERLOCK_RETRIES ) { tries = 0; pthread_yield(); }
    1.24      }
    1.25     
    1.26  
    1.27 @@ -158,7 +161,7 @@
    1.28     CoreLoopEndPt:
    1.29        //first free shutdown VP that jumped here -- it first restores the
    1.30        // coreloop's stack, so addr of currPr in stack frame is still correct
    1.31 -   VMS__handle_dissipate_reqst( currPr );
    1.32 +   VMS__dissipate_procr( currPr );
    1.33     pthread_exit( NULL );
    1.34   }
    1.35  
    1.36 @@ -206,17 +209,15 @@
    1.37     readyToAnimateQ  = _VMSMasterEnv->readyToAnimateQs[thisCoresIdx];
    1.38     currPr = (VirtProcr *) readSRSWQ_NonBlocking( readyToAnimateQ );
    1.39     if( currPr == NULL )
    1.40 -    { if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] )
    1.41 -         printf("back to back MasterVP\n");
    1.42 -      _VMSMasterEnv->numMasterInARow[thisCoresIdx] = TRUE;
    1.43 +    { if( _VMSMasterEnv->numMasterInARow[thisCoresIdx] > 1000 )
    1.44 +       { printf("too many back to back MasterVP\n"); exit(1); }
    1.45 +      _VMSMasterEnv->numMasterInARow[thisCoresIdx] += 1;
    1.46 +      
    1.47        currPr = _VMSMasterEnv->masterVPs[thisCoresIdx];
    1.48      }
    1.49     else
    1.50 -    _VMSMasterEnv->numMasterInARow[thisCoresIdx] = FALSE;
    1.51 +      _VMSMasterEnv->numMasterInARow[thisCoresIdx] = 0;
    1.52  
    1.53 -         PRINT2_DEBUG("core %d loop procr addr: %d\n",\
    1.54 -                       coreLoopThdParams->coreNum,    \
    1.55 -                       (int)currPr )
    1.56  
    1.57        //switch to virt procr's stack and frame ptr then jump to virt procr
    1.58     void *stackPtr, *framePtr, *jmpPt, *coreLoopFramePtrAddr, \
    1.59 @@ -259,6 +260,6 @@
    1.60        // all the threads to die will proceed, gather the result, and
    1.61        // return to the calling application.
    1.62  SeqCoreLoopEndPt:
    1.63 -   VMS__handle_dissipate_reqst( currPr ); //free shutdown pr, that jmpd here
    1.64 +   VMS__dissipate_procr( currPr ); //free shutdown pr, that jmpd here
    1.65     return;
    1.66   }

     2.1 --- a/MasterLoop.c	Sat Oct 30 21:53:55 2010 -0700
     2.2 +++ b/MasterLoop.c	Mon Nov 01 21:21:32 2010 -0700
     2.3 @@ -7,7 +7,6 @@
     2.4  
     2.5  
     2.6  #include <stdio.h>
     2.7 -#include <malloc.h>
     2.8  #include <stddef.h>
     2.9  
    2.10  #include "VMS.h"

     3.1 --- a/VMS.c	Sat Oct 30 21:53:55 2010 -0700
     3.2 +++ b/VMS.c	Mon Nov 01 21:21:32 2010 -0700
     3.3 @@ -33,6 +33,8 @@
     3.4  MallocProlog *
     3.5  create_free_list();
     3.6  
     3.7 +void
     3.8 +endOSThreadFn( void *initData, VirtProcr *animatingPr );
     3.9  
    3.10  pthread_mutex_t suspendLock = PTHREAD_MUTEX_INITIALIZER;
    3.11  pthread_cond_t  suspend_cond  = PTHREAD_COND_INITIALIZER;
    3.12 @@ -89,55 +91,55 @@
    3.13     int              coreIdx;
    3.14     VirtProcr      **masterVPs;
    3.15     SchedSlot     ***allSchedSlots; //ptr to array of ptrs
    3.16 -   
    3.17 +
    3.18 +
    3.19        //Make the master env, which holds everything else
    3.20     _VMSMasterEnv = malloc( sizeof(MasterEnv) );
    3.21 +
    3.22 +        //Very first thing put into the master env is the free-list, seeded
    3.23 +        // with a massive initial chunk of memory.
    3.24 +        //After this, all other mallocs are VMS__malloc.
    3.25 +   _VMSMasterEnv->freeListHead        = VMS_ext__create_free_list();
    3.26 +
    3.27 +   //===================== Only VMS__malloc after this ====================
    3.28     masterEnv     = _VMSMasterEnv;
    3.29 -      //Need to set start pt here 'cause used by seed procr, which is created
    3.30 -      // before the first core loop starts up. -- not sure how yet..
    3.31 -//   masterEnv->coreLoopStartPt = ;
    3.32 -//   masterEnv->coreLoopEndPt   = ;
    3.33     
    3.34        //Make a readyToAnimateQ for each core loop
    3.35 -   readyToAnimateQs = malloc( NUM_CORES * sizeof(SRSWQueueStruc *) );
    3.36 -   masterVPs        = malloc( NUM_CORES * sizeof(VirtProcr *) );
    3.37 +   readyToAnimateQs = VMS__malloc( NUM_CORES * sizeof(SRSWQueueStruc *) );
    3.38 +   masterVPs        = VMS__malloc( NUM_CORES * sizeof(VirtProcr *) );
    3.39  
    3.40        //One array for each core, 3 in array, core's masterVP scheds all
    3.41 -   allSchedSlots    = malloc( NUM_CORES * sizeof(SchedSlot *) );
    3.42 +   allSchedSlots    = VMS__malloc( NUM_CORES * sizeof(SchedSlot *) );
    3.43  
    3.44 +   _VMSMasterEnv->numProcrsCreated = 0;  //used by create procr
    3.45     for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    3.46 -    {    //running in main thread -- normal malloc inside makeSRSWQ
    3.47 +    {    
    3.48        readyToAnimateQs[ coreIdx ] = makeSRSWQ();
    3.49        
    3.50           //Q: should give masterVP core-specific info as its init data?
    3.51 -      masterVPs[ coreIdx ] = VMS_ext__create_procr( &masterLoop, masterEnv );
    3.52 +      masterVPs[ coreIdx ] = VMS__create_procr( &masterLoop, masterEnv );
    3.53        masterVPs[ coreIdx ]->coreAnimatedBy = coreIdx;
    3.54        allSchedSlots[ coreIdx ] = create_sched_slots(); //makes for one core
    3.55 -      _VMSMasterEnv->numMasterInARow[ coreIdx ] = FALSE;
    3.56 +      _VMSMasterEnv->numMasterInARow[ coreIdx ] = 0;
    3.57      }
    3.58     _VMSMasterEnv->readyToAnimateQs = readyToAnimateQs;
    3.59     _VMSMasterEnv->masterVPs        = masterVPs;
    3.60     _VMSMasterEnv->masterLock       = UNLOCKED;
    3.61     _VMSMasterEnv->allSchedSlots    = allSchedSlots;
    3.62 -   _VMSMasterEnv->numProcrsCreated = 0;
    3.63  
    3.64  
    3.65        //Aug 19, 2010:  no longer need to place initial masterVP into queue
    3.66        // because coreLoop now controls -- animates its masterVP when no work
    3.67  
    3.68 -   _VMSMasterEnv->freeListHead        = VMS__create_free_list();
    3.69 -   _VMSMasterEnv->amtOfOutstandingMem = 0; //none allocated yet
    3.70  
    3.71     //============================= MEASUREMENT STUFF ========================
    3.72     #ifdef STATS__TURN_ON_PROBES
    3.73 -      //creates intervalProbes array and sets pointer to it in masterEnv too
    3.74     _VMSMasterEnv->dynIntervalProbesInfo =
    3.75 -                  makeDynArrayOfSize( &(_VMSMasterEnv->intervalProbes), 20 );
    3.76 +              makePrivDynArrayOfSize( &(_VMSMasterEnv->intervalProbes), 200);
    3.77  
    3.78 -   _VMSMasterEnv->probeNameHashTbl = makeHashTable( 1000, NULL );
    3.79 -   _VMSMasterEnv->masterCreateProbeID =
    3.80 -       VMS_ext__record_time_point_into_new_probe( "masterCreateProbe" );
    3.81 -      //Also put creation time directly into master env, for fast retrieval
    3.82 +   _VMSMasterEnv->probeNameHashTbl = makeHashTable( 1000, &VMS__free );
    3.83 +   
    3.84 +      //put creation time directly into master env, for fast retrieval
    3.85     struct timeval timeStamp;
    3.86     gettimeofday( &(timeStamp), NULL);
    3.87     _VMSMasterEnv->createPtInSecs =
    3.88 @@ -152,11 +154,11 @@
    3.89   { SchedSlot  **schedSlots;
    3.90     int i;
    3.91  
    3.92 -   schedSlots  = malloc( NUM_SCHED_SLOTS * sizeof(SchedSlot *) );
    3.93 +   schedSlots  = VMS__malloc( NUM_SCHED_SLOTS * sizeof(SchedSlot *) );
    3.94  
    3.95     for( i = 0; i < NUM_SCHED_SLOTS; i++ )
    3.96      {
    3.97 -      schedSlots[i] = malloc( sizeof(SchedSlot) );
    3.98 +      schedSlots[i] = VMS__malloc( sizeof(SchedSlot) );
    3.99  
   3.100           //Set state to mean "handling requests done, slot needs filling"
   3.101        schedSlots[i]->workIsDone         = FALSE;
   3.102 @@ -171,9 +173,9 @@
   3.103   { int i;
   3.104     for( i = 0; i < NUM_SCHED_SLOTS; i++ )
   3.105      {
   3.106 -      free( schedSlots[i] );
   3.107 +      VMS__free( schedSlots[i] );
   3.108      }
   3.109 -   free( schedSlots );
   3.110 +   VMS__free( schedSlots );
   3.111   }
   3.112  
   3.113  
   3.114 @@ -191,7 +193,7 @@
   3.115  
   3.116        //Make the threads that animate the core loops
   3.117     for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
   3.118 -    { coreLoopThdParams[coreIdx]          = malloc( sizeof(ThdParams) );
   3.119 +    { coreLoopThdParams[coreIdx]          = VMS__malloc( sizeof(ThdParams) );
   3.120        coreLoopThdParams[coreIdx]->coreNum = coreIdx;
   3.121  
   3.122        retCode =
   3.123 @@ -263,11 +265,12 @@
   3.124   {
   3.125     char  *stackPtr;
   3.126  
   3.127 -   newPr->procrID     = _VMSMasterEnv->numProcrsCreated++;
   3.128 -   newPr->nextInstrPt = fnPtr;
   3.129 -   newPr->initialData = initialData;
   3.130 -   newPr->requests    = NULL;
   3.131 -   newPr->schedSlot   = NULL;
   3.132 +   newPr->startOfStack = stackLocs;
   3.133 +   newPr->procrID      = _VMSMasterEnv->numProcrsCreated++;
   3.134 +   newPr->nextInstrPt  = fnPtr;
   3.135 +   newPr->initialData  = initialData;
   3.136 +   newPr->requests     = NULL;
   3.137 +   newPr->schedSlot    = NULL;
   3.138  
   3.139        //fnPtr takes two params -- void *initData & void *animProcr
   3.140        //alloc stack locations, make stackPtr be the highest addr minus room
   3.141 @@ -301,7 +304,6 @@
   3.142     stackLocs  = VMS__malloc( VIRT_PROCR_STACK_SIZE );
   3.143     if( stackLocs == 0 )
   3.144      { perror("VMS__malloc stack"); exit(1); }
   3.145 -   newPr->startOfStack = stackLocs;
   3.146  
   3.147     return create_procr_helper( newPr, fnPtr, initialData, stackLocs );
   3.148   }
   3.149 @@ -319,7 +321,6 @@
   3.150     stackLocs  = malloc( VIRT_PROCR_STACK_SIZE );
   3.151     if( stackLocs == 0 )
   3.152      { perror("malloc stack"); exit(1); }
   3.153 -   newPr->startOfStack = stackLocs;
   3.154  
   3.155     return create_procr_helper( newPr, fnPtr, initialData, stackLocs );
   3.156   }
   3.157 @@ -445,7 +446,7 @@
   3.158   * pears -- making that suspend the last thing in the virt procr's trace.
   3.159   */
   3.160  void
   3.161 -VMS__dissipate_procr( VirtProcr *procrToDissipate )
   3.162 +VMS__send_dissipate_req( VirtProcr *procrToDissipate )
   3.163   { VMSReqst req;
   3.164  
   3.165     req.reqType                = dissipate;
   3.166 @@ -477,18 +478,26 @@
   3.167  
   3.168  
   3.169  
   3.170 -/*This inserts the semantic-layer's request data into standard VMS carrier
   3.171 - * request data-struct is allocated on stack of this call & ptr to it sent
   3.172 - * to plugin
   3.173 +/*This call's name indicates that request is malloc'd -- so req handler
   3.174 + * has to free any extra requests tacked on before a send, using this.
   3.175 + *
   3.176 + * This inserts the semantic-layer's request data into standard VMS carrier
   3.177 + * request data-struct that is mallocd.  The sem request doesn't need to
   3.178 + * be malloc'd if this is called inside the same call chain before the
   3.179 + * send of the last request is called.
   3.180 + *
   3.181 + *The request handler has to call VMS__free_VMSReq for any of these
   3.182   */
   3.183  inline void
   3.184 -VMS__add_sem_request( void *semReqData, VirtProcr *callingPr )
   3.185 - { VMSReqst req;
   3.186 +VMS__add_sem_request_in_mallocd_VMSReqst( void *semReqData,
   3.187 +                                          VirtProcr *callingPr )
   3.188 + { VMSReqst *req;
   3.189  
   3.190 -   req.reqType         = semantic;
   3.191 -   req.semReqData      = semReqData;
   3.192 -   req.nextReqst       = callingPr->requests;
   3.193 -   callingPr->requests = &req;
   3.194 +   req = VMS__malloc( sizeof(VMSReqst) );
   3.195 +   req->reqType         = semantic;
   3.196 +   req->semReqData      = semReqData;
   3.197 +   req->nextReqst       = callingPr->requests;
   3.198 +   callingPr->requests = req;
   3.199   }
   3.200  
   3.201  /*This inserts the semantic-layer's request data into standard VMS carrier
   3.202 @@ -573,10 +582,12 @@
   3.203     memcpy( newProbe->nameStr, semReq->nameStr, nameLen );
   3.204     newProbe->hist    = NULL;
   3.205     newProbe->schedChoiceWasRecorded = FALSE;
   3.206 +
   3.207 +      //This runs in masterVP, so no race-condition worries
   3.208     newProbe->probeID =
   3.209               addToDynArray( newProbe, _VMSMasterEnv->dynIntervalProbesInfo );
   3.210  
   3.211 -   requestingPr->dataReturnedFromReq = newProbe;
   3.212 +   requestingPr->dataRetFromReq = newProbe;
   3.213  
   3.214     (*resumePrFnPtr)( requestingPr, semEnv );
   3.215   }
   3.216 @@ -601,7 +612,7 @@
   3.217   * of dis-owning it.
   3.218   */
   3.219  void
   3.220 -VMS__handle_dissipate_reqst( VirtProcr *animatingPr )
   3.221 +VMS__dissipate_procr( VirtProcr *animatingPr )
   3.222   {
   3.223        //dis-own all locations owned by this processor, causing to be freed
   3.224        // any locations that it is (was) sole owner of
   3.225 @@ -619,15 +630,13 @@
   3.226   }
   3.227  
   3.228  
   3.229 -//TODO: re-architect so that have clean separation between request handler
   3.230 +//TODO: look at architecting cleanest separation between request handler
   3.231  // and master loop, for dissipate, create, shutdown, and other non-semantic
   3.232  // requests.  Issue is chain: one removes requests from AppVP, one dispatches
   3.233  // on type of request, and one handles each type..  but some types require
   3.234  // action from both request handler and master loop -- maybe just give the
   3.235  // request handler calls like:  VMS__handle_X_request_type
   3.236  
   3.237 -void
   3.238 -endOSThreadFn( void *initData, VirtProcr *animatingPr );
   3.239  
   3.240  /*This is called by the semantic layer's request handler when it decides its
   3.241   * time to shut down the VMS system.  Calling this causes the core loop OS
   3.242 @@ -641,10 +650,9 @@
   3.243   * masterVP any AppVPs that might still be allocated and sitting in the
   3.244   * semantic environment, or have been orphaned in the _VMSWorkQ.
   3.245   * 
   3.246 - *NOTE: the semantic plug-in is expected to use VMS__malloc_to to get all the
   3.247 + *NOTE: the semantic plug-in is expected to use VMS__malloc to get all the
   3.248   * locations it needs, and give ownership to masterVP.  Then, they will be
   3.249 - * automatically freed when the masterVP is dissipated.  (This happens after
   3.250 - * the core loop threads have all exited)
   3.251 + * automatically freed.
   3.252   *
   3.253   *In here,create one core-loop shut-down processor for each core loop and put
   3.254   * them all directly into the readyToAnimateQ.
   3.255 @@ -655,7 +663,7 @@
   3.256   * point is it sure that all results have completed.
   3.257   */
   3.258  void
   3.259 -VMS__handle_shutdown_reqst( void *dummy, VirtProcr *animatingPr )
   3.260 +VMS__shutdown()
   3.261   { int coreIdx;
   3.262     VirtProcr *shutDownPr;
   3.263  
   3.264 @@ -703,19 +711,19 @@
   3.265   }
   3.266  
   3.267  
   3.268 -/*This is called after the threads have shut down and control has returned
   3.269 - * to the semantic layer, in the entry point function in the main thread.
   3.270 - * It has to free anything allocated during VMS_init, and any other alloc'd
   3.271 - * locations that might be left over.
   3.272 +/*This is called from the startup & shutdown
   3.273   */
   3.274  void
   3.275 -VMS__cleanup_after_shutdown()
   3.276 +VMS__cleanup_at_end_of_shutdown()
   3.277   { 
   3.278     SRSWQueueStruc **readyToAnimateQs;
   3.279     int              coreIdx;
   3.280     VirtProcr      **masterVPs;
   3.281     SchedSlot     ***allSchedSlots; //ptr to array of ptrs
   3.282  
   3.283 +      //All the environment data has been allocated with VMS__malloc, so just
   3.284 +      // free its internal big-chunk and all inside it disappear.
   3.285 +/*
   3.286     readyToAnimateQs = _VMSMasterEnv->readyToAnimateQs;
   3.287     masterVPs        = _VMSMasterEnv->masterVPs;
   3.288     allSchedSlots    = _VMSMasterEnv->allSchedSlots;
   3.289 @@ -724,23 +732,23 @@
   3.290      {
   3.291        freeSRSWQ( readyToAnimateQs[ coreIdx ] );
   3.292           //master VPs were created external to VMS, so use external free
   3.293 -      VMS_ext__dissipate_procr( masterVPs[ coreIdx ] );
   3.294 +      VMS__dissipate_procr( masterVPs[ coreIdx ] );
   3.295        
   3.296        freeSchedSlots( allSchedSlots[ coreIdx ] );
   3.297      }
   3.298     
   3.299 -   free( _VMSMasterEnv->readyToAnimateQs );
   3.300 -   free( _VMSMasterEnv->masterVPs );
   3.301 -   free( _VMSMasterEnv->allSchedSlots );
   3.302 +   VMS__free( _VMSMasterEnv->readyToAnimateQs );
   3.303 +   VMS__free( _VMSMasterEnv->masterVPs );
   3.304 +   VMS__free( _VMSMasterEnv->allSchedSlots );
   3.305     
   3.306 -   VMS_ext__free_free_list( _VMSMasterEnv->freeListHead );
   3.307 -
   3.308     //============================= MEASUREMENT STUFF ========================
   3.309     #ifdef STATS__TURN_ON_PROBES
   3.310 -   freeDynArrayDeep( _VMSMasterEnv->dynIntervalProbesInfo, &free );
   3.311 +   freeDynArrayDeep( _VMSMasterEnv->dynIntervalProbesInfo, &VMS__free_probe);
   3.312     #endif
   3.313     //========================================================================
   3.314 -
   3.315 -   free( _VMSMasterEnv );
   3.316 +*/
   3.317 +      //These are the only two that use system free 
   3.318 +   VMS_ext__free_free_list( _VMSMasterEnv->freeListHead );
   3.319 +   free( (void *)_VMSMasterEnv );
   3.320   }
   3.321  

     4.1 --- a/VMS.h	Sat Oct 30 21:53:55 2010 -0700
     4.2 +++ b/VMS.h	Mon Nov 01 21:21:32 2010 -0700
     4.3 @@ -65,7 +65,7 @@
     4.4  
     4.5  #define MIN_WORK_UNIT_CYCLES 20000
     4.6  
     4.7 -#define READYTOANIMATE_RETRIES 10000
     4.8 +#define MASTERLOCK_RETRIES 10000
     4.9  
    4.10     // stack
    4.11  #define VIRT_PROCR_STACK_SIZE 0x4000
    4.12 @@ -163,7 +163,7 @@
    4.13     VMSReqst   *requests;
    4.14  
    4.15     void       *semanticData; //this lives here for the life of VP
    4.16 -   void       *dataReturnedFromReq;//values returned from plugin to VP go here
    4.17 +   void       *dataRetFromReq;//values returned from plugin to VP go here
    4.18  
    4.19        //=========== MEASUREMENT STUFF ==========
    4.20     #ifdef MEAS__TIME_STAMP_SUSP
    4.21 @@ -205,7 +205,7 @@
    4.22  
    4.23        //=========== MEASUREMENT STUFF =============
    4.24     IntervalProbe  **intervalProbes;
    4.25 -   DynArrayInfo    *dynIntervalProbesInfo;
    4.26 +   PrivDynArrayInfo    *dynIntervalProbesInfo;
    4.27     HashTable       *probeNameHashTbl;
    4.28     int32            masterCreateProbeID;
    4.29     float64          createPtInSecs;
    4.30 @@ -245,7 +245,8 @@
    4.31  
    4.32  //===========================  Function Prototypes  =========================
    4.33  
    4.34 -//============== Setup and shutdown =============
    4.35 +
    4.36 +//========== Setup and shutdown ==========
    4.37  void
    4.38  VMS__init();
    4.39  
    4.40 @@ -261,16 +262,22 @@
    4.41  VirtProcr *
    4.42  VMS__create_procr( VirtProcrFnPtr fnPtr, void *initialData );
    4.43  
    4.44 +void
    4.45 +VMS__dissipate_procr( VirtProcr *procrToDissipate );
    4.46 +
    4.47     //Use this to create processor inside entry point & other places outside
    4.48     // the VMS system boundary (IE, not run in slave nor Master)
    4.49  VirtProcr *
    4.50  VMS_ext__create_procr( VirtProcrFnPtr fnPtr, void *initialData );
    4.51  
    4.52 -VirtProcr *
    4.53 -VMS__create_the_shutdown_procr();
    4.54 +void
    4.55 +VMS_ext__dissipate_procr( VirtProcr *procrToDissipate );
    4.56  
    4.57  void
    4.58 -VMS__cleanup_after_shutdown();
    4.59 +VMS__shutdown();
    4.60 +
    4.61 +void
    4.62 +VMS__cleanup_at_end_of_shutdown();
    4.63  
    4.64  
    4.65  //==============  Request Related  ===============
    4.66 @@ -279,49 +286,31 @@
    4.67  VMS__suspend_procr( VirtProcr *callingPr );
    4.68  
    4.69  inline void
    4.70 -VMS__add_sem_request( void *semReqData, VirtProcr *callingPr );
    4.71 +VMS__add_sem_request_in_mallocd_VMSReqst( void *semReqData, VirtProcr *callingPr );
    4.72 +
    4.73 +inline void
    4.74 +VMS__send_sem_request( void *semReqData, VirtProcr *callingPr );
    4.75  
    4.76  void
    4.77  VMS__send_create_procr_req( void *semReqData, VirtProcr *reqstingPr );
    4.78  
    4.79 +void inline
    4.80 +VMS__send_dissipate_req( VirtProcr *prToDissipate );
    4.81 +
    4.82  inline void
    4.83  VMS__send_VMSSem_request( void *semReqData, VirtProcr *callingPr );
    4.84  
    4.85 -void
    4.86 -VMS__free_request( VMSReqst *req );
    4.87 -
    4.88 -void
    4.89 -VMS__remove_and_free_top_request( VirtProcr *reqstingPr );
    4.90 -
    4.91  VMSReqst *
    4.92  VMS__take_next_request_out_of( VirtProcr *procrWithReq );
    4.93  
    4.94  inline void *
    4.95  VMS__take_sem_reqst_from( VMSReqst *req );
    4.96 -//
    4.97 -//VMSReqst *
    4.98 -//VMS__take_top_request_from( VirtProcr *reqstingPr );
    4.99 -//
   4.100 -//inline int
   4.101 -//VMS__isSemanticReqst( VMSReqst *req );
   4.102 -//
   4.103 -//inline int
   4.104 -//VMS__isDissipateReqst( VMSReqst *req );
   4.105 -//
   4.106 -//inline int
   4.107 -//VMS__isCreateReqst( VMSReqst *req );
   4.108  
   4.109 -//==========================
   4.110  
   4.111 -void inline
   4.112 -VMS__dissipate_procr( VirtProcr *prToDissipate );
   4.113  
   4.114 -void
   4.115 -VMS__handle_dissipate_reqst( VirtProcr *procrToDissipate );
   4.116 +//======================== STATS ======================
   4.117  
   4.118 -
   4.119 -
   4.120 -//===================== RDTSC wrapper ==================
   4.121 +//===== RDTSC wrapper =====
   4.122  
   4.123  #define saveTimeStampCountInto(low, high) \
   4.124     asm volatile("RDTSC;                   \
   4.125 @@ -339,8 +328,7 @@
   4.126     /* inputs  */ :                        \
   4.127     /* clobber */ : "%eax", "%edx"         \
   4.128                  );
   4.129 -
   4.130 -//======================== STATS ======================
   4.131 +//=====
   4.132  
   4.133  #include "probes.h"
   4.134  

     5.1 --- a/probes.c	Sat Oct 30 21:53:55 2010 -0700
     5.2 +++ b/probes.c	Mon Nov 01 21:21:32 2010 -0700
     5.3 @@ -31,17 +31,6 @@
     5.4  
     5.5  //====================  Probes =================
     5.6  #ifdef STATS__USE_TSC_PROBES
     5.7 -int32
     5.8 -VMS__create_single_interval_probe( char *nameStr )
     5.9 - { IntervalProbe *newProbe;
    5.10 -   int32 idx;
    5.11 -
    5.12 -   newProbe = malloc( sizeof(IntervalProbe) );
    5.13 -   newProbe->nameStr = nameStr;  //caller frees if not constant on stack
    5.14 -   newProbe->hist = NULL;
    5.15 -   idx = addToDynArray( newProbe, _VMSMasterEnv->dynIntervalProbesInfo );
    5.16 -   return idx;
    5.17 - }
    5.18  
    5.19  int32
    5.20  VMS__create_histogram_probe( int32 numBins, float32 startValue,
    5.21 @@ -131,7 +120,7 @@
    5.22  
    5.23     VMS__send_VMSSem_request( &reqData, animPr );
    5.24  
    5.25 -   return animPr->dataReturnedFromReq;
    5.26 +   return animPr->dataRetFromReq;
    5.27   }
    5.28  
    5.29  /*Use this version from outside VMS -- it uses external malloc, and modifies
    5.30 @@ -154,8 +143,19 @@
    5.31     return newProbe;
    5.32   }
    5.33  
    5.34 +
    5.35 +/*Only call from inside master or main startup/shutdown thread
    5.36 + */
    5.37 +void
    5.38 +VMS_impl__free_probe( IntervalProbe *probe )
    5.39 + { if( probe->hist != NULL )   freeDblHist( probe->hist );
    5.40 +   if( probe->nameStr != NULL) VMS__free( probe->nameStr );
    5.41 +   VMS__free( probe );
    5.42 + }
    5.43 +
    5.44 +
    5.45  int32
    5.46 -VMS_impl__record_time_point_into_new_probe( char *nameStr, VirtProcr *animPr )
    5.47 +VMS_impl__record_time_point_into_new_probe( char *nameStr, VirtProcr *animPr)
    5.48   { IntervalProbe *newProbe;
    5.49     struct timeval *startStamp;
    5.50     float64 startSecs;
    5.51 @@ -243,7 +243,7 @@
    5.52     probe->schedChoiceWasRecorded = TRUE;
    5.53     probe->coreNum = animatingPr->coreAnimatedBy;
    5.54     probe->procrID = animatingPr->procrID;
    5.55 -   probe->procrCreateSecs = 0;
    5.56 +   probe->procrCreateSecs = animatingPr->createPtInSecs;
    5.57   }
    5.58  
    5.59  /*Everything is local to the animating procr, so no need for request, do

     6.1 --- a/probes.h	Sat Oct 30 21:53:55 2010 -0700
     6.2 +++ b/probes.h	Mon Nov 01 21:21:32 2010 -0700
     6.3 @@ -87,6 +87,10 @@
     6.4                                            binWidth, nameStr, animPr )       \
     6.5          VMS_impl__create_histogram_probe( numBins, startValue,              \
     6.6                                            binWidth, nameStr, animPr )
     6.7 +void
     6.8 +VMS_impl__free_probe( IntervalProbe *probe );
     6.9 +#define VMS__free_probe( probe ) \
    6.10 +        VMS_impl__free_probe( probe )
    6.11  
    6.12  void
    6.13  VMS_impl__index_probe_by_its_name( int32 probeID, VirtProcr *animPr );

     7.1 --- a/vmalloc.c	Sat Oct 30 21:53:55 2010 -0700
     7.2 +++ b/vmalloc.c	Mon Nov 01 21:21:32 2010 -0700
     7.3 @@ -8,6 +8,7 @@
     7.4   */
     7.5  
     7.6  #include <malloc.h>
     7.7 +#include <stdlib.h>
     7.8  
     7.9  #include "VMS.h"
    7.10  
    7.11 @@ -123,14 +124,19 @@
    7.12   { MallocProlog *elemToFree, *nextLowerElem, *nextHigherElem;
    7.13     int32         lowerExistsAndIsFree, higherExistsAndIsFree, sizeOfElem;
    7.14  
    7.15 -   if( ptrToFree < _VMSMasterEnv->freeListHead->nextLowerInMem ||
    7.16 -       ptrToFree > _VMSMasterEnv->freeListHead->nextHigherInMem )
    7.17 +   if( ptrToFree < (void*)_VMSMasterEnv->freeListHead->nextLowerInMem ||
    7.18 +       ptrToFree > (void*)_VMSMasterEnv->freeListHead->nextHigherInMem )
    7.19      {    //outside the range of data owned by VMS's malloc, so do nothing
    7.20        return;
    7.21      }
    7.22        //subtract size of prolog to get pointer to prolog, then cast
    7.23     elemToFree = (MallocProlog *)((char *)ptrToFree - sizeof(MallocProlog));
    7.24     sizeOfElem =(int32)((char*)elemToFree->nextHigherInMem-(char*)elemToFree);
    7.25 +
    7.26 +   if( elemToFree->prevChunkInFreeList != NULL )
    7.27 +    { printf( "error: freeing same element twice!" ); exit(1);
    7.28 +    }
    7.29 +
    7.30     _VMSMasterEnv->amtOfOutstandingMem -= sizeOfElem;
    7.31  
    7.32     nextLowerElem  = elemToFree->nextLowerInMem;
    7.33 @@ -209,10 +215,73 @@
    7.34   }
    7.35  
    7.36  
    7.37 +/*Allocates memory from the external system -- higher overhead
    7.38 + *
    7.39 + *Because of Linux's malloc throwing bizarre random faults when malloc is
    7.40 + * used inside a VMS virtual processor, have to pass this as a request and
    7.41 + * have the core loop do it when it gets around to it -- will look for these
    7.42 + * chores leftover from the previous animation of masterVP the next time it
    7.43 + * goes to animate the masterVP -- so it takes two separate masterVP
    7.44 + * animations, separated by work, to complete an external malloc or
    7.45 + * external free request.
    7.46 + *
    7.47 + *Thinking core loop accepts signals -- just looks if signal-location is
    7.48 + * empty or not --
    7.49 + */
    7.50 +void *
    7.51 +VMS__malloc_in_ext( int32 sizeRequested )
    7.52 + {
    7.53 + /*
    7.54 +      //This is running in the master, so no chance for multiple cores to be
    7.55 +      // competing for the core's flag.
    7.56 +   if(  *(_VMSMasterEnv->coreLoopSignalAddr[ 0 ]) != 0 )
    7.57 +    {    //something has already signalled to core loop, so save the signal
    7.58 +         // and look, next time master animated, to see if can send it.
    7.59 +         //Note, the addr to put a signal is in the coreloop's frame, so just
    7.60 +         // checks it each time through -- make it volatile to avoid GCC
    7.61 +         // optimizations -- it's a coreloop local var that only changes
    7.62 +         // after jumping away.  The signal includes the addr to send the
    7.63 +         //return to -- even if just empty return completion-signal
    7.64 +         //
    7.65 +         //save the signal in some queue that the master looks at each time
    7.66 +         // it starts up -- one loc says if empty for fast common case --
    7.67 +         //something like that -- want to hide this inside this call -- but
    7.68 +         // think this has to come as a request -- req handler gives procr
    7.69 +         // back to master loop, which gives it back to req handler at point
    7.70 +         // it sees that core loop has sent return signal.  Something like
    7.71 +         // that.
    7.72 +      saveTheSignal
    7.73 +
    7.74 +    }
    7.75 +  coreSigData->type = malloc;
    7.76 +  coreSigData->sizeToMalloc = sizeRequested;
    7.77 +  coreSigData->locToSignalCompletion = &figureOut;
    7.78 +   _VMSMasterEnv->coreLoopSignals[ 0 ] = coreSigData;
    7.79 +  */
    7.80 +      //just risk system-stack faults until get this figured out
    7.81 +   return malloc( sizeRequested );
    7.82 + }
    7.83 +
    7.84 +
    7.85 +/*Frees memory that was allocated in the external system -- higher overhead
    7.86 + *
    7.87 + *As noted in external malloc comment, this is clunky 'cause the free has
    7.88 + * to be called in the core loop.
    7.89 + */
    7.90 +void
    7.91 +VMS__free_in_ext( void *ptrToFree )
    7.92 + {
    7.93 +      //just risk system-stack faults until get this figured out
    7.94 +   free( ptrToFree );
    7.95 +
    7.96 +      //TODO: fix this -- so 
    7.97 + }
    7.98 +
    7.99 +
   7.100  /*Designed to be called from the main thread outside of VMS, during init
   7.101   */
   7.102  MallocProlog *
   7.103 -VMS__create_free_list()
   7.104 +VMS_ext__create_free_list()
   7.105   { MallocProlog *freeListHead, *firstChunk;
   7.106  
   7.107        //Note, this is running in the main thread -- all increases in malloc
   7.108 @@ -226,16 +295,18 @@
   7.109        //Use this addr to free the heap when cleanup
   7.110     freeListHead->nextLowerInMem      = firstChunk;
   7.111        //to identify top-of-heap elem, compare this addr to elem's next higher
   7.112 -   freeListHead->nextHigherInMem     = (char *)firstChunk +
   7.113 -                                          MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE;
   7.114 +   freeListHead->nextHigherInMem     = (void*)( (char*)firstChunk +
   7.115 +                                         MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE);
   7.116     freeListHead->nextChunkInFreeList = firstChunk;
   7.117  
   7.118     firstChunk->nextChunkInFreeList   = NULL;
   7.119     firstChunk->prevChunkInFreeList   = freeListHead;
   7.120        //next Higher has to be set to top of chunk, so can calc size in malloc
   7.121 -   firstChunk->nextHigherInMem       = (char *)firstChunk +
   7.122 -                                          MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE;
   7.123 +   firstChunk->nextHigherInMem       = (void*)( (char*)firstChunk +
   7.124 +                                         MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE);
   7.125     firstChunk->nextLowerInMem        = NULL; //identifies as bott of heap
   7.126 +   
   7.127 +   _VMSMasterEnv->amtOfOutstandingMem = 0; //none allocated yet
   7.128  
   7.129     return freeListHead;
   7.130   }

     8.1 --- a/vmalloc.h	Sat Oct 30 21:53:55 2010 -0700
     8.2 +++ b/vmalloc.h	Mon Nov 01 21:21:32 2010 -0700
     8.3 @@ -34,8 +34,19 @@
     8.4  void
     8.5  VMS__free( void *ptrToFree );
     8.6  
     8.7 +/*Allocates memory from the external system -- higher overhead
     8.8 + */
     8.9 +void *
    8.10 +VMS__malloc_in_ext( int32 sizeRequested );
    8.11 +
    8.12 +/*Frees memory that was allocated in the external system -- higher overhead
    8.13 + */
    8.14 +void
    8.15 +VMS__free_in_ext( void *ptrToFree );
    8.16 +
    8.17 +
    8.18  MallocProlog *
    8.19 -VMS__create_free_list();
    8.20 +VMS_ext__create_free_list();
    8.21  
    8.22  void
    8.23  VMS_ext__free_free_list( MallocProlog *freeListHead );