1.1 --- a/VMS.c	Tue Oct 26 18:31:34 2010 -0700
     1.2 +++ b/VMS.c	Thu Nov 11 06:19:51 2010 -0800
     1.3 @@ -6,7 +6,9 @@
     1.4  
     1.5  #include <stdio.h>
     1.6  #include <stdlib.h>
     1.7 +#include <string.h>
     1.8  #include <malloc.h>
     1.9 +#include <sys/time.h>
    1.10  
    1.11  #include "VMS.h"
    1.12  #include "Queue_impl/BlockingQueue.h"
    1.13 @@ -28,6 +30,12 @@
    1.14  void
    1.15  create_the_coreLoop_OS_threads();
    1.16  
    1.17 +MallocProlog *
    1.18 +create_free_list();
    1.19 +
    1.20 +void
    1.21 +endOSThreadFn( void *initData, VirtProcr *animatingPr );
    1.22 +
    1.23  pthread_mutex_t suspendLock = PTHREAD_MUTEX_INITIALIZER;
    1.24  pthread_cond_t  suspend_cond  = PTHREAD_COND_INITIALIZER;
    1.25  
    1.26 @@ -83,34 +91,43 @@
    1.27     int              coreIdx;
    1.28     VirtProcr      **masterVPs;
    1.29     SchedSlot     ***allSchedSlots; //ptr to array of ptrs
    1.30 -   
    1.31 +
    1.32 +
    1.33        //Make the master env, which holds everything else
    1.34     _VMSMasterEnv = malloc( sizeof(MasterEnv) );
    1.35 +
    1.36 +        //Very first thing put into the master env is the free-list, seeded
    1.37 +        // with a massive initial chunk of memory.
    1.38 +        //After this, all other mallocs are VMS__malloc.
    1.39 +   _VMSMasterEnv->freeListHead        = VMS_ext__create_free_list();
    1.40 +
    1.41 +   //===================== Only VMS__malloc after this ====================
    1.42     masterEnv     = _VMSMasterEnv;
    1.43 -      //Need to set start pt here 'cause used by seed procr, which is created
    1.44 -      // before the first core loop starts up. -- not sure how yet..
    1.45 -//   masterEnv->coreLoopStartPt = ;
    1.46 -//   masterEnv->coreLoopEndPt   = ;
    1.47     
    1.48        //Make a readyToAnimateQ for each core loop
    1.49 -   readyToAnimateQs = malloc( NUM_CORES * sizeof(VMSQueueStruc *) );
    1.50 -   masterVPs        = malloc( NUM_CORES * sizeof(VirtProcr *) );
    1.51 +   readyToAnimateQs = VMS__malloc( NUM_CORES * sizeof(VMSQueueStruc *) );
    1.52 +   masterVPs        = VMS__malloc( NUM_CORES * sizeof(VirtProcr *) );
    1.53  
    1.54        //One array for each core, 3 in array, core's masterVP scheds all
    1.55 -   allSchedSlots    = malloc( NUM_CORES * sizeof(SchedSlot *) );
    1.56 +   allSchedSlots    = VMS__malloc( NUM_CORES * sizeof(SchedSlot *) );
    1.57  
    1.58 +   _VMSMasterEnv->numProcrsCreated = 0;  //used by create procr
    1.59     for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    1.60 -    {
    1.61 -      readyToAnimateQs[ coreIdx ] = makeSRSWQ();
    1.62 +    {    
    1.63 +      readyToAnimateQs[ coreIdx ] = makeVMSQ();
    1.64        
    1.65 -         //Q: should give masterVP core-specific into as its init data?
    1.66 +         //Q: should give masterVP core-specific info as its init data?
    1.67        masterVPs[ coreIdx ] = VMS__create_procr( &masterLoop, masterEnv );
    1.68        masterVPs[ coreIdx ]->coreAnimatedBy = coreIdx;
    1.69        allSchedSlots[ coreIdx ] = create_sched_slots(); //makes for one core
    1.70 +      _VMSMasterEnv->numMasterInARow[ coreIdx ] = 0;
    1.71 +      _VMSMasterEnv->workStealingGates[ coreIdx ] = NULL;
    1.72      }
    1.73     _VMSMasterEnv->readyToAnimateQs = readyToAnimateQs;
    1.74     _VMSMasterEnv->masterVPs        = masterVPs;
    1.75 +   _VMSMasterEnv->masterLock       = UNLOCKED;
    1.76     _VMSMasterEnv->allSchedSlots    = allSchedSlots;
    1.77 +   _VMSMasterEnv->workStealingLock = UNLOCKED;
    1.78  
    1.79     //============================= MEASUREMENT STUFF ========================
    1.80     #ifdef MEAS__TIME_MASTER
    1.81 @@ -125,59 +142,33 @@
    1.82        // because coreLoop now controls -- animates its masterVP when no work
    1.83  
    1.84  
    1.85 -   //==================== malloc substitute ========================
    1.86 -   //
    1.87 -   //Testing whether malloc is using thread-local storage and therefore
    1.88 -   // causing unreliable behavior.
    1.89 -   //Just allocate a massive chunk of memory and roll own malloc/free and
    1.90 -   // make app use VMS__malloc_to, which will suspend and perform malloc
    1.91 -   // in the master, taking from this massive chunk.
    1.92 +   //============================= MEASUREMENT STUFF ========================
    1.93 +   #ifdef STATS__TURN_ON_PROBES
    1.94 +   _VMSMasterEnv->dynIntervalProbesInfo =
    1.95 +              makePrivDynArrayOfSize( &(_VMSMasterEnv->intervalProbes), 200);
    1.96  
    1.97 -//   initFreeList();
    1.98 +   _VMSMasterEnv->probeNameHashTbl = makeHashTable( 1000, &VMS__free );
    1.99 +   
   1.100 +      //put creation time directly into master env, for fast retrieval
   1.101 +   struct timeval timeStamp;
   1.102 +   gettimeofday( &(timeStamp), NULL);
   1.103 +   _VMSMasterEnv->createPtInSecs =
   1.104 +                           timeStamp.tv_sec +(timeStamp.tv_usec/1000000.0);
   1.105 +   #endif
   1.106 +   //========================================================================
   1.107  
   1.108   }
   1.109  
   1.110 -/*
   1.111 -void
   1.112 -initMasterMalloc()
   1.113 - {
   1.114 -   _VMSMasterEnv->mallocChunk = malloc( MASSIVE_MALLOC_SIZE );
   1.115 -
   1.116 -      //The free-list element is the first several locations of an
   1.117 -      // allocated chunk -- the address given to the application is pre-
   1.118 -      // pended with both the ownership structure and the free-list struc.
   1.119 -      //So, write the values of these into the first locations of
   1.120 -      // mallocChunk -- which marks it as free & puts in its size.
   1.121 -   listElem = (FreeListElem *)_VMSMasterEnv->mallocChunk;
   1.122 -   listElem->size = MASSIVE_MALLOC_SIZE - NUM_PREPEND_BYTES
   1.123 -   listElem->next = NULL;
   1.124 - }
   1.125 -
   1.126 -void
   1.127 -dissipateMasterMalloc()
   1.128 - {
   1.129 -      //Just foo code -- to get going -- doing as if free list were link-list
   1.130 -   currElem = _VMSMasterEnv->freeList;
   1.131 -   while( currElem != NULL )
   1.132 -    {
   1.133 -      nextElem = currElem->next;
   1.134 -      masterFree( currElem );
   1.135 -      currElem = nextElem;
   1.136 -    }
   1.137 -   free( _VMSMasterEnv->freeList );
   1.138 - }
   1.139 - */
   1.140 -
   1.141  SchedSlot **
   1.142  create_sched_slots()
   1.143   { SchedSlot  **schedSlots;
   1.144     int i;
   1.145  
   1.146 -   schedSlots  = malloc( NUM_SCHED_SLOTS * sizeof(SchedSlot *) );
   1.147 +   schedSlots  = VMS__malloc( NUM_SCHED_SLOTS * sizeof(SchedSlot *) );
   1.148  
   1.149     for( i = 0; i < NUM_SCHED_SLOTS; i++ )
   1.150      {
   1.151 -      schedSlots[i] = malloc( sizeof(SchedSlot) );
   1.152 +      schedSlots[i] = VMS__malloc( sizeof(SchedSlot) );
   1.153  
   1.154           //Set state to mean "handling requests done, slot needs filling"
   1.155        schedSlots[i]->workIsDone         = FALSE;
   1.156 @@ -192,9 +183,9 @@
   1.157   { int i;
   1.158     for( i = 0; i < NUM_SCHED_SLOTS; i++ )
   1.159      {
   1.160 -      free( schedSlots[i] );
   1.161 +      VMS__free( schedSlots[i] );
   1.162      }
   1.163 -   free( schedSlots );
   1.164 +   VMS__free( schedSlots );
   1.165   }
   1.166  
   1.167  
   1.168 @@ -203,7 +194,7 @@
   1.169   {
   1.170     //========================================================================
   1.171     //                      Create the Threads
   1.172 -   int coreIdx, retCode, i;
   1.173 +   int coreIdx, retCode;
   1.174  
   1.175        //create the arrays used to measure TSC offsets between cores
   1.176     pongNums  = malloc( NUM_CORES * sizeof( int ) );
   1.177 @@ -227,7 +218,7 @@
   1.178  
   1.179        //Make the threads that animate the core loops
   1.180     for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
   1.181 -    { coreLoopThdParams[coreIdx]          = malloc( sizeof(ThdParams) );
   1.182 +    { coreLoopThdParams[coreIdx]          = VMS__malloc( sizeof(ThdParams) );
   1.183        coreLoopThdParams[coreIdx]->coreNum = coreIdx;
   1.184  
   1.185        retCode =
   1.186 @@ -235,7 +226,7 @@
   1.187                          thdAttrs,
   1.188                         &coreLoop,
   1.189                 (void *)(coreLoopThdParams[coreIdx]) );
   1.190 -      if(retCode){printf("ERROR creating thread: %d\n", retCode); exit(0);}
   1.191 +      if(retCode){printf("ERROR creating thread: %d\n", retCode); exit(1);}
   1.192      }
   1.193   }
   1.194  
   1.195 @@ -307,10 +298,11 @@
   1.196   * animator state to return to --
   1.197   *
   1.198   */
   1.199 -VirtProcr *
   1.200 -VMS__create_procr( VirtProcrFnPtr fnPtr, void *initialData )
   1.201 - { VirtProcr *newPr;
   1.202 -   char      *stackLocs, *stackPtr;
   1.203 +inline VirtProcr *
   1.204 +create_procr_helper( VirtProcr *newPr,       VirtProcrFnPtr  fnPtr,
   1.205 +                     void      *initialData, char           *stackLocs )
   1.206 + {
   1.207 +   char  *stackPtr;
   1.208  
   1.209     //============================= MEASUREMENT STUFF ========================
   1.210     #ifdef MEAS__TIME_MASTER
   1.211 @@ -318,23 +310,19 @@
   1.212     saveLowTimeStampCountInto( startStamp );
   1.213     #endif
   1.214     //========================================================================
   1.215 -
   1.216 -   newPr              = malloc( sizeof(VirtProcr) );
   1.217 -   newPr->procrID     = numProcrsCreated++;
   1.218 -   newPr->nextInstrPt = fnPtr;
   1.219 -   newPr->initialData = initialData;
   1.220 -   newPr->requests    = NULL;
   1.221 -   newPr->schedSlot   = NULL;
   1.222 -//   newPr->coreLoopStartPt = _VMSMasterEnv->coreLoopStartPt;
   1.223 +   newPr->startOfStack = stackLocs;
   1.224 +   newPr->procrID      = _VMSMasterEnv->numProcrsCreated++;
   1.225 +   newPr->nextInstrPt  = fnPtr;
   1.226 +   newPr->initialData  = initialData;
   1.227 +   newPr->requests     = NULL;
   1.228 +   newPr->schedSlot    = NULL;
   1.229  
   1.230        //fnPtr takes two params -- void *initData & void *animProcr
   1.231        //alloc stack locations, make stackPtr be the highest addr minus room
   1.232        // for 2 params + return addr.  Return addr (NULL) is in loc pointed to
   1.233        // by stackPtr, initData at stackPtr + 4 bytes, animatingPr just above
   1.234 -   stackLocs = malloc( VIRT_PROCR_STACK_SIZE );
   1.235 -         if(stackLocs == 0) {perror("error: malloc stack"); exit(1);}
   1.236 -   newPr->startOfStack = stackLocs;
   1.237     stackPtr = ( (char *)stackLocs + VIRT_PROCR_STACK_SIZE - 0x10 );
   1.238 +   
   1.239        //setup __cdecl on stack -- coreloop will switch to stackPtr before jmp
   1.240     *( (int *)stackPtr + 2 ) = (int) newPr; //rightmost param -- 32bit pointer
   1.241     *( (int *)stackPtr + 1 ) = (int) initialData;  //next  param to left
   1.242 @@ -347,12 +335,48 @@
   1.243     saveLowTimeStampCountInto( endStamp );
   1.244     addIntervalToHist( startStamp, endStamp,
   1.245                        _VMSMasterEnv->stats->createHist );
   1.246 +   //============================= MEASUREMENT STUFF ========================
   1.247 +   #ifdef STATS__TURN_ON_PROBES
   1.248 +   struct timeval timeStamp;
   1.249 +   gettimeofday( &(timeStamp), NULL);
   1.250 +   newPr->createPtInSecs = timeStamp.tv_sec +(timeStamp.tv_usec/1000000.0) -
   1.251 +                                               _VMSMasterEnv->createPtInSecs;
   1.252     #endif
   1.253     //========================================================================
   1.254 -   
   1.255 +
   1.256     return newPr;
   1.257   }
   1.258  
   1.259 +inline VirtProcr *
   1.260 +VMS__create_procr( VirtProcrFnPtr fnPtr, void *initialData )
   1.261 + { VirtProcr *newPr;
   1.262 +   char      *stackLocs;
   1.263 +
   1.264 +   newPr      = VMS__malloc( sizeof(VirtProcr) );
   1.265 +   stackLocs  = VMS__malloc( VIRT_PROCR_STACK_SIZE );
   1.266 +   if( stackLocs == 0 )
   1.267 +    { perror("VMS__malloc stack"); exit(1); }
   1.268 +
   1.269 +   return create_procr_helper( newPr, fnPtr, initialData, stackLocs );
   1.270 + }
   1.271 +
   1.272 +/* "ext" designates that it's for use outside the VMS system -- should only
   1.273 + * be called from main thread or other thread -- never from code animated by
   1.274 + * a VMS virtual processor.
   1.275 + */
   1.276 +inline VirtProcr *
   1.277 +VMS_ext__create_procr( VirtProcrFnPtr fnPtr, void *initialData )
   1.278 + { VirtProcr *newPr;
   1.279 +   char      *stackLocs;
   1.280 +
   1.281 +   newPr      = malloc( sizeof(VirtProcr) );
   1.282 +   stackLocs  = malloc( VIRT_PROCR_STACK_SIZE );
   1.283 +   if( stackLocs == 0 )
   1.284 +    { perror("malloc stack"); exit(1); }
   1.285 +
   1.286 +   return create_procr_helper( newPr, fnPtr, initialData, stackLocs );
   1.287 + }
   1.288 +
   1.289  
   1.290  /*there is a label inside this function -- save the addr of this label in
   1.291   * the callingPr struc, as the pick-up point from which to start the next
   1.292 @@ -365,8 +389,7 @@
   1.293   */
   1.294  void
   1.295  VMS__suspend_procr( VirtProcr *animatingPr )
   1.296 - { void *jmpPt, *stackPtrAddr, *framePtrAddr, *coreLoopStackPtr;
   1.297 -   void *coreLoopFramePtr;
   1.298 + { 
   1.299  
   1.300        //The request to master will cause this suspended virt procr to get
   1.301        // scheduled again at some future point -- to resume, core loop jumps
   1.302 @@ -376,24 +399,6 @@
   1.303  
   1.304        //return ownership of the virt procr and sched slot to Master virt pr
   1.305     animatingPr->schedSlot->workIsDone = TRUE;
   1.306 -//   coreIdx = callingPr->coreAnimatedBy;
   1.307 -
   1.308 -   stackPtrAddr      = &(animatingPr->stackPtr);
   1.309 -   framePtrAddr      = &(animatingPr->framePtr);
   1.310 -
   1.311 -   jmpPt             = _VMSMasterEnv->coreLoopStartPt;
   1.312 -   coreLoopFramePtr  = animatingPr->coreLoopFramePtr;//need this only
   1.313 -   coreLoopStackPtr  = animatingPr->coreLoopStackPtr;//safety
   1.314 -
   1.315 -      //Save the virt procr's stack and frame ptrs,
   1.316 -   asm volatile("movl %0,     %%eax;  \
   1.317 -                 movl %%esp, (%%eax); \
   1.318 -                 movl %1,     %%eax;  \
   1.319 -                 movl %%ebp, (%%eax) "\
   1.320 -   /* outputs */ : "=g" (stackPtrAddr), "=g" (framePtrAddr) \
   1.321 -   /* inputs  */ :        \
   1.322 -   /* clobber */ : "%eax" \
   1.323 -                );
   1.324  
   1.325     //===========================  Measurement stuff ========================
   1.326     #ifdef MEAS__TIME_STAMP_SUSP
   1.327 @@ -402,20 +407,10 @@
   1.328     #endif
   1.329     //=======================================================================
   1.330  
   1.331 -      //restore coreloop's frame ptr, then jump back to "start" of core loop
   1.332 -      //Note, GCC compiles to assembly that saves esp and ebp in the stack
   1.333 -      // frame -- so have to explicitly do assembly that saves to memory
   1.334 -   asm volatile("movl %0, %%eax;      \
   1.335 -                 movl %1, %%esp;      \
   1.336 -                 movl %2, %%ebp;      \
   1.337 -                 jmp  %%eax    "      \
   1.338 -   /* outputs */ :                    \
   1.339 -   /* inputs  */ : "m" (jmpPt), "m"(coreLoopStackPtr), "m"(coreLoopFramePtr)\
   1.340 -   /* clobber */ : "memory", "%eax", "%ebx", "%ecx", "%edx", "%edi","%esi"  \
   1.341 -                ); //list everything as clobbered to force GCC to save all
   1.342 -                   // live vars that are in regs on stack before this
   1.343 -                   // assembly, so that stack pointer is correct, before jmp
   1.344  
   1.345 +   SwitchToCoreLoop( animatingPr )
   1.346 +
   1.347 +   //=======================================================================
   1.348  ResumePt:
   1.349     #ifdef MEAS__TIME_STAMP_SUSP
   1.350        //NOTE: only take low part of count -- do sanity check when take diff
   1.351 @@ -427,6 +422,31 @@
   1.352  
   1.353  
   1.354  
   1.355 +/*For this implementation of VMS, it may not make much sense to have the
   1.356 + * system of requests for creating a new processor done this way.. but over
   1.357 + * the scope of single-master, multi-master, mult-tasking, OS-implementing,
   1.358 + * distributed-memory, and so on, this gives VMS implementation a chance to
   1.359 + * do stuff before suspend, in the AppVP, and in the Master before the plugin
   1.360 + * is called, as well as in the lang-lib before this is called, and in the
   1.361 + * plugin.  So, this gives both VMS and language implementations a chance to
   1.362 + * intercept at various points and do order-dependent stuff.
   1.363 + *Having a standard VMSNewPrReqData struc allows the language to create and
   1.364 + * free the struc, while VMS knows how to get the newPr if it wants it, and
   1.365 + * it lets the lang have lang-specific data related to creation transported
   1.366 + * to the plugin.
   1.367 + */
   1.368 +void
   1.369 +VMS__send_create_procr_req( void *semReqData, VirtProcr *reqstingPr )
   1.370 + { VMSReqst req;
   1.371 +
   1.372 +   req.reqType          = createReq;
   1.373 +   req.semReqData       = semReqData;
   1.374 +   req.nextReqst        = reqstingPr->requests;
   1.375 +   reqstingPr->requests = &req;
   1.376 +
   1.377 +   VMS__suspend_procr( reqstingPr );
   1.378 + }
   1.379 +
   1.380  
   1.381  /*
   1.382   *This adds a request to dissipate, then suspends the processor so that the
   1.383 @@ -450,81 +470,102 @@
   1.384   * pears -- making that suspend the last thing in the virt procr's trace.
   1.385   */
   1.386  void
   1.387 -VMS__dissipate_procr( VirtProcr *procrToDissipate )
   1.388 +VMS__send_dissipate_req( VirtProcr *procrToDissipate )
   1.389 + { VMSReqst req;
   1.390 +
   1.391 +   req.reqType                = dissipate;
   1.392 +   req.nextReqst              = procrToDissipate->requests;
   1.393 +   procrToDissipate->requests = &req;
   1.394 +
   1.395 +   VMS__suspend_procr( procrToDissipate );
   1.396 + }
   1.397 +
   1.398 +
   1.399 +/* "ext" designates that it's for use outside the VMS system -- should only
   1.400 + * be called from main thread or other thread -- never from code animated by
   1.401 + * a VMS virtual processor.
   1.402 + *
   1.403 + *Use this version to dissipate VPs created outside the VMS system.
   1.404 + */
   1.405 +void
   1.406 +VMS_ext__dissipate_procr( VirtProcr *procrToDissipate )
   1.407 + {
   1.408 +      //NOTE: initialData was given to the processor, so should either have
   1.409 +      // been alloc'd with VMS__malloc, or freed by the level above animPr.
   1.410 +      //So, all that's left to free here is the stack and the VirtProcr struc
   1.411 +      // itself
   1.412 +      //Note, should not stack-allocate initial data -- no guarantee, in
   1.413 +      // general that creating processor will outlive ones it creates.
   1.414 +   free( procrToDissipate->startOfStack );
   1.415 +   free( procrToDissipate );
   1.416 + }
   1.417 +
   1.418 +
   1.419 +
   1.420 +/*This call's name indicates that request is malloc'd -- so req handler
   1.421 + * has to free any extra requests tacked on before a send, using this.
   1.422 + *
   1.423 + * This inserts the semantic-layer's request data into standard VMS carrier
   1.424 + * request data-struct that is mallocd.  The sem request doesn't need to
   1.425 + * be malloc'd if this is called inside the same call chain before the
   1.426 + * send of the last request is called.
   1.427 + *
   1.428 + *The request handler has to call VMS__free_VMSReq for any of these
   1.429 + */
   1.430 +inline void
   1.431 +VMS__add_sem_request_in_mallocd_VMSReqst( void *semReqData,
   1.432 +                                          VirtProcr *callingPr )
   1.433   { VMSReqst *req;
   1.434  
   1.435 -   req = malloc( sizeof(VMSReqst) );
   1.436 -//   req->virtProcrFrom      = callingPr;
   1.437 -   req->reqType               = dissipate;
   1.438 -   req->nextReqst             = procrToDissipate->requests;
   1.439 -   procrToDissipate->requests = req;
   1.440 -   
   1.441 -   VMS__suspend_procr( procrToDissipate );
   1.442 -}
   1.443 -
   1.444 -
   1.445 -/*This inserts the semantic-layer's request data into standard VMS carrier
   1.446 - */
   1.447 -inline void
   1.448 -VMS__add_sem_request( void *semReqData, VirtProcr *callingPr )
   1.449 - { VMSReqst *req;
   1.450 -
   1.451 -   req = malloc( sizeof(VMSReqst) );
   1.452 -//   req->virtProcrFrom      = callingPr;
   1.453 -   req->reqType        = semantic;
   1.454 -   req->semReqData     = semReqData;
   1.455 -   req->nextReqst      = callingPr->requests;
   1.456 +   req = VMS__malloc( sizeof(VMSReqst) );
   1.457 +   req->reqType         = semantic;
   1.458 +   req->semReqData      = semReqData;
   1.459 +   req->nextReqst       = callingPr->requests;
   1.460     callingPr->requests = req;
   1.461   }
   1.462  
   1.463 +/*This inserts the semantic-layer's request data into standard VMS carrier
   1.464 + * request data-struct is allocated on stack of this call & ptr to it sent
   1.465 + * to plugin
   1.466 + *Then it does suspend, to cause request to be sent.
   1.467 + */
   1.468 +inline void
   1.469 +VMS__send_sem_request( void *semReqData, VirtProcr *callingPr )
   1.470 + { VMSReqst req;
   1.471  
   1.472 -/*Use this to get first request before starting request handler's loop
   1.473 +   req.reqType         = semantic;
   1.474 +   req.semReqData      = semReqData;
   1.475 +   req.nextReqst       = callingPr->requests;
   1.476 +   callingPr->requests = &req;
   1.477 +   
   1.478 +   VMS__suspend_procr( callingPr );
   1.479 + }
   1.480 +
   1.481 +
   1.482 +inline void
   1.483 +VMS__send_VMSSem_request( void *semReqData, VirtProcr *callingPr )
   1.484 + { VMSReqst req;
   1.485 +
   1.486 +   req.reqType         = VMSSemantic;
   1.487 +   req.semReqData      = semReqData;
   1.488 +   req.nextReqst       = callingPr->requests; //gab any other preceeding 
   1.489 +   callingPr->requests = &req;
   1.490 +
   1.491 +   VMS__suspend_procr( callingPr );
   1.492 + }
   1.493 +
   1.494 +
   1.495 +/*
   1.496   */
   1.497  VMSReqst *
   1.498 -VMS__take_top_request_from( VirtProcr *procrWithReq )
   1.499 - { VMSReqst *req;
   1.500 -
   1.501 -   req = procrWithReq->requests;
   1.502 -   if( req == NULL ) return req;
   1.503 -
   1.504 -   procrWithReq->requests = procrWithReq->requests->nextReqst;
   1.505 -   return req;
   1.506 - }
   1.507 -
   1.508 -/*A subtle bug due to freeing then accessing "next" after freed caused this
   1.509 - * form of call to be put in -- so call this at end of request handler loop
   1.510 - * that iterates through the requests.
   1.511 - */
   1.512 -VMSReqst *
   1.513 -VMS__free_top_and_give_next_request_from( VirtProcr *procrWithReq )
   1.514 +VMS__take_next_request_out_of( VirtProcr *procrWithReq )
   1.515   { VMSReqst *req;
   1.516  
   1.517     req = procrWithReq->requests;
   1.518     if( req == NULL ) return NULL;
   1.519  
   1.520     procrWithReq->requests = procrWithReq->requests->nextReqst;
   1.521 -   VMS__free_request( req );
   1.522 -   return procrWithReq->requests;
   1.523 - }
   1.524 -
   1.525 -
   1.526 -//TODO: add a semantic-layer supplied "freer" for the semantic-data portion
   1.527 -// of a request -- IE call with both a virt procr and a fn-ptr to request
   1.528 -// freer (also maybe put sem request freer as a field in virt procr?)
   1.529 -//MeasVMS relies right now on this only freeing VMS layer of request -- the
   1.530 -// semantic portion of request is alloc'd and freed by request handler
   1.531 -void
   1.532 -VMS__free_request( VMSReqst *req )
   1.533 - {
   1.534 -   free( req );
   1.535 - }
   1.536 -
   1.537 -
   1.538 -
   1.539 -inline int
   1.540 -VMS__isSemanticReqst( VMSReqst *req )
   1.541 - {
   1.542 -   return ( req->reqType == semantic );
   1.543 +   return req;
   1.544   }
   1.545  
   1.546  
   1.547 @@ -534,36 +575,52 @@
   1.548     return req->semReqData;
   1.549   }
   1.550  
   1.551 -inline int
   1.552 -VMS__isDissipateReqst( VMSReqst *req )
   1.553 - {
   1.554 -   return ( req->reqType == dissipate );
   1.555 - }
   1.556  
   1.557 -inline int
   1.558 -VMS__isCreateReqst( VMSReqst *req )
   1.559 - {
   1.560 -   return ( req->reqType == regCreated );
   1.561 - }
   1.562  
   1.563 -void
   1.564 -VMS__send_req_to_register_new_procr(VirtProcr *newPr, VirtProcr *reqstingPr)
   1.565 - { VMSReqst *req;
   1.566 +/* This is for OS requests and VMS infrastructure requests, such as to create
   1.567 + *  a probe -- a probe is inside the heart of VMS-core, it's not part of any
   1.568 + *  language -- but it's also a semantic thing that's triggered from and used
   1.569 + *  in the application.. so it crosses abstractions..  so, need some special
   1.570 + *  pattern here for handling such requests.
   1.571 + * Doing this just like it were a second language sharing VMS-core.
   1.572 + * 
   1.573 + * This is called from the language's request handler when it sees a request
   1.574 + *  of type VMSSemReq
   1.575 + *
   1.576 + * TODO: Later change this, to give probes their own separate plugin & have
   1.577 + *  VMS-core steer the request to appropriate plugin
   1.578 + * Do the same for OS calls -- look later at it..
   1.579 + */
   1.580 +void inline
   1.581 +VMS__handle_VMSSemReq( VMSReqst *req, VirtProcr *requestingPr, void *semEnv,
   1.582 +                       ResumePrFnPtr resumePrFnPtr )
   1.583 + { VMSSemReq     *semReq;
   1.584 +   IntervalProbe *newProbe;
   1.585 +   int32          nameLen;
   1.586  
   1.587 -   req                  = malloc( sizeof(VMSReqst) );
   1.588 -   req->reqType         = regCreated;
   1.589 -   req->semReqData      = newPr;
   1.590 -   req->nextReqst       = reqstingPr->requests;
   1.591 -   reqstingPr->requests = req;
   1.592 +   semReq = req->semReqData;
   1.593  
   1.594 -   VMS__suspend_procr( reqstingPr );
   1.595 +   newProbe          = VMS__malloc( sizeof(IntervalProbe) );
   1.596 +   nameLen = strlen( semReq->nameStr );
   1.597 +   newProbe->nameStr = VMS__malloc( nameLen );
   1.598 +   memcpy( newProbe->nameStr, semReq->nameStr, nameLen );
   1.599 +   newProbe->hist    = NULL;
   1.600 +   newProbe->schedChoiceWasRecorded = FALSE;
   1.601 +
   1.602 +      //This runs in masterVP, so no race-condition worries
   1.603 +   newProbe->probeID =
   1.604 +             addToDynArray( newProbe, _VMSMasterEnv->dynIntervalProbesInfo );
   1.605 +
   1.606 +   requestingPr->dataRetFromReq = newProbe;
   1.607 +
   1.608 +   (*resumePrFnPtr)( requestingPr, semEnv );
   1.609   }
   1.610  
   1.611  
   1.612  
   1.613  /*This must be called by the request handler plugin -- it cannot be called
   1.614   * from the semantic library "dissipate processor" function -- instead, the
   1.615 - * semantic layer has to generate a request for the plug-in to call this
   1.616 + * semantic layer has to generate a request, and the plug-in calls this
   1.617   * function.
   1.618   *The reason is that this frees the virtual processor's stack -- which is
   1.619   * still in use inside semantic library calls!
   1.620 @@ -579,33 +636,31 @@
   1.621   * of dis-owning it.
   1.622   */
   1.623  void
   1.624 -VMS__handle_dissipate_reqst( VirtProcr *animatingPr )
   1.625 +VMS__dissipate_procr( VirtProcr *animatingPr )
   1.626   {
   1.627        //dis-own all locations owned by this processor, causing to be freed
   1.628        // any locations that it is (was) sole owner of
   1.629  //TODO: implement VMS__malloc system, including "give up ownership"
   1.630  
   1.631 -      //The dissipate request might still be attached, so remove and free it
   1.632 -   VMS__free_top_and_give_next_request_from( animatingPr );
   1.633  
   1.634        //NOTE: initialData was given to the processor, so should either have
   1.635        // been alloc'd with VMS__malloc, or freed by the level above animPr.
   1.636        //So, all that's left to free here is the stack and the VirtProcr struc
   1.637        // itself
   1.638 -   free( animatingPr->startOfStack );
   1.639 -   free( animatingPr );
   1.640 +      //Note, should not stack-allocate initial data -- no guarantee, in
   1.641 +      // general that creating processor will outlive ones it creates.
   1.642 +   VMS__free( animatingPr->startOfStack );
   1.643 +   VMS__free( animatingPr );
   1.644   }
   1.645  
   1.646  
   1.647 -//TODO: re-architect so that have clean separation between request handler
   1.648 +//TODO: look at architecting cleanest separation between request handler
   1.649  // and master loop, for dissipate, create, shutdown, and other non-semantic
   1.650  // requests.  Issue is chain: one removes requests from AppVP, one dispatches
   1.651  // on type of request, and one handles each type..  but some types require
   1.652  // action from both request handler and master loop -- maybe just give the
   1.653  // request handler calls like:  VMS__handle_X_request_type
   1.654  
   1.655 -void
   1.656 -endOSThreadFn( void *initData, VirtProcr *animatingPr );
   1.657  
   1.658  /*This is called by the semantic layer's request handler when it decides its
   1.659   * time to shut down the VMS system.  Calling this causes the core loop OS
   1.660 @@ -619,10 +674,9 @@
   1.661   * masterVP any AppVPs that might still be allocated and sitting in the
   1.662   * semantic environment, or have been orphaned in the _VMSWorkQ.
   1.663   * 
   1.664 - *NOTE: the semantic plug-in is expected to use VMS__malloc_to to get all the
   1.665 + *NOTE: the semantic plug-in is expected to use VMS__malloc to get all the
   1.666   * locations it needs, and give ownership to masterVP.  Then, they will be
   1.667 - * automatically freed when the masterVP is dissipated.  (This happens after
   1.668 - * the core loop threads have all exited)
   1.669 + * automatically freed.
   1.670   *
   1.671   *In here,create one core-loop shut-down processor for each core loop and put
   1.672   * them all directly into the readyToAnimateQ.
   1.673 @@ -633,16 +687,16 @@
   1.674   * point is it sure that all results have completed.
   1.675   */
   1.676  void
   1.677 -VMS__handle_shutdown_reqst( void *dummy, VirtProcr *animatingPr )
   1.678 +VMS__shutdown()
   1.679   { int coreIdx;
   1.680     VirtProcr *shutDownPr;
   1.681  
   1.682        //create the shutdown processors, one for each core loop -- put them
   1.683        // directly into the Q -- each core will die when gets one
   1.684     for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
   1.685 -    {
   1.686 +    {    //Note, this is running in the master
   1.687        shutDownPr = VMS__create_procr( &endOSThreadFn, NULL );
   1.688 -      writeSRSWQ( shutDownPr, _VMSMasterEnv->readyToAnimateQs[coreIdx] );
   1.689 +      writeVMSQ( shutDownPr, _VMSMasterEnv->readyToAnimateQs[coreIdx] );
   1.690      }
   1.691  
   1.692   }
   1.693 @@ -681,49 +735,60 @@
   1.694   }
   1.695  
   1.696  
   1.697 -/*This is called after the threads have shut down and control has returned
   1.698 - * to the semantic layer, in the entry point function in the main thread.
   1.699 - * It has to free anything allocated during VMS_init, and any other alloc'd
   1.700 - * locations that might be left over.
   1.701 +/*This is called from the startup & shutdown
   1.702   */
   1.703  void
   1.704 -VMS__cleanup_after_shutdown()
   1.705 +VMS__cleanup_at_end_of_shutdown()
   1.706   { 
   1.707     VMSQueueStruc **readyToAnimateQs;
   1.708     int              coreIdx;
   1.709     VirtProcr      **masterVPs;
   1.710     SchedSlot     ***allSchedSlots; //ptr to array of ptrs
   1.711  
   1.712 +      //All the environment data has been allocated with VMS__malloc, so just
   1.713 +      // free its internal big-chunk and all inside it disappear.
   1.714 +/*
   1.715     readyToAnimateQs = _VMSMasterEnv->readyToAnimateQs;
   1.716     masterVPs        = _VMSMasterEnv->masterVPs;
   1.717     allSchedSlots    = _VMSMasterEnv->allSchedSlots;
   1.718     
   1.719     for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
   1.720      {
   1.721 -      freeSRSWQ( readyToAnimateQs[ coreIdx ] );
   1.722 -
   1.723 -      VMS__handle_dissipate_reqst( masterVPs[ coreIdx ] );
   1.724 +      freeVMSQ( readyToAnimateQs[ coreIdx ] );
   1.725 +         //master VPs were created external to VMS, so use external free
   1.726 +      VMS__dissipate_procr( masterVPs[ coreIdx ] );
   1.727        
   1.728        freeSchedSlots( allSchedSlots[ coreIdx ] );
   1.729      }
   1.730     
   1.731 -   free( _VMSMasterEnv->readyToAnimateQs );
   1.732 -   free( _VMSMasterEnv->masterVPs );
   1.733 -   free( _VMSMasterEnv->allSchedSlots );
   1.734 -
   1.735 -   free( _VMSMasterEnv );
   1.736 +   VMS__free( _VMSMasterEnv->readyToAnimateQs );
   1.737 +   VMS__free( _VMSMasterEnv->masterVPs );
   1.738 +   VMS__free( _VMSMasterEnv->allSchedSlots );
   1.739 +   
   1.740 +   //============================= MEASUREMENT STUFF ========================
   1.741 +   #ifdef STATS__TURN_ON_PROBES
   1.742 +   freeDynArrayDeep( _VMSMasterEnv->dynIntervalProbesInfo, &VMS__free_probe);
   1.743 +   #endif
   1.744 +   //========================================================================
   1.745 +*/
   1.746 +      //These are the only two that use system free 
   1.747 +   VMS_ext__free_free_list( _VMSMasterEnv->freeListHead );
   1.748 +   free( (void *)_VMSMasterEnv );
   1.749   }
   1.750  
   1.751  
   1.752 -//===========================================================================
   1.753 +//================================
   1.754  
   1.755 -inline TSCount getTSC()
   1.756 - { unsigned int low, high;
   1.757 -   TSCount  out;
   1.758  
   1.759 -   saveTimeStampCountInto( low, high );
   1.760 -   out = high;
   1.761 -   out = (out << 32) + low;
   1.762 -   return out;
   1.763 +/*Later, improve this -- for now, just exits the application after printing
   1.764 + * the error message.
   1.765 + */
   1.766 +void
   1.767 +VMS__throw_exception( char *msgStr, VirtProcr *reqstPr, VMSExcp *excpData )
   1.768 + {
   1.769 +   printf(msgStr);
   1.770 +   fflush(stdin);
   1.771 +   exit(1);
   1.772   }
   1.773  
   1.774 +