/*
 * Copyright 2010  OpenSourceStewardshipFoundation
 *
 * Licensed under BSD
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <malloc.h>
#include <inttypes.h>
#include <sys/time.h>

#include "VMS.h"
#include "ProcrContext.h"
#include "Queue_impl/BlockingQueue.h"
#include "Histogram/Histogram.h"


#define thdAttrs NULL

//===========================================================================
void
shutdownFn( void *dummy, VirtProcr *dummy2 );

SchedSlot **
create_sched_slots();

void
create_masterEnv();

void
create_the_coreLoop_OS_threads();

MallocProlog *
create_free_list();

void
endOSThreadFn( void *initData, VirtProcr *animatingPr );

pthread_mutex_t suspendLock = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t  suspend_cond  = PTHREAD_COND_INITIALIZER;

//===========================================================================

/*Setup has two phases:
 * 1) Semantic layer first calls init_VMS, which creates masterEnv, and puts
 *    the master virt procr into the work-queue, ready for first "call"
 * 2) Semantic layer then does its own init, which creates the seed virt
 *    procr inside the semantic layer, ready to schedule it when
 *    asked by the first run of the masterLoop.
 *
 *This part is bit weird because VMS really wants to be "always there", and
 * have applications attach and detach..  for now, this VMS is part of
 * the app, so the VMS system starts up as part of running the app.
 *
 *The semantic layer is isolated from the VMS internals by making the
 * semantic layer do setup to a state that it's ready with its
 * initial virt procrs, ready to schedule them to slots when the masterLoop
 * asks.  Without this pattern, the semantic layer's setup would
 * have to modify slots directly to assign the initial virt-procrs, and put
 * them into the readyToAnimateQ itself, breaking the isolation completely.
 *
 * 
 *The semantic layer creates the initial virt procr(s), and adds its
 * own environment to masterEnv, and fills in the pointers to
 * the requestHandler and slaveScheduler plug-in functions
 */

/*This allocates VMS data structures, populates the master VMSProc,
 * and master environment, and returns the master environment to the semantic
 * layer.
 */
void
VMS__init()
 {
   create_masterEnv();
   create_the_coreLoop_OS_threads();
 }

#ifdef SEQUENTIAL

/*To initialize the sequential version, just don't create the threads
 */
void
VMS__init_Seq()
 {
   create_masterEnv();
 }

#endif

void
create_masterEnv()
 { MasterEnv       *masterEnv;
   VMSQueueStruc **readyToAnimateQs;
   int              coreIdx;
   VirtProcr      **masterVPs;
   SchedSlot     ***allSchedSlots; //ptr to array of ptrs


      //Make the master env, which holds everything else
   _VMSMasterEnv = malloc( sizeof(MasterEnv) );

        //Very first thing put into the master env is the free-list, seeded
        // with a massive initial chunk of memory.
        //After this, all other mallocs are VMS__malloc.
   _VMSMasterEnv->freeListHead        = VMS_ext__create_free_list();


   //============================= MEASUREMENT STUFF ========================
   #ifdef MEAS__TIME_MALLOC
   _VMSMasterEnv->mallocTimeHist  = makeFixedBinHistExt( 100, 0, 100,
                                                       "malloc_time_hist");
   _VMSMasterEnv->freeTimeHist  = makeFixedBinHistExt( 80, 0, 100,
                                                       "free_time_hist");
   #endif
   #ifdef MEAS__TIME_PLUGIN
   _VMSMasterEnv->reqHdlrLowTimeHist  = makeFixedBinHistExt( 1000, 0, 100,
                                                     "plugin_low_time_hist");
   _VMSMasterEnv->reqHdlrHighTimeHist  = makeFixedBinHistExt( 1000, 0, 100,
                                                    "plugin_high_time_hist");
   #endif
   //========================================================================

   //===================== Only VMS__malloc after this ====================
   masterEnv     = (MasterEnv*)_VMSMasterEnv;
   
      //Make a readyToAnimateQ for each core loop
   readyToAnimateQs = VMS__malloc( NUM_CORES * sizeof(VMSQueueStruc *) );
   masterVPs        = VMS__malloc( NUM_CORES * sizeof(VirtProcr *) );

      //One array for each core, 3 in array, core's masterVP scheds all
   allSchedSlots    = VMS__malloc( NUM_CORES * sizeof(SchedSlot *) );

   _VMSMasterEnv->numProcrsCreated = 0;  //used by create procr
   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    {    
      readyToAnimateQs[ coreIdx ] = makeVMSQ();
      
         //Q: should give masterVP core-specific info as its init data?
      masterVPs[ coreIdx ] = VMS__create_procr( (VirtProcrFnPtr)&masterLoop, (void*)masterEnv );
      masterVPs[ coreIdx ]->coreAnimatedBy = coreIdx;
      allSchedSlots[ coreIdx ] = create_sched_slots(); //makes for one core
      _VMSMasterEnv->numMasterInARow[ coreIdx ] = 0;
      _VMSMasterEnv->workStealingGates[ coreIdx ] = NULL;
    }
   _VMSMasterEnv->readyToAnimateQs = readyToAnimateQs;
   _VMSMasterEnv->masterVPs        = masterVPs;
   _VMSMasterEnv->masterLock       = UNLOCKED;
   _VMSMasterEnv->allSchedSlots    = allSchedSlots;
   _VMSMasterEnv->workStealingLock = UNLOCKED;


      //Aug 19, 2010:  no longer need to place initial masterVP into queue
      // because coreLoop now controls -- animates its masterVP when no work


   //============================= MEASUREMENT STUFF ========================
   #ifdef STATS__TURN_ON_PROBES
   _VMSMasterEnv->dynIntervalProbesInfo =
              makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->intervalProbes), 200);

   _VMSMasterEnv->probeNameHashTbl = makeHashTable( 1000, &VMS__free );
   
      //put creation time directly into master env, for fast retrieval
   struct timeval timeStamp;
   gettimeofday( &(timeStamp), NULL);
   _VMSMasterEnv->createPtInSecs =
                           timeStamp.tv_sec +(timeStamp.tv_usec/1000000.0);
   #endif
   #ifdef MEAS__TIME_MASTER_LOCK
   _VMSMasterEnv->masterLockLowTimeHist  = makeFixedBinHist( 50, 0, 2,
                                                "master lock low time hist");
   _VMSMasterEnv->masterLockHighTimeHist  = makeFixedBinHist( 50, 0, 100,
                                               "master lock high time hist");
   #endif
   
   MakeTheMeasHists();
   //========================================================================

 }

SchedSlot **
create_sched_slots()
 { SchedSlot  **schedSlots;
   int i;

   schedSlots  = VMS__malloc( NUM_SCHED_SLOTS * sizeof(SchedSlot *) );

   for( i = 0; i < NUM_SCHED_SLOTS; i++ )
    {
      schedSlots[i] = VMS__malloc( sizeof(SchedSlot) );

         //Set state to mean "handling requests done, slot needs filling"
      schedSlots[i]->workIsDone         = FALSE;
      schedSlots[i]->needsProcrAssigned = TRUE;
    }
   return schedSlots;
 }


void
freeSchedSlots( SchedSlot **schedSlots )
 { int i;
   for( i = 0; i < NUM_SCHED_SLOTS; i++ )
    {
      VMS__free( schedSlots[i] );
    }
   VMS__free( schedSlots );
 }


void
create_the_coreLoop_OS_threads()
 {
   //========================================================================
   //                      Create the Threads
   int coreIdx, retCode;

      //Need the threads to be created suspended, and wait for a signal
      // before proceeding -- gives time after creating to initialize other
      // stuff before the coreLoops set off.
   _VMSMasterEnv->setupComplete = 0;

      //Make the threads that animate the core loops
   for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
    { coreLoopThdParams[coreIdx]          = VMS__malloc( sizeof(ThdParams) );
      coreLoopThdParams[coreIdx]->coreNum = coreIdx;

      retCode =
      pthread_create( &(coreLoopThdHandles[coreIdx]),
                        thdAttrs,
                       &coreLoop,
               (void *)(coreLoopThdParams[coreIdx]) );
      if(retCode){printf("ERROR creating thread: %d\n", retCode); exit(1);}
    }
 }

/*Semantic layer calls this when it want the system to start running..
 *
 *This starts the core loops running then waits for them to exit.
 */
void
VMS__start_the_work_then_wait_until_done()
 { int coreIdx;
      //Start the core loops running
   
      //tell the core loop threads that setup is complete
      //get lock, to lock out any threads still starting up -- they'll see
      // that setupComplete is true before entering while loop, and so never
      // wait on the condition
   pthread_mutex_lock(     &suspendLock );
   _VMSMasterEnv->setupComplete = 1;
   pthread_mutex_unlock(   &suspendLock );
   pthread_cond_broadcast( &suspend_cond );
   
   
      //wait for all to complete
   for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
    {
      pthread_join( coreLoopThdHandles[coreIdx], NULL );
    }
   
      //NOTE: do not clean up VMS env here -- semantic layer has to have
      // a chance to clean up its environment first, then do a call to free
      // the Master env and rest of VMS locations
 }

#ifdef SEQUENTIAL
/*Only difference between version with an OS thread pinned to each core and
 * the sequential version of VMS is VMS__init_Seq, this, and coreLoop_Seq.
 */
void
VMS__start_the_work_then_wait_until_done_Seq()
 {
         //Instead of un-suspending threads, just call the one and only
         // core loop (sequential version), in the main thread.
      coreLoop_Seq( NULL );
      flushRegisters();

 }
#endif

inline VirtProcr *
VMS__create_procr( VirtProcrFnPtr fnPtr, void *initialData )
 { VirtProcr *newPr;
   void      *stackLocs;

   newPr      = VMS__malloc( sizeof(VirtProcr) );
   stackLocs  = VMS__malloc( VIRT_PROCR_STACK_SIZE );
   if( stackLocs == 0 )
    { perror("VMS__malloc stack"); exit(1); }

   return create_procr_helper( newPr, fnPtr, initialData, stackLocs );
 }

/* "ext" designates that it's for use outside the VMS system -- should only
 * be called from main thread or other thread -- never from code animated by
 * a VMS virtual processor.
 */
inline VirtProcr *
VMS_ext__create_procr( VirtProcrFnPtr fnPtr, void *initialData )
 { VirtProcr *newPr;
   char      *stackLocs;

   newPr      = malloc( sizeof(VirtProcr) );
   stackLocs  = malloc( VIRT_PROCR_STACK_SIZE );
   if( stackLocs == 0 )
    { perror("malloc stack"); exit(1); }

   return create_procr_helper( newPr, fnPtr, initialData, stackLocs );
 }


/*Anticipating multi-tasking
 */
void *
VMS__give_sem_env_for( VirtProcr *animPr )
 {
   return _VMSMasterEnv->semanticEnv;
 }
//===========================================================================
/*there is a label inside this function -- save the addr of this label in
 * the callingPr struc, as the pick-up point from which to start the next
 * work-unit for that procr.  If turns out have to save registers, then
 * save them in the procr struc too.  Then do assembly jump to the CoreLoop's
 * "done with work-unit" label.  The procr struc is in the request in the
 * slave that animated the just-ended work-unit, so all the state is saved
 * there, and will get passed along, inside the request handler, to the
 * next work-unit for that procr.
 */
void
VMS__suspend_procr( VirtProcr *animatingPr )
 { 

      //The request to master will cause this suspended virt procr to get
      // scheduled again at some future point -- to resume, core loop jumps
      // to the resume point (below), which causes restore of saved regs and
      // "return" from this call.
   //animatingPr->nextInstrPt = &&ResumePt;

      //return ownership of the virt procr and sched slot to Master virt pr
   animatingPr->schedSlot->workIsDone = TRUE;

   //===========================  Measurement stuff ========================
   #ifdef MEAS__TIME_STAMP_SUSP
      //record time stamp: compare to time-stamp recorded below
   saveLowTimeStampCountInto( animatingPr->preSuspTSCLow );
   #endif
   //=======================================================================

   switchToCoreLoop(animatingPr);
   flushRegisters();

   //=======================================================================

   #ifdef MEAS__TIME_STAMP_SUSP
      //NOTE: only take low part of count -- do sanity check when take diff
   saveLowTimeStampCountInto( animatingPr->postSuspTSCLow );
   #endif

   return;
 }



/*For this implementation of VMS, it may not make much sense to have the
 * system of requests for creating a new processor done this way.. but over
 * the scope of single-master, multi-master, mult-tasking, OS-implementing,
 * distributed-memory, and so on, this gives VMS implementation a chance to
 * do stuff before suspend, in the AppVP, and in the Master before the plugin
 * is called, as well as in the lang-lib before this is called, and in the
 * plugin.  So, this gives both VMS and language implementations a chance to
 * intercept at various points and do order-dependent stuff.
 *Having a standard VMSNewPrReqData struc allows the language to create and
 * free the struc, while VMS knows how to get the newPr if it wants it, and
 * it lets the lang have lang-specific data related to creation transported
 * to the plugin.
 */
void
VMS__send_create_procr_req( void *semReqData, VirtProcr *reqstingPr )
 { VMSReqst req;

   req.reqType          = createReq;
   req.semReqData       = semReqData;
   req.nextReqst        = reqstingPr->requests;
   reqstingPr->requests = &req;

   VMS__suspend_procr( reqstingPr );
 }


/*
 *This adds a request to dissipate, then suspends the processor so that the
 * request handler will receive the request.  The request handler is what
 * does the work of freeing memory and removing the processor from the
 * semantic environment's data structures.
 *The request handler also is what figures out when to shutdown the VMS
 * system -- which causes all the core loop threads to die, and returns from
 * the call that started up VMS to perform the work.
 *
 *This form is a bit misleading to understand if one is trying to figure out
 * how VMS works -- it looks like a normal function call, but inside it
 * sends a request to the request handler and suspends the processor, which
 * jumps out of the VMS__dissipate_procr function, and out of all nestings
 * above it, transferring the work of dissipating to the request handler,
 * which then does the actual work -- causing the processor that animated
 * the call of this function to disappear and the "hanging" state of this
 * function to just poof into thin air -- the virtual processor's trace
 * never returns from this call, but instead the virtual processor's trace
 * gets suspended in this call and all the virt processor's state disap-
 * pears -- making that suspend the last thing in the virt procr's trace.
 */
void
VMS__send_dissipate_req( VirtProcr *procrToDissipate )
 { VMSReqst req;

   req.reqType                = dissipate;
   req.nextReqst              = procrToDissipate->requests;
   procrToDissipate->requests = &req;

   VMS__suspend_procr( procrToDissipate );
 }


/* "ext" designates that it's for use outside the VMS system -- should only
 * be called from main thread or other thread -- never from code animated by
 * a VMS virtual processor.
 *
 *Use this version to dissipate VPs created outside the VMS system.
 */
void
VMS_ext__dissipate_procr( VirtProcr *procrToDissipate )
 {
      //NOTE: initialData was given to the processor, so should either have
      // been alloc'd with VMS__malloc, or freed by the level above animPr.
      //So, all that's left to free here is the stack and the VirtProcr struc
      // itself
      //Note, should not stack-allocate initial data -- no guarantee, in
      // general that creating processor will outlive ones it creates.
   free( procrToDissipate->startOfStack );
   free( procrToDissipate );
 }



/*This call's name indicates that request is malloc'd -- so req handler
 * has to free any extra requests tacked on before a send, using this.
 *
 * This inserts the semantic-layer's request data into standard VMS carrier
 * request data-struct that is mallocd.  The sem request doesn't need to
 * be malloc'd if this is called inside the same call chain before the
 * send of the last request is called.
 *
 *The request handler has to call VMS__free_VMSReq for any of these
 */
inline void
VMS__add_sem_request_in_mallocd_VMSReqst( void *semReqData,
                                          VirtProcr *callingPr )
 { VMSReqst *req;

   req = VMS__malloc( sizeof(VMSReqst) );
   req->reqType         = semantic;
   req->semReqData      = semReqData;
   req->nextReqst       = callingPr->requests;
   callingPr->requests = req;
 }

/*This inserts the semantic-layer's request data into standard VMS carrier
 * request data-struct is allocated on stack of this call & ptr to it sent
 * to plugin
 *Then it does suspend, to cause request to be sent.
 */
inline void
VMS__send_sem_request( void *semReqData, VirtProcr *callingPr )
 { VMSReqst req;

   req.reqType         = semantic;
   req.semReqData      = semReqData;
   req.nextReqst       = callingPr->requests;
   callingPr->requests = &req;
   
   VMS__suspend_procr( callingPr );
 }


inline void
VMS__send_VMSSem_request( void *semReqData, VirtProcr *callingPr )
 { VMSReqst req;

   req.reqType         = VMSSemantic;
   req.semReqData      = semReqData;
   req.nextReqst       = callingPr->requests; //gab any other preceeding 
   callingPr->requests = &req;

   VMS__suspend_procr( callingPr );
 }


/*
 */
VMSReqst *
VMS__take_next_request_out_of( VirtProcr *procrWithReq )
 { VMSReqst *req;

   req = procrWithReq->requests;
   if( req == NULL ) return NULL;

   procrWithReq->requests = procrWithReq->requests->nextReqst;
   return req;
 }


inline void *
VMS__take_sem_reqst_from( VMSReqst *req )
 {
   return req->semReqData;
 }



/* This is for OS requests and VMS infrastructure requests, such as to create
 *  a probe -- a probe is inside the heart of VMS-core, it's not part of any
 *  language -- but it's also a semantic thing that's triggered from and used
 *  in the application.. so it crosses abstractions..  so, need some special
 *  pattern here for handling such requests.
 * Doing this just like it were a second language sharing VMS-core.
 * 
 * This is called from the language's request handler when it sees a request
 *  of type VMSSemReq
 *
 * TODO: Later change this, to give probes their own separate plugin & have
 *  VMS-core steer the request to appropriate plugin
 * Do the same for OS calls -- look later at it..
 */
void inline
VMS__handle_VMSSemReq( VMSReqst *req, VirtProcr *requestingPr, void *semEnv,
                       ResumePrFnPtr resumePrFnPtr )
 { VMSSemReq     *semReq;
   IntervalProbe *newProbe;

   semReq = req->semReqData;

   newProbe          = VMS__malloc( sizeof(IntervalProbe) );
   newProbe->nameStr = VMS__strDup( semReq->nameStr );
   newProbe->hist    = NULL;
   newProbe->schedChoiceWasRecorded = FALSE;

      //This runs in masterVP, so no race-condition worries
   newProbe->probeID =
             addToDynArray( newProbe, _VMSMasterEnv->dynIntervalProbesInfo );

   requestingPr->dataRetFromReq = newProbe;

   (*resumePrFnPtr)( requestingPr, semEnv );
 }



/*This must be called by the request handler plugin -- it cannot be called
 * from the semantic library "dissipate processor" function -- instead, the
 * semantic layer has to generate a request, and the plug-in calls this
 * function.
 *The reason is that this frees the virtual processor's stack -- which is
 * still in use inside semantic library calls!
 *
 *This frees or recycles all the state owned by and comprising the VMS
 * portion of the animating virtual procr.  The request handler must first
 * free any semantic data created for the processor that didn't use the
 * VMS_malloc mechanism.  Then it calls this, which first asks the malloc
 * system to disown any state that did use VMS_malloc, and then frees the
 * statck and the processor-struct itself.
 *If the dissipated processor is the sole (remaining) owner of VMS__malloc'd
 * state, then that state gets freed (or sent to recycling) as a side-effect
 * of dis-owning it.
 */
void
VMS__dissipate_procr( VirtProcr *animatingPr )
 {
      //dis-own all locations owned by this processor, causing to be freed
      // any locations that it is (was) sole owner of
//TODO: implement VMS__malloc system, including "give up ownership"


      //NOTE: initialData was given to the processor, so should either have
      // been alloc'd with VMS__malloc, or freed by the level above animPr.
      //So, all that's left to free here is the stack and the VirtProcr struc
      // itself
      //Note, should not stack-allocate initial data -- no guarantee, in
      // general that creating processor will outlive ones it creates.
   VMS__free( animatingPr->startOfStack );
   VMS__free( animatingPr );
 }


//TODO: look at architecting cleanest separation between request handler
// and master loop, for dissipate, create, shutdown, and other non-semantic
// requests.  Issue is chain: one removes requests from AppVP, one dispatches
// on type of request, and one handles each type..  but some types require
// action from both request handler and master loop -- maybe just give the
// request handler calls like:  VMS__handle_X_request_type


/*This is called by the semantic layer's request handler when it decides its
 * time to shut down the VMS system.  Calling this causes the core loop OS
 * threads to exit, which unblocks the entry-point function that started up
 * VMS, and allows it to grab the result and return to the original single-
 * threaded application.
 * 
 *The _VMSMasterEnv is needed by this shut down function, so the create-seed-
 * and-wait function has to free a bunch of stuff after it detects the
 * threads have all died: the masterEnv, the thread-related locations,
 * masterVP any AppVPs that might still be allocated and sitting in the
 * semantic environment, or have been orphaned in the _VMSWorkQ.
 * 
 *NOTE: the semantic plug-in is expected to use VMS__malloc to get all the
 * locations it needs, and give ownership to masterVP.  Then, they will be
 * automatically freed.
 *
 *In here,create one core-loop shut-down processor for each core loop and put
 * them all directly into the readyToAnimateQ.
 *Note, this function can ONLY be called after the semantic environment no
 * longer cares if AppVPs get animated after the point this is called.  In
 * other words, this can be used as an abort, or else it should only be
 * called when all AppVPs have finished dissipate requests -- only at that
 * point is it sure that all results have completed.
 */
void
VMS__shutdown()
 { int coreIdx;
   VirtProcr *shutDownPr;

      //create the shutdown processors, one for each core loop -- put them
      // directly into the Q -- each core will die when gets one
   for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
    {    //Note, this is running in the master
      shutDownPr = VMS__create_procr( &endOSThreadFn, NULL );
      writeVMSQ( shutDownPr, _VMSMasterEnv->readyToAnimateQs[coreIdx] );
    }

 }


/*Am trying to be cute, avoiding IF statement in coreLoop that checks for
 * a special shutdown procr.  Ended up with extra-complex shutdown sequence.
 *This function has the sole purpose of setting the stack and framePtr
 * to the coreLoop's stack and framePtr.. it does that then jumps to the
 * core loop's shutdown point -- might be able to just call Pthread_exit
 * from here, but am going back to the pthread's stack and setting everything
 * up just as if it never jumped out, before calling pthread_exit.
 *The end-point of core loop will free the stack and so forth of the
 * processor that animates this function, (this fn is transfering the
 * animator of the AppVP that is in turn animating this function over
 * to core loop function -- note that this slices out a level of virtual
 * processors).
 */
void
endOSThreadFn( void *initData, VirtProcr *animatingPr )
 { 
#ifdef SEQUENTIAL
    asmTerminateCoreLoopSeq(animatingPr);
#else
    asmTerminateCoreLoop(animatingPr);
#endif
 }


/*This is called from the startup & shutdown
 */
void
VMS__cleanup_at_end_of_shutdown()
 { 
   //unused
   //VMSQueueStruc **readyToAnimateQs;
   //int              coreIdx;
   //VirtProcr      **masterVPs;
   //SchedSlot     ***allSchedSlots; //ptr to array of ptrs

      //Before getting rid of everything, print out any measurements made
   forAllInDynArrayDo( _VMSMasterEnv->measHistsInfo, (DynArrayFnPtr)&printHist );
   forAllInDynArrayDo( _VMSMasterEnv->measHistsInfo, (DynArrayFnPtr)&saveHistToFile);
   //forAllInDynArrayDo( _VMSMasterEnv->measHistsInfo, &freeHistExt );
   #ifdef MEAS__TIME_PLUGIN
   printHist( _VMSMasterEnv->reqHdlrLowTimeHist );
   saveHistToFile( _VMSMasterEnv->reqHdlrLowTimeHist );
   printHist( _VMSMasterEnv->reqHdlrHighTimeHist );
   saveHistToFile( _VMSMasterEnv->reqHdlrHighTimeHist );
   freeHistExt( _VMSMasterEnv->reqHdlrLowTimeHist );
   freeHistExt( _VMSMasterEnv->reqHdlrHighTimeHist );
   #endif
   #ifdef MEAS__TIME_MALLOC
   printHist( _VMSMasterEnv->mallocTimeHist   );
   saveHistToFile( _VMSMasterEnv->mallocTimeHist   );
   printHist( _VMSMasterEnv->freeTimeHist     );
   saveHistToFile( _VMSMasterEnv->freeTimeHist     );
   freeHistExt( _VMSMasterEnv->mallocTimeHist );
   freeHistExt( _VMSMasterEnv->freeTimeHist   );
   #endif
   #ifdef MEAS__TIME_MASTER_LOCK
   printHist( _VMSMasterEnv->masterLockLowTimeHist );
   printHist( _VMSMasterEnv->masterLockHighTimeHist );
   #endif
   #ifdef MEAS__TIME_MASTER
   printHist( _VMSMasterEnv->pluginTimeHist );
   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    {
      freeVMSQ( readyToAnimateQs[ coreIdx ] );
         //master VPs were created external to VMS, so use external free
      VMS__dissipate_procr( masterVPs[ coreIdx ] );

      freeSchedSlots( allSchedSlots[ coreIdx ] );
    }
   #endif
   #ifdef MEAS__TIME_STAMP_SUSP
   printHist( _VMSMasterEnv->pluginTimeHist );
   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    {
      freeVMSQ( readyToAnimateQs[ coreIdx ] );
         //master VPs were created external to VMS, so use external free
      VMS__dissipate_procr( masterVPs[ coreIdx ] );

      freeSchedSlots( allSchedSlots[ coreIdx ] );
    }
   #endif

      //All the environment data has been allocated with VMS__malloc, so just
      // free its internal big-chunk and all inside it disappear.
/*
   readyToAnimateQs = _VMSMasterEnv->readyToAnimateQs;
   masterVPs        = _VMSMasterEnv->masterVPs;
   allSchedSlots    = _VMSMasterEnv->allSchedSlots;
   
   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    {
      freeVMSQ( readyToAnimateQs[ coreIdx ] );
         //master VPs were created external to VMS, so use external free
      VMS__dissipate_procr( masterVPs[ coreIdx ] );
      
      freeSchedSlots( allSchedSlots[ coreIdx ] );
    }
   
   VMS__free( _VMSMasterEnv->readyToAnimateQs );
   VMS__free( _VMSMasterEnv->masterVPs );
   VMS__free( _VMSMasterEnv->allSchedSlots );
   
   //============================= MEASUREMENT STUFF ========================
   #ifdef STATS__TURN_ON_PROBES
   freeDynArrayDeep( _VMSMasterEnv->dynIntervalProbesInfo, &VMS__free_probe);
   #endif
   //========================================================================
*/
      //These are the only two that use system free 
   VMS_ext__free_free_list( _VMSMasterEnv->freeListHead );
   free( (void *)_VMSMasterEnv );
 }


//================================


/*Later, improve this -- for now, just exits the application after printing
 * the error message.
 */
void
VMS__throw_exception( char *msgStr, VirtProcr *reqstPr, VMSExcp *excpData )
 {
   printf("%s",msgStr);
   fflush(stdin);
   exit(1);
 }

