1.1 --- a/CoreLoop.c	Mon Jul 26 16:42:59 2010 -0700
     1.2 +++ b/CoreLoop.c	Wed Jul 28 13:12:10 2010 -0700
     1.3 @@ -66,7 +66,7 @@
     1.4        //To get label addr in non-gcc compiler, can trick it by making a call
     1.5        // to a fn that does asm that pulls the "return"
     1.6        // addr off the stack and stores it in a pointed-to location.
     1.7 -   _VMSMasterEnv->coreLoopShutDownPt = &&CoreLoopEndPt;
     1.8 +   _VMSMasterEnv->coreLoopEndPt = &&CoreLoopEndPt;
     1.9     
    1.10        //Core loop has no values live upon CoreLoopStartPt except workQ
    1.11        // every value in the code is defined by a statement in core loop,
    1.12 @@ -146,7 +146,7 @@
    1.13        //To get label addr in non-gcc compiler, can trick it by making a call
    1.14        // to a fn that does asm that pulls the "return"
    1.15        // addr off the stack and stores it in a pointed-to location.
    1.16 -   _VMSMasterEnv->coreLoopShutDownPt = &&CoreLoopEndPt;
    1.17 +   _VMSMasterEnv->coreLoopEndPt = &&CoreLoopEndPt;
    1.18  
    1.19        //Core loop has no values live upon CoreLoopStartPt except workQ
    1.20        // every value in the code is defined by a statement in core loop,
    1.21 @@ -204,8 +204,13 @@
    1.22                  );
    1.23  
    1.24     //========================================================================
    1.25 -
    1.26 -      //jmp to here when want to shut down the VMS system
    1.27 +      //jmp to here when want to shut down the VMS system.  A shutdown VP is
    1.28 +      // made, one for each core loop -- its function sets the stack back
    1.29 +      // to the core loop's stack then jumps to here.  This causes the thread
    1.30 +      // to exit, and so the entry point function, which has been waiting for
    1.31 +      // all the threads to die will proceed, gather the result, and
    1.32 +      // return to the calling application.
    1.33     CoreLoopEndPt:
    1.34 +   VMS__handle_dissipate_reqst( currPr ); //free shutdown pr, that jmpd here
    1.35     return;
    1.36   }

     2.1 --- a/MasterLoop.c	Mon Jul 26 16:42:59 2010 -0700
     2.2 +++ b/MasterLoop.c	Wed Jul 28 13:12:10 2010 -0700
     2.3 @@ -133,9 +133,10 @@
     2.4              currSlot->needsProcrAssigned  = FALSE;
     2.5  
     2.6              filledSlots[ numFilled ]      = currSlot;
     2.7 -            numFilled += 1;
     2.8  
     2.9              writeVMSQ( schedVirtPr, workQ );
    2.10 +            numFilled += 1;
    2.11 +            
    2.12              if( numFilled == masterEnv->numToPrecede )
    2.13               {
    2.14                 writeVMSQ( masterEnv->masterVirtPr, workQ );
    2.15 @@ -158,7 +159,7 @@
    2.16  //TODO: look at dynamic behavior -- time-average numToPrecede or something
    2.17     if( numFilled < NUM_CORES - 1 )
    2.18      { 
    2.19 -      masterEnv->numToPrecede = 0;
    2.20 +      masterEnv->numToPrecede = 1;
    2.21      }
    2.22     else
    2.23      { masterEnv->numToPrecede = numFilled - NUM_CORES + 1;

     3.1 --- a/VMS.c	Mon Jul 26 16:42:59 2010 -0700
     3.2 +++ b/VMS.c	Wed Jul 28 13:12:10 2010 -0700
     3.3 @@ -389,8 +389,9 @@
     3.4   { VMSReqst *req;
     3.5  
     3.6     req = procrWithReq->requests;
     3.7 +   if( req == NULL ) return;
     3.8     procrWithReq->requests = procrWithReq->requests->nextReqst;
     3.9 -   free( req );
    3.10 +   VMS__free_request( req );
    3.11   }
    3.12  
    3.13  
    3.14 @@ -455,34 +456,6 @@
    3.15   }
    3.16  
    3.17  
    3.18 -/*The semantic layer figures out when the work is done ( perhaps by a call
    3.19 - * in the application to "work all done", or perhaps all the virtual
    3.20 - * processors have dissipated.. a.s.o. )
    3.21 - *
    3.22 - *The semantic layer is responsible for making sure all work has fully
    3.23 - * completed before using this to shutdown the VMS system.
    3.24 - *
    3.25 - *After the semantic layer has determined it wants to shut down, the
    3.26 - * next time the Master Loop calls the scheduler plug-in, the scheduler
    3.27 - * then calls this function and returns the virtual processor it gets back.
    3.28 - *
    3.29 - *When the shut-down processor runs, it first frees all locations malloc'd to
    3.30 - * the VMS system (that wasn't
    3.31 - * specified as return-locations).  Then it creates one core-loop shut-down
    3.32 - * processor for each core loop and puts them all into the workQ.  When a
    3.33 - * core loop animates a core loop shut-down processor, it causes exit-thread
    3.34 - * to run, and when all core loop threads have exited, then the "wait for
    3.35 - * work to finish" in the main thread is woken, and the function-call that
    3.36 - * started all the work returns.
    3.37 - *
    3.38 - *The function animated by this processor performs the shut-down work.
    3.39 - */
    3.40 -VirtProcr *
    3.41 -VMS__create_the_shutdown_procr()
    3.42 - {
    3.43 -   return VMS__create_procr( &shutdownFn, NULL );
    3.44 - }
    3.45 -
    3.46  
    3.47  /*This must be called by the request handler plugin -- it cannot be called
    3.48   * from the semantic library "dissipate processor" function -- instead, the
    3.49 @@ -502,15 +475,14 @@
    3.50   * of dis-owning it.
    3.51   */
    3.52  void
    3.53 -VMS__free_procr_locs( VirtProcr *animatingPr )
    3.54 +VMS__handle_dissipate_reqst( VirtProcr *animatingPr )
    3.55   {
    3.56        //dis-own all locations owned by this processor, causing to be freed
    3.57        // any locations that it is (was) sole owner of
    3.58 -   //TODO: implement VMS__malloc system, including "give up ownership"
    3.59 +//TODO: implement VMS__malloc system, including "give up ownership"
    3.60  
    3.61        //The dissipate request might still be attached, so remove and free it
    3.62     VMS__remove_and_free_top_request( animatingPr );
    3.63 -   free( animatingPr->startOfStack );
    3.64  
    3.65        //NOTE: initialData was given to the processor, so should either have
    3.66        // been alloc'd with VMS__malloc, or freed by the level above animPr.
    3.67 @@ -521,54 +493,98 @@
    3.68   }
    3.69  
    3.70  
    3.71 +//TODO: re-architect so that have clean separation between request handler
    3.72 +// and master loop, for dissipate, create, shutdown, and other non-semantic
    3.73 +// requests.  Issue is chain: one removes requests from AppVP, one dispatches
    3.74 +// on type of request, and one handles each type..  but some types require
    3.75 +// action from both request handler and master loop -- maybe just give the
    3.76 +// request handler calls like:  VMS__handle_X_request_type
    3.77  
    3.78 -/*This is the function run by the special "shut-down" processor
    3.79 +void
    3.80 +endOSThreadFn( void *initData, VirtProcr *animatingPr );
    3.81 +
    3.82 +/*This is called by the semantic layer's request handler when it decides its
    3.83 + * time to shut down the VMS system.  Calling this causes the core loop OS
    3.84 + * threads to exit, which unblocks the entry-point function that started up
    3.85 + * VMS, and allows it to grab the result and return to the original single-
    3.86 + * threaded application.
    3.87   * 
    3.88 - *The _VMSMasterEnv is needed by this shut down function, so the "wait"
    3.89 - * function run in the main loop has to free it, and the thread-related
    3.90 - * locations (coreLoopThdParams a.s.o.).
    3.91 - *However, the semantic environment and all data malloc'd to VMS can be
    3.92 - * freed here.
    3.93 + *The _VMSMasterEnv is needed by this shut down function, so the create-seed-
    3.94 + * and-wait function has to free a bunch of stuff after it detects the
    3.95 + * threads have all died: the masterEnv, the thread-related locations,
    3.96 + * masterVP any AppVPs that might still be allocated and sitting in the
    3.97 + * semantic environment, or have been orphaned in the _VMSWorkQ.
    3.98 + * 
    3.99 + *NOTE: the semantic plug-in is expected to use VMS__malloc_to to get all the
   3.100 + * locations it needs, and give ownership to masterVP.  Then, they will be
   3.101 + * automatically freed when the masterVP is dissipated.  (This happens after
   3.102 + * the core loop threads have all exited)
   3.103   *
   3.104 - *NOTE: the semantic plug-in is expected to use VMS__malloc to get all the
   3.105 - * locations it needs -- they will be automatically freed by the standard
   3.106 - * "free all owned locations"
   3.107 - *
   3.108 - *Free any locations malloc'd to the VMS system (that weren't
   3.109 - * specified as return-locations).
   3.110 - *Then create one core-loop shut-down processor for each core loop and puts
   3.111 - * them all into the workQ.
   3.112 + *In here,create one core-loop shut-down processor for each core loop and put
   3.113 + * them all directly into the workQ.
   3.114 + *Note, this function can ONLY be called after the semantic environment no
   3.115 + * longer cares if AppVPs get animated after the point this is called.  In
   3.116 + * other words, this can be used as an abort, or else it should only be
   3.117 + * called when all AppVPs have finished dissipate requests -- only at that
   3.118 + * point is it sure that all results have completed.
   3.119   */
   3.120  void
   3.121 -shutdownFn( void *dummy, VirtProcr *animatingPr )
   3.122 +VMS__handle_shutdown_reqst( void *dummy, VirtProcr *animatingPr )
   3.123   { int coreIdx;
   3.124     VirtProcr *shutDownPr;
   3.125     VMSQueueStruc *workQ = _VMSWorkQ;
   3.126  
   3.127 -      //free all the locations owned within the VMS system
   3.128 -   //TODO: write VMS__malloc and free.. -- take the DKU malloc as starting pt
   3.129 -
   3.130 -      //make the core loop shut-down processors and put them into the workQ
   3.131 +      //create the shutdown processors, one for each core loop -- put them
   3.132 +      // directly into _VMSWorkQ -- each core will die when gets one, so
   3.133 +      // the system distributes them evenly itself.
   3.134     for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
   3.135      {
   3.136 -      shutDownPr = VMS__create_procr( NULL, NULL );
   3.137 -      shutDownPr->nextInstrPt = _VMSMasterEnv->coreLoopShutDownPt;
   3.138 +      shutDownPr = VMS__create_procr( &endOSThreadFn, NULL );
   3.139        writeVMSQ( shutDownPr, workQ );
   3.140      }
   3.141  
   3.142 -      //This is an issue: the animating processor of this function may not
   3.143 -      // get its request handled before all the cores have shutdown.
   3.144 -      //TODO: after all the threads stop, clean out the MasterEnv, the
   3.145 -      // SemanticEnv, and the workQ before returning.
   3.146 -   VMS__dissipate_procr( animatingPr );  //will never come back from this
   3.147   }
   3.148  
   3.149  
   3.150 -/*This has to free anything allocated during VMS_init, and any other alloc'd
   3.151 +/*Am trying to be cute, avoiding IF statement in coreLoop that checks for
   3.152 + * a special shutdown procr.  Ended up with extra-complex shutdown sequence.
   3.153 + *This function has the sole purpose of setting the stack and framePtr
   3.154 + * to the coreLoop's stack and framePtr.. it does that then jumps to the
   3.155 + * core loop's shutdown point -- might be able to just call Pthread_exit
   3.156 + * from here, but going back to the pthread's stack and setting everything
   3.157 + * up just as if it never jumped out, before calling pthread_exit.
   3.158 + *The end-point of core loop will free the stack and so forth of the
   3.159 + * processor that animates this function, (this fn is transfering the
   3.160 + * animator of the AppVP that is in turn animating this function over
   3.161 + * to core loop function -- note that this slices out a level of virtual
   3.162 + * processors).
   3.163 + */
   3.164 +void
   3.165 +endOSThreadFn( void *initData, VirtProcr *animatingPr )
   3.166 + { void *jmpPt, *coreLoopStackPtr, *coreLoopFramePtr;
   3.167 +
   3.168 +   jmpPt             = _VMSMasterEnv->coreLoopEndPt;
   3.169 +   coreLoopStackPtr  = animatingPr->coreLoopStackPtr;
   3.170 +   coreLoopFramePtr  = animatingPr->coreLoopFramePtr;
   3.171 +
   3.172 +
   3.173 +   asm volatile("movl %0, %%eax;      \
   3.174 +                 movl %1, %%esp;      \
   3.175 +                 movl %2, %%ebp;      \
   3.176 +                 jmp  %%eax    "      \
   3.177 +   /* outputs */ :                    \
   3.178 +   /* inputs  */ : "m" (jmpPt), "m"(coreLoopStackPtr), "m"(coreLoopFramePtr)\
   3.179 +   /* clobber */ : "memory", "%eax", "%ebx", "%ecx", "%edx", "%edi","%esi"  \
   3.180 +                );
   3.181 + }
   3.182 +
   3.183 +
   3.184 +
   3.185 +/*This is called has to free anything allocated during VMS_init, and any other alloc'd
   3.186   * locations that might be left over.
   3.187   */
   3.188  void
   3.189 -VMS__shutdown()
   3.190 +VMS__cleanup_after_shutdown()
   3.191   { int i;
   3.192   
   3.193     free( _VMSWorkQ );
   3.194 @@ -579,7 +595,7 @@
   3.195      }
   3.196  
   3.197     free( _VMSMasterEnv->schedSlots);
   3.198 -   VMS__free_procr_locs( _VMSMasterEnv->masterVirtPr );
   3.199 +   VMS__handle_dissipate_reqst( _VMSMasterEnv->masterVirtPr );
   3.200     
   3.201     free( _VMSMasterEnv );
   3.202   }

     4.1 --- a/VMS.h	Mon Jul 26 16:42:59 2010 -0700
     4.2 +++ b/VMS.h	Wed Jul 28 13:12:10 2010 -0700
     4.3 @@ -19,7 +19,8 @@
     4.4  #define NUM_CORES        4
     4.5  
     4.6     // make double-num-cores scheduling slots, plus extra for master
     4.7 -#define NUM_SCHED_SLOTS  (2 * NUM_CORES + 1)
     4.8 +//#define NUM_SCHED_SLOTS  (2 * NUM_CORES + 1)
     4.9 +#define NUM_SCHED_SLOTS  3
    4.10  
    4.11     //128K stack.. compromise, want 10K virtPr
    4.12  #define VIRT_PROCR_STACK_SIZE 0x10000
    4.13 @@ -117,7 +118,7 @@
    4.14     void       *semanticEnv;
    4.15     void       *OSEventStruc;    //for future, when add I/O to BLIS
    4.16  
    4.17 -   void       *coreLoopShutDownPt; //addr to jump to to shut down a coreLoop
    4.18 +   void       *coreLoopEndPt; //addr to jump to to shut down a coreLoop
    4.19  
    4.20     int         setupComplete;
    4.21   }
    4.22 @@ -203,7 +204,10 @@
    4.23  VMS__dissipate_procr( VirtProcr *prToDissipate );
    4.24  
    4.25  void
    4.26 -VMS__shutdown();
    4.27 +VMS__handle_dissipate_reqst( VirtProcr *procrToDissipate );
    4.28 +
    4.29 +void
    4.30 +VMS__cleanup_after_shutdown();
    4.31  
    4.32  //============================= Statistics ==================================
    4.33