1.1 --- a/VSs.c	Fri Jul 13 17:35:49 2012 +0200
     1.2 +++ b/VSs.c	Wed Aug 01 00:18:53 2012 -0700
     1.3 @@ -73,8 +73,10 @@
     1.4   */
     1.5  void
     1.6  VSs__create_seed_slave_and_do_work( TopLevelFnPtr fnPtr, void *initData )
     1.7 - { VSsSemEnv *semEnv;
     1.8 -   SlaveVP *seedSlv;
     1.9 + { VSsSemEnv   *semEnv;
    1.10 +   SlaveVP     *seedSlv;
    1.11 +   VSsSemData  *semData;
    1.12 +   VSsTaskStub *explPrTaskStub;
    1.13  
    1.14     VSs__init();      //normal multi-thd
    1.15     
    1.16 @@ -83,12 +85,18 @@
    1.17        //VSs starts with one processor, which is put into initial environ,
    1.18        // and which then calls create() to create more, thereby expanding work
    1.19     seedSlv = VSs__create_slave_helper( fnPtr, initData,
    1.20 -                                      semEnv, semEnv->nextCoreToGetNewSlv++ );
    1.21 +                                     semEnv, semEnv->nextCoreToGetNewSlv++ );
    1.22     
    1.23 -      //seedVP doesn't do tasks
    1.24 -   ((VSsSemData *)seedSlv->semanticData)->needsTaskAssigned = FALSE;
    1.25 +      //seed slave is an explicit processor, so make one of the special
    1.26 +      // task stubs for explicit processors, and attach it to the slave
    1.27 +   explPrTaskStub = create_expl_proc_task_stub( initData );
    1.28 +   
    1.29 +   semData = (VSsSemData *)seedSlv->semanticData;
    1.30 +      //seedVP already has a permanent task
    1.31 +   semData->needsTaskAssigned = FALSE;
    1.32 +   semData->taskStub = explPrTaskStub;
    1.33  
    1.34 -   resume_slaveVP( seedSlv, semEnv );
    1.35 +   resume_slaveVP( seedSlv, semEnv ); //returns right away, just queues Slv
    1.36     
    1.37     VMS_SS__start_the_work_then_wait_until_done();      //normal multi-thd
    1.38  
    1.39 @@ -169,8 +177,7 @@
    1.40  void
    1.41  VSs__init_Helper()
    1.42   { VSsSemEnv       *semanticEnv;
    1.43 -   PrivQueueStruc **readyVPQs;
    1.44 -   int              coreIdx, i, j;
    1.45 +   int32            i, coreNum, slotNum;
    1.46   
    1.47        //Hook up the semantic layer's plug-ins to the Master virt procr
    1.48     _VMSMasterEnv->requestHandler = &VSs__Request_Handler;
    1.49 @@ -190,43 +197,35 @@
    1.50  
    1.51     semanticEnv->shutdownInitiated = FALSE;
    1.52     semanticEnv->coreIsDone = VMS_int__malloc( NUM_CORES * sizeof( bool32 ) );
    1.53 -   for( i = 0; i < NUM_CORES; ++i )
    1.54 -    { semanticEnv->coreIsDone[i] = FALSE;
    1.55 -      for( j = 0; j < NUM_ANIM_SLOTS; ++j )
    1.56 -       {
    1.57 -         semanticEnv->idleSlv[i][j] = VMS_int__create_slaveVP(&idle_fn,NULL);
    1.58 -         semanticEnv->idleSlv[i][j]->coreAnimatedBy = i;
    1.59 +      //For each animation slot, there is an idle slave, and an initial
    1.60 +      // slave assigned as the current-task-slave.  Create them here.
    1.61 +   SlaveVP *idleSlv, *currTaskSlv;
    1.62 +   for( coreNum = 0; coreNum < NUM_CORES; coreNum++ )
    1.63 +    { semanticEnv->coreIsDone[coreNum] = FALSE; //use during shutdown
    1.64 +    
    1.65 +      for( slotNum = 0; slotNum < NUM_ANIM_SLOTS; ++slotNum )
    1.66 +       { idleSlv = VMS_int__create_slaveVP(&idle_fn,NULL);
    1.67 +         idleSlv->coreAnimatedBy                = coreNum;
    1.68 +         idleSlv->animSlotAssignedTo            = slotNum;
    1.69 +         semanticEnv->idleSlv[coreNum][slotNum] = idleSlv;
    1.70 +         
    1.71 +         currTaskSlv = VMS_int__create_slaveVP( &idle_fn, NULL );
    1.72 +         currTaskSlv->coreAnimatedBy                 = coreNum;
    1.73 +         currTaskSlv->animSlotAssignedTo             = slotNum;
    1.74 +         semanticEnv->currTaskSlvs[coreNum][slotNum] = currTaskSlv;
    1.75         }
    1.76      }
    1.77  
    1.78 -   #ifdef HOLISTIC__TURN_ON_OBSERVE_UCC
    1.79 -   semanticEnv->unitList = makeListOfArrays(sizeof(Unit),128);
    1.80 -   semanticEnv->ctlDependenciesList = makeListOfArrays(sizeof(Dependency),128);
    1.81 -   semanticEnv->commDependenciesList = makeListOfArrays(sizeof(Dependency),128);
    1.82 -   semanticEnv->dynDependenciesList = makeListOfArrays(sizeof(Dependency),128);
    1.83 -   semanticEnv->ntonGroupsInfo = makePrivDynArrayOfSize((void***)&(semanticEnv->ntonGroups),8);
    1.84 -   
    1.85 -   semanticEnv->hwArcs = makeListOfArrays(sizeof(Dependency),128);
    1.86 -   memset(semanticEnv->last_in_slot,0,sizeof(NUM_CORES * NUM_ANIM_SLOTS * sizeof(Unit)));
    1.87 -   #endif
    1.88 -
    1.89 -      //create the ready queue, hash tables used for matching and so forth
    1.90 -   readyVPQs = VMS_int__malloc( NUM_CORES * sizeof(PrivQueueStruc *) );
    1.91 -
    1.92 -   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
    1.93 -    {
    1.94 -      readyVPQs[ coreIdx ] = makeVMSQ();
    1.95 -    }
    1.96 -   
    1.97 -   semanticEnv->readyVPQs = readyVPQs;
    1.98 -   
    1.99 -   semanticEnv->taskReadyQ = makeVMSQ();
   1.100 -   
   1.101 -   semanticEnv->nextCoreToGetNewSlv = 0;
   1.102 -   semanticEnv->numSlaveVP = 0;
   1.103 +      //create the ready queues, hash tables used for matching and so forth
   1.104 +   semanticEnv->slavesReadyToResumeQ = makeVMSQ();
   1.105 +   semanticEnv->extraTaskSlvQ        = makeVMSQ();
   1.106 +   semanticEnv->taskReadyQ           = makeVMSQ();
   1.107     
   1.108     semanticEnv->argPtrHashTbl  = makeHashTable32( 16, &VMS_int__free );
   1.109     semanticEnv->commHashTbl    = makeHashTable32( 16, &VMS_int__free );
   1.110 +   
   1.111 +   semanticEnv->nextCoreToGetNewSlv = 0;
   1.112 +   
   1.113  
   1.114     //TODO: bug -- turn these arrays into dyn arrays to eliminate limit
   1.115     //semanticEnv->singletonHasBeenExecutedFlags = makeDynArrayInfo( );
   1.116 @@ -239,6 +238,19 @@
   1.117        semanticEnv->fnSingletons[i].waitQ             = makeVMSQ();
   1.118        semanticEnv->transactionStrucs[i].waitingVPQ   = makeVMSQ();
   1.119      }
   1.120 +
   1.121 +   semanticEnv->numAdditionalSlvs   = 0; //must be last
   1.122 +
   1.123 +   #ifdef HOLISTIC__TURN_ON_OBSERVE_UCC
   1.124 +   semanticEnv->unitList = makeListOfArrays(sizeof(Unit),128);
   1.125 +   semanticEnv->ctlDependenciesList = makeListOfArrays(sizeof(Dependency),128);
   1.126 +   semanticEnv->commDependenciesList = makeListOfArrays(sizeof(Dependency),128);
   1.127 +   semanticEnv->dynDependenciesList = makeListOfArrays(sizeof(Dependency),128);
   1.128 +   semanticEnv->ntonGroupsInfo = makePrivDynArrayOfSize((void***)&(semanticEnv->ntonGroups),8);
   1.129 +   
   1.130 +   semanticEnv->hwArcs = makeListOfArrays(sizeof(Dependency),128);
   1.131 +   memset(semanticEnv->last_in_slot,0,sizeof(NUM_CORES * NUM_ANIM_SLOTS * sizeof(Unit)));
   1.132 +   #endif
   1.133   }
   1.134  
   1.135  

     2.1 --- a/VSs.h	Fri Jul 13 17:35:49 2012 +0200
     2.2 +++ b/VSs.h	Wed Aug 01 00:18:53 2012 -0700
     2.3 @@ -61,9 +61,7 @@
     2.4   }
     2.5  VSsPointerEntry;
     2.6  
     2.7 -typedef struct _VSsTaskStub VSsTaskStub;
     2.8 -
     2.9 -struct _VSsTaskStub
    2.10 +typedef struct
    2.11   {
    2.12     void       **args; //ctld args must come first, as ptrs
    2.13     VSsTaskType *taskType;
    2.14 @@ -74,17 +72,19 @@
    2.15     void*        parent;
    2.16     bool32       parentIsTask;
    2.17     int32        numChildTasks;
    2.18 -   bool32         isWaiting;
    2.19 +   bool32       isWaiting;
    2.20   }
    2.21 -;
    2.22 +VSsTaskStub;
    2.23  
    2.24 -typedef struct {
    2.25 +typedef struct 
    2.26 + {
    2.27      void* parent;
    2.28      bool32       parentIsTask;
    2.29      int32        numChildTasks;
    2.30      bool32       isWaiting;
    2.31     SlaveVP     *slaveAssignedTo;    
    2.32 -} VSsThreadInfo;
    2.33 + }
    2.34 +VSsThreadInfo;
    2.35  
    2.36  typedef struct
    2.37   {
    2.38 @@ -185,11 +185,13 @@
    2.39  
    2.40  typedef struct
    2.41   {
    2.42 -   PrivQueueStruc **readyVPQs;
    2.43 -   PrivQueueStruc  *taskReadyQ;  //Q: shared or local?
    2.44 +   PrivQueueStruc **slavesReadyToResumeQ; //Shared (slaves not pinned)
    2.45 +   PrivQueueStruc **extraTaskSlvQ;     //Shared
    2.46 +   PrivQueueStruc  *taskReadyQ;        //Shared (tasks not pinned)
    2.47 +   SlaveVP         *currTaskSlvs[NUM_CORES][NUM_ANIM_SLOTS];
    2.48     HashTable       *argPtrHashTbl;
    2.49     HashTable       *commHashTbl;
    2.50 -   int32            numSlaveVP;
    2.51 +   int32            numAdditionalSlvs;
    2.52     int32            nextCoreToGetNewSlv;
    2.53     int32            primitiveStartTime;
    2.54  

     3.1 --- a/VSs_PluginFns.c	Fri Jul 13 17:35:49 2012 +0200
     3.2 +++ b/VSs_PluginFns.c	Wed Aug 01 00:18:53 2012 -0700
     3.3 @@ -15,26 +15,42 @@
     3.4  void
     3.5  resume_slaveVP( SlaveVP *slave, VSsSemEnv *semEnv );
     3.6  
     3.7 -void
     3.8 +inline void
     3.9  handleSemReq( VMSReqst *req, SlaveVP *requestingSlv, VSsSemEnv *semEnv );
    3.10  
    3.11 -void
    3.12 +inline void
    3.13  handleDissipate(                SlaveVP *requestingSlv, VSsSemEnv *semEnv );
    3.14  
    3.15 -void
    3.16 +inline void
    3.17  handleCreate(    VMSReqst *req, SlaveVP *requestingSlv, VSsSemEnv *semEnv );
    3.18  
    3.19  //============================== Assigner ==================================
    3.20  //
    3.21 -/*For VSs, assigning a slave simply takes the next work-unit off the
    3.22 - * ready-to-go work-unit queue and assigns it to the offered slot.
    3.23 - *If the ready-to-go work-unit queue is empty, then nothing to assign
    3.24 - * to the animation slot -- return FALSE to let Master loop know assigning
    3.25 - * that slot failed.
    3.26 +/*The assigner is complicated by having both tasks and explicitly created
    3.27 + * VPs, and by tasks being able to suspend.
    3.28 + *It can't use an explicit slave to animate a task because of stack
    3.29 + * pollution. So, it has to keep the two kinds separate.
    3.30 + *Simplest way for the assigner logic is with a Q for extra empty task
    3.31 + * slaves, and another Q for slaves of both types that are ready to resume.
    3.32 + *
    3.33 + *Keep a current task slave for each anim slot. The request handler manages
    3.34 + * it by pulling from the extraTaskSlvQ when a task suspends, or else
    3.35 + * creating a new task slave if taskSlvQ empty. 
    3.36 + *Assigner only assigns a task to the current task slave for the slot.
    3.37 + *If no more tasks, then takes a ready to resume slave, if also none of them
    3.38 + * then dissipates extra task slaves (one per invocation).
    3.39 + *Shutdown condition is: must have no suspended tasks, and no suspended
    3.40 + * explicit slaves and no more tasks in taskQ.  Will only have the masters
    3.41 + * plus a current task slave for each slot.. detects this condition. 
    3.42 + * 
    3.43 + *Having the two types of slave is part of having communications directly
    3.44 + * between tasks, and tasks to explicit slaves, which requires the ability
    3.45 + * to suspend both kinds, but also to keep explicit slave stacks clean from
    3.46 + * the junk tasks are allowed to leave behind.
    3.47   */
    3.48  SlaveVP *
    3.49  VSs__assign_slaveVP_to_slot( void *_semEnv, AnimSlot *slot )
    3.50 - { SlaveVP    *assignSlv;
    3.51 + { SlaveVP    *returnSlv;
    3.52     VSsSemEnv  *semEnv;
    3.53     VSsSemData *semData;
    3.54     int32       coreNum, slotNum;
    3.55 @@ -44,111 +60,113 @@
    3.56     
    3.57     semEnv  = (VSsSemEnv *)_semEnv;
    3.58  
    3.59 -   /*At this point, could do an optimization -- have one slave for each slot
    3.60 -    * and make it ALWAYS the one to assign to that slot -- so there is no
    3.61 -    * read fromQ.  However, going to keep this compatible with other
    3.62 -    * languages, like VOMP and SSR.  So, leave the normal slave fetch
    3.63 -    * from readyQ. For example, allows SSR constructs, to create extra
    3.64 -    * slaves, and send communications direction between them, while still
    3.65 -    * having the StarSs-style spawning of tasks..  so one of the tasks
    3.66 -    * can now suspend and do more interesting things..  means keep a pool
    3.67 -    * of slaves, and take one from pool when a task suspends.
    3.68 -    */
    3.69 -   //TODO: fix false sharing in array
    3.70 -   assignSlv = readPrivQ( semEnv->readyVPQs[coreNum] );
    3.71 -   if( assignSlv == NULL )
    3.72 +   
    3.73 +   //Speculatively set the return slave to the current taskSlave
    3.74 +   //TODO: false sharing ?  Always read..
    3.75 +   returnSlv = semEnv->currTaskSlvs[coreNum][slotNum];
    3.76 +
    3.77 +/* request handlers do this now..  move it to there..
    3.78 +   if( returnSlv == NULL )
    3.79      {    //make a new slave to animate
    3.80           //This happens for the first task on the core and when all available
    3.81 -         // slaves are blocked by constructs like send, or mutex, and so on..
    3.82 -      assignSlv = VSs__create_slave_helper( NULL, NULL, semEnv, coreNum );
    3.83 +         //slaves are blocked by constructs like send, or mutex, and so on.
    3.84 +      returnSlv = VSs__create_slave_helper( NULL, NULL, semEnv, coreNum );
    3.85      }
    3.86 -   semData = (VSsSemData *)assignSlv->semanticData;
    3.87 -      //slave could be resuming a task in progress, check for this
    3.88 -   if( semData->needsTaskAssigned )
    3.89 -    {    //no, not resuming, needs a task..
    3.90 -      VSsTaskStub *newTaskStub;
    3.91 -      SlaveVP *extraSlv;
    3.92 -      newTaskStub = readPrivQ( semEnv->taskReadyQ );
    3.93 -      if( newTaskStub == NULL )
    3.94 -       { //No task, so slave unused, so put it back and return "no-slave"
    3.95 -         //But first check if have extra free slaves
    3.96 -         extraSlv = readPrivQ( semEnv->readyVPQs[coreNum] );
    3.97 -         if( extraSlv == NULL )
    3.98 -          {    //means no tasks and no slave on this core can generate more
    3.99 -            //TODO: false sharing
   3.100 -            if( semEnv->coreIsDone[coreNum] == FALSE)
   3.101 + */ 
   3.102 +   semData = (VSsSemData *)returnSlv->semanticData;
   3.103 +
   3.104 +      //There is always a curr task slave, and it always needs a task
   3.105 +      //(task slaves that are resuming are in resumeQ)
   3.106 +   VSsTaskStub *newTaskStub;
   3.107 +   SlaveVP *extraSlv;
   3.108 +   newTaskStub = readPrivQ( semEnv->taskReadyQ );
   3.109 +   if( newTaskStub != NULL )
   3.110 +    { //point slave to task's function, and mark slave as having task
   3.111 +      VMS_int__reset_slaveVP_to_TopLvlFn( returnSlv, 
   3.112 +                          newTaskStub->taskType->fn, newTaskStub->args );
   3.113 +      semData->taskStub            = newTaskStub;
   3.114 +      newTaskStub->slaveAssignedTo = returnSlv;
   3.115 +      semData->needsTaskAssigned   = FALSE;
   3.116 +      if( semEnv->coreIsDone[coreNum] == TRUE ) //reads are higher perf
   3.117 +         semEnv->coreIsDone[coreNum] = FALSE;   //don't just write always
   3.118 +      goto ReturnTheSlv;
   3.119 +    }
   3.120 +   else
   3.121 +    { //no task, so try to get a ready to resume slave
   3.122 +      returnSlv = readPrivQ( semEnv->slavesReadyToResumeQ );
   3.123 +      if( returnSlv != NULL )  //Yes, have a slave, so return it.
   3.124 +       { returnSlv->coreAnimatedBy   = coreNum;
   3.125 +         if( semEnv->coreIsDone[coreNum] == TRUE ) //reads are higher perf
   3.126 +            semEnv->coreIsDone[coreNum] = FALSE;   //don't just write always
   3.127 +         goto ReturnTheSlv;
   3.128 +       }
   3.129 +      //If get here, then no task, so check if have extra free slaves
   3.130 +      extraSlv = readPrivQ( semEnv->extraTaskSlvQ );
   3.131 +      if( extraSlv != NULL )
   3.132 +       {    //means have two slaves need tasks -- redundant, kill one
   3.133 +         handleDissipate( extraSlv, semEnv );
   3.134 +            //then return NULL
   3.135 +         returnSlv = NULL;
   3.136 +         goto ReturnTheSlv;
   3.137 +       }
   3.138 +      else
   3.139 +       { //candidate for shutdown.. if all extras dissipated, and no tasks
   3.140 +         // and no ready to resume slaves, then then no way to generate
   3.141 +         // more tasks..
   3.142 +         if( semEnv->numAdditionalSlvs == 0 ) //means none suspended
   3.143 +          { //This core sees no way to generate more tasks, so say it
   3.144 +            if( semEnv->coreIsDone[coreNum] == FALSE )
   3.145               { semEnv->numCoresDone += 1;
   3.146                 semEnv->coreIsDone[coreNum] = TRUE;
   3.147                 #ifdef DEBUG__TURN_ON_SEQUENTIAL_MODE
   3.148                 semEnv->shutdownInitiated = TRUE;
   3.149                 #else
   3.150                 if( semEnv->numCoresDone == NUM_CORES )
   3.151 -                {    //means no cores have work, and none can generate more
   3.152 +                { //means no cores have work, and none can generate more
   3.153                    semEnv->shutdownInitiated = TRUE;
   3.154                  }
   3.155                 #endif
   3.156               }
   3.157 -               //put slave back into Q and return NULL
   3.158 -            writePrivQ( assignSlv, semEnv->readyVPQs[coreNum] );
   3.159 -            assignSlv = NULL;
   3.160 -               //except if shutdown has been initiated by this or other core
   3.161 -            if(semEnv->shutdownInitiated) 
   3.162 -             { assignSlv = VMS_SS__create_shutdown_slave();
   3.163 -             }
   3.164            }
   3.165 -         else //extra slave exists, but no tasks for either slave
   3.166 -          { if(((VSsSemData *)extraSlv->semanticData)->needsTaskAssigned == TRUE)
   3.167 -             {    //means have two slaves need tasks -- redundant, kill one
   3.168 -               handleDissipate( extraSlv, semEnv );
   3.169 -                  //then put other back into Q and return NULL
   3.170 -               writePrivQ( assignSlv, semEnv->readyVPQs[coreNum] );
   3.171 -               assignSlv = NULL;
   3.172 -             }
   3.173 -            else
   3.174 -             {    //extra slave has work -- so take it instead
   3.175 -               writePrivQ( assignSlv, semEnv->readyVPQs[coreNum] );
   3.176 -               assignSlv = extraSlv;
   3.177 -               //semData = (VSsSemData *)assignSlv->semanticData; Don't use
   3.178 -             }
   3.179 +            //return NULL.. no task and none to resume
   3.180 +         returnSlv = NULL;
   3.181 +            //except if shutdown has been initiated by this or other core
   3.182 +         if(semEnv->shutdownInitiated) 
   3.183 +          { returnSlv = VMS_SS__create_shutdown_slave();
   3.184            }
   3.185 -       }
   3.186 -      else //have a new task for the slave.
   3.187 -       { //point slave to task's function, and mark slave as having task
   3.188 -         VMS_int__reset_slaveVP_to_TopLvlFn( assignSlv, 
   3.189 -                             newTaskStub->taskType->fn, newTaskStub->args );
   3.190 -         semData->taskStub = newTaskStub;
   3.191 -         newTaskStub->slaveAssignedTo = assignSlv;
   3.192 -         semData->needsTaskAssigned = FALSE;
   3.193 -       }
   3.194 -    } //outcome: 1)slave didn't need a new task 2)slave just pointed at one
   3.195 -      //         3)no tasks, so slave NULL
   3.196 -   
   3.197 +         goto ReturnTheSlv; //don't need, but completes pattern
   3.198 +       } //if( extraSlv != NULL )
   3.199 +    } //if( newTaskStub == NULL )
   3.200 +   //outcome: 1)slave was just pointed to task, 2)no tasks, so slave NULL
   3.201 +
   3.202 +ReturnTheSlv:  //Nina, doing gotos to here should help with holistic..
   3.203 +
   3.204     #ifdef HOLISTIC__TURN_ON_OBSERVE_UCC
   3.205 -   if( assignSlv == NULL )
   3.206 -    { assignSlv = semEnv->idleSlv[coreNum][slotNum]; 
   3.207 +   if( returnSlv == NULL )
   3.208 +    { returnSlv = semEnv->idleSlv[coreNum][slotNum]; 
   3.209        if(semEnv->shutdownInitiated) 
   3.210 -       { assignSlv = VMS_SS__create_shutdown_slave();
   3.211 +       { returnSlv = VMS_SS__create_shutdown_slave();
   3.212         }
   3.213           //things that would normally happen in resume(), but these VPs
   3.214           // never go there
   3.215 -      assignSlv->assignCount++; //Somewhere here!
   3.216 +      returnSlv->assignCount++; //Somewhere here!
   3.217        Unit newu;
   3.218 -      newu.vp = assignSlv->slaveID;
   3.219 -      newu.task = assignSlv->assignCount;
   3.220 +      newu.vp = returnSlv->slaveID;
   3.221 +      newu.task = returnSlv->assignCount;
   3.222        addToListOfArrays(Unit,newu,semEnv->unitList);
   3.223  
   3.224 -      if (assignSlv->assignCount > 1)
   3.225 +      if (returnSlv->assignCount > 1)
   3.226         { Dependency newd;
   3.227 -         newd.from_vp = assignSlv->slaveID;
   3.228 -         newd.from_task = assignSlv->assignCount - 1;
   3.229 -         newd.to_vp = assignSlv->slaveID;
   3.230 -         newd.to_task = assignSlv->assignCount;
   3.231 +         newd.from_vp = returnSlv->slaveID;
   3.232 +         newd.from_task = returnSlv->assignCount - 1;
   3.233 +         newd.to_vp = returnSlv->slaveID;
   3.234 +         newd.to_task = returnSlv->assignCount;
   3.235           addToListOfArrays(Dependency, newd ,semEnv->ctlDependenciesList);  
   3.236         }
   3.237      }
   3.238     #endif
   3.239     #ifdef HOLISTIC__TURN_ON_OBSERVE_UCC
   3.240 -   if( assignSlv != NULL )
   3.241 +   if( returnSlv != NULL )
   3.242      { //assignSlv->numTimesAssigned++;
   3.243        Unit prev_in_slot = 
   3.244           semEnv->last_in_slot[coreNum * NUM_ANIM_SLOTS + slotNum];
   3.245 @@ -156,23 +174,24 @@
   3.246         { Dependency newd;
   3.247           newd.from_vp = prev_in_slot.vp;
   3.248           newd.from_task = prev_in_slot.task;
   3.249 -         newd.to_vp = assignSlv->slaveID;
   3.250 -         newd.to_task = assignSlv->assignCount;
   3.251 +         newd.to_vp = returnSlv->slaveID;
   3.252 +         newd.to_task = returnSlv->assignCount;
   3.253           addToListOfArrays(Dependency,newd,semEnv->hwArcs);   
   3.254         }
   3.255 -      prev_in_slot.vp = assignSlv->slaveID;
   3.256 -      prev_in_slot.task = assignSlv->assignCount;
   3.257 +      prev_in_slot.vp = returnSlv->slaveID;
   3.258 +      prev_in_slot.task = returnSlv->assignCount;
   3.259        semEnv->last_in_slot[coreNum * NUM_ANIM_SLOTS + slotNum] =
   3.260           prev_in_slot;        
   3.261      }
   3.262     #endif
   3.263 -   return( assignSlv );
   3.264 +   return( returnSlv );
   3.265   }
   3.266  
   3.267  
   3.268  //===========================  Request Handler  ============================
   3.269  //
   3.270  /*
   3.271 + * (BTW not inline because invoked indirectly via a pointer)
   3.272   */
   3.273  void
   3.274  VSs__Request_Handler( SlaveVP *requestingSlv, void *_semEnv )
   3.275 @@ -186,14 +205,14 @@
   3.276     while( req != NULL )
   3.277      {
   3.278        switch( req->reqType )
   3.279 -       { case semantic:     handleSemReq(         req, requestingSlv, semEnv);
   3.280 +       { case semantic:     handleSemReq(        req, requestingSlv, semEnv);
   3.281              break;
   3.282 -         case createReq:    handleCreate(         req, requestingSlv, semEnv);
   3.283 +         case createReq:    handleCreate(        req, requestingSlv, semEnv);
   3.284              break;
   3.285 -         case dissipate:    handleDissipate(           requestingSlv, semEnv);
   3.286 +         case dissipate:    handleDissipate(          requestingSlv, semEnv);
   3.287              break;
   3.288           case VMSSemantic:  VMS_PI__handle_VMSSemReq(req, requestingSlv, semEnv,
   3.289 -                                             (ResumeSlvFnPtr) &resume_slaveVP);
   3.290 +                                           (ResumeSlvFnPtr) &resume_slaveVP);
   3.291              break;
   3.292           default:
   3.293              break;
   3.294 @@ -205,7 +224,7 @@
   3.295   }
   3.296  
   3.297  
   3.298 -void
   3.299 +inline void
   3.300  handleSemReq( VMSReqst *req, SlaveVP *reqSlv, VSsSemEnv *semEnv )
   3.301   { VSsSemReq *semReq;
   3.302  
   3.303 @@ -213,22 +232,22 @@
   3.304     if( semReq == NULL ) return;
   3.305     switch( semReq->reqType )  //sem handlers are all in other file
   3.306      {
   3.307 -      case submit_task:     handleSubmitTask(   semReq,        semEnv);
   3.308 +      case submit_task:     handleSubmitTask(   semReq,         semEnv);
   3.309 +         break; 
   3.310 +      case end_task:        handleEndTask(      semReq,         semEnv);
   3.311           break;
   3.312 -      case end_task:        handleEndTask(      semReq,        semEnv);
   3.313 +      case send_type_to:    handleSendTypeTo(   semReq,         semEnv);
   3.314           break;
   3.315 -      case send_type_to:    handleSendTypeTo(   semReq,        semEnv);
   3.316 +      case send_from_to:    handleSendFromTo(   semReq,         semEnv);
   3.317           break;
   3.318 -      case send_from_to:    handleSendFromTo(   semReq,        semEnv);
   3.319 +      case receive_type_to: handleReceiveTypeTo(semReq,         semEnv);
   3.320           break;
   3.321 -      case receive_type_to: handleReceiveTypeTo(semReq,        semEnv);
   3.322 +      case receive_from_to: handleReceiveFromTo(semReq,         semEnv);
   3.323           break;
   3.324 -      case receive_from_to: handleReceiveFromTo(semReq,        semEnv);
   3.325 -         break;
   3.326 +      case taskwait:        handleTaskwait(     semReq, reqSlv, semEnv);
   3.327 +           break;
   3.328           
   3.329        //====================================================================
   3.330 -       case taskwait:   handleTaskwait(semReq, reqSlv, semEnv);
   3.331 -           break;
   3.332        case malloc_req:      handleMalloc(       semReq, reqSlv, semEnv);
   3.333           break;
   3.334        case free_req:        handleFree(         semReq, reqSlv, semEnv);
   3.335 @@ -255,10 +274,14 @@
   3.336  //=========================== VMS Request Handlers ==============================
   3.337  /*SlaveVP dissipate  (NOT task-end!)
   3.338   */
   3.339 -void
   3.340 +inline void
   3.341  handleDissipate( SlaveVP *requestingSlv, VSsSemEnv *semEnv )
   3.342   {
   3.343 -    DEBUG__printf1(dbgRqstHdlr,"Dissipate request from processor %d",requestingSlv->slaveID)
   3.344 +         DEBUG__printf1(dbgRqstHdlr,"Dissipate request from processor %d",
   3.345 +                                                     requestingSlv->slaveID)
   3.346 +       
   3.347 +   semEnv->numAdditionalSlvs -= 1;
   3.348 +   
   3.349        //free any semantic data allocated to the virt procr
   3.350     VMS_PI__free( requestingSlv->semanticData );
   3.351  
   3.352 @@ -268,7 +291,7 @@
   3.353  
   3.354  /*Re-use this in the entry-point fn
   3.355   */
   3.356 -SlaveVP *
   3.357 +inline SlaveVP *
   3.358  VSs__create_slave_helper( TopLevelFnPtr fnPtr, void *initData,
   3.359                            VSsSemEnv *semEnv,    int32 coreToAssignOnto )
   3.360   { SlaveVP    *newSlv;
   3.361 @@ -277,7 +300,7 @@
   3.362        //This is running in master, so use internal version
   3.363     newSlv = VMS_PI__create_slaveVP( fnPtr, initData );
   3.364  
   3.365 -   semEnv->numSlaveVP += 1;
   3.366 +   semEnv->numAdditionalSlvs += 1;
   3.367  
   3.368     semData = VMS_PI__malloc( sizeof(VSsSemData) );
   3.369     semData->highestTransEntered = -1;
   3.370 @@ -298,7 +321,7 @@
   3.371     newSlv->coreAnimatedBy = 0;
   3.372  
   3.373     #else
   3.374 -
   3.375 +      //Assigning slaves to cores is part of SSR code..
   3.376     if(coreToAssignOnto < 0 || coreToAssignOnto >= NUM_CORES )
   3.377      {    //out-of-range, so round-robin assignment
   3.378        newSlv->coreAnimatedBy = semEnv->nextCoreToGetNewSlv;
   3.379 @@ -317,9 +340,17 @@
   3.380     return newSlv;
   3.381   }
   3.382  
   3.383 -/*SlaveVP create  (NOT task create!)
   3.384 +/*This has been removed, because have changed things.. the only way to 
   3.385 + * create a slaveVP now is to either do an explicit create in the app, or
   3.386 + * else for req hdlr to create it when a task suspends if no extras are
   3.387 + * free.
   3.388 + *So, only have handleExplCreate for now.. and have the req hdlrs use the
   3.389 + * helper
   3.390 + *SlaveVP create  (NOT task create!)
   3.391 + *
   3.392   */
   3.393 -void
   3.394 +/*
   3.395 +inline void
   3.396  handleCreate( VMSReqst *req, SlaveVP *requestingSlv, VSsSemEnv *semEnv  )
   3.397   { VSsSemReq *semReq;
   3.398     SlaveVP    *newSlv;
   3.399 @@ -327,12 +358,13 @@
   3.400     
   3.401     semReq = VMS_PI__take_sem_reqst_from( req );
   3.402   
   3.403 -   newSlv = VSs__create_slave_helper( semReq->fnPtr, semReq->initData, semEnv,
   3.404 -                                     semReq->coreToAssignOnto );
   3.405 +   newSlv = VSs__create_slave_helper( semReq->fnPtr, semReq->initData,
   3.406 +                                         semEnv, semReq->coreToAssignOnto );
   3.407     
   3.408     ((VSsSemData*)newSlv->semanticData)->threadInfo->parent = requestingSlv;
   3.409     
   3.410 -         DEBUG__printf2(dbgRqstHdlr,"Create from: %d, new VP: %d", requestingSlv->slaveID, newSlv->slaveID)
   3.411 +         DEBUG__printf2(dbgRqstHdlr,"Create from: %d, new VP: %d",
   3.412 +                                    requestingSlv->slaveID, newSlv->slaveID)
   3.413  
   3.414     #ifdef HOLISTIC__TURN_ON_OBSERVE_UCC
   3.415     Dependency newd;
   3.416 @@ -340,7 +372,75 @@
   3.417     newd.from_task = requestingSlv->assignCount;
   3.418     newd.to_vp = newSlv->slaveID;
   3.419     newd.to_task = 1;
   3.420 -   //addToListOfArraysDependency(newd,semEnv->commDependenciesList);  
   3.421 +   addToListOfArrays(Dependency,newd,semEnv->commDependenciesList);   
   3.422 +   #endif
   3.423 +
   3.424 +      //For VSs, caller needs ptr to created processor returned to it
   3.425 +   requestingSlv->dataRetFromReq = newSlv;
   3.426 +
   3.427 +   resume_slaveVP( requestingSlv, semEnv );
   3.428 +   resume_slaveVP( newSlv,        semEnv );
   3.429 + }
   3.430 +*/
   3.431 +
   3.432 +VSsTaskStub *
   3.433 +create_expl_proc_task_stub( void *initData )
   3.434 + { VSsTaskStub *newStub;
   3.435 +         
   3.436 +   newStub = VMS_PI__malloc( sizeof(VSsTaskStub) );
   3.437 +   newStub->numBlockingProp = 0;
   3.438 +   newStub->slaveAssignedTo = NULL; //set later
   3.439 +   newStub->taskType        = NULL; //Identifies as an explicit processor
   3.440 +   newStub->ptrEntries      = NULL;
   3.441 +   newStub->args            = initData;  
   3.442 +   newStub->numChildTasks   = 0;
   3.443 +   newStub->parent          = NULL;
   3.444 +   newStub->isWaiting       = FALSE;
   3.445 +   newStub->taskID          = NULL;
   3.446 +   newStub->parentIsTask    = FALSE;
   3.447 +
   3.448 +   return newStub;
   3.449 + }
   3.450 +
   3.451 +/*Application invokes this when it explicitly creates a "processor" via the
   3.452 + * "SSR__create_processor()" command.
   3.453 + * 
   3.454 + *Make everything in VSs be a task.  An explicitly created VP is just a
   3.455 + * suspendable task, and the seedVP is also a suspendable task. 
   3.456 + *So, here, create a task Stub. 
   3.457 + * Then, see if there are any extra slaveVPs hanging around, and if not,
   3.458 + * call the helper to make a new one.
   3.459 + * Then, put the task stub into the slave's semantic Data.
   3.460 + *When the slave calls dissipate, have to recycle the task stub.
   3.461 + */
   3.462 +inline void
   3.463 +handleExplCreate( VMSReqst *req, SlaveVP *requestingSlv, VSsSemEnv *semEnv )
   3.464 + { VSsSemReq  *semReq;
   3.465 +   SlaveVP    *newSlv;
   3.466 +   VSsSemData *semData;
   3.467 +   
   3.468 +   semReq = VMS_PI__take_sem_reqst_from( req );
   3.469 +
   3.470 +   taskStub = create_expl_proc_task_stub( initData );
   3.471 +   
   3.472 +   newSlv = readPrivQ( semEnv->extraTaskSlvQ );
   3.473 +   if( newSlv == NULL )
   3.474 +      newSlv = VSs__create_slave_helper( semReq->fnPtr, semReq->initData,
   3.475 +                                         semEnv, semReq->coreToAssignOnto );
   3.476 +   
   3.477 +   semData = ( (VSsSemData *)newSlv->semanticData );
   3.478 +   semData->threadInfo->parent = requestingSlv;
   3.479 +   semData->taskStub = newPrTaskStub;
   3.480 +   
   3.481 +         DEBUG__printf2(dbgRqstHdlr,"Create from: %d, new VP: %d",
   3.482 +                                    requestingSlv->slaveID, newSlv->slaveID)
   3.483 +
   3.484 +   #ifdef HOLISTIC__TURN_ON_OBSERVE_UCC
   3.485 +   Dependency newd;
   3.486 +   newd.from_vp = requestingSlv->slaveID;
   3.487 +   newd.from_task = requestingSlv->assignCount;
   3.488 +   newd.to_vp = newSlv->slaveID;
   3.489 +   newd.to_task = 1;
   3.490     addToListOfArrays(Dependency,newd,semEnv->commDependenciesList);   
   3.491     #endif
   3.492  
   3.493 @@ -356,6 +456,9 @@
   3.494  void
   3.495  resume_slaveVP( SlaveVP *slave, VSsSemEnv *semEnv )
   3.496   {
   3.497 +      //both suspended tasks and suspended explicit slaves resumed with this
   3.498 +   writePrivQ( slave, semEnv->slavesReadyToResumeQ );
   3.499 +   
   3.500     #ifdef HOLISTIC__TURN_ON_PERF_COUNTERS
   3.501  /*
   3.502     int lastRecordIdx = slave->counter_history_array_info->numInArray -1;
   3.503 @@ -379,5 +482,4 @@
   3.504          addToListOfArrays(Dependency, newd ,semEnv->ctlDependenciesList);  
   3.505     }
   3.506     #endif
   3.507 -   writePrivQ( slave, semEnv->readyVPQs[ slave->coreAnimatedBy] );
   3.508   }

     4.1 --- a/VSs_Request_Handlers.c	Fri Jul 13 17:35:49 2012 +0200
     4.2 +++ b/VSs_Request_Handlers.c	Wed Aug 01 00:18:53 2012 -0700
     4.3 @@ -212,7 +212,7 @@
     4.4   * 
     4.5   *That should be it -- that should work.
     4.6   */
     4.7 -void
     4.8 +inline void
     4.9  handleSubmitTask( VSsSemReq *semReq, VSsSemEnv *semEnv )
    4.10   { uint32           key[3];
    4.11     HashEntry       *rawHashEntry; //has char *, but use with uint32 *
    4.12 @@ -335,11 +335,6 @@
    4.13     return;
    4.14   }
    4.15  
    4.16 -inline void
    4.17 -handleSubmitTaskWID( VSsSemReq *semReq, VSsSemEnv *semEnv)
    4.18 - {
    4.19 - }
    4.20 -
    4.21  
    4.22  /* ========================== end of task ===========================
    4.23   * 
    4.24 @@ -376,7 +371,7 @@
    4.25   *TODO: Might be safe to delete an entry when task ends and waiterQ empty
    4.26   * and no readers and no writers..
    4.27   */
    4.28 -void
    4.29 +inline void
    4.30  handleEndTask( VSsSemReq *semReq, VSsSemEnv *semEnv )
    4.31   { uint32            key[3];
    4.32     HashEntry        *rawHashEntry;
    4.33 @@ -402,21 +397,24 @@
    4.34     ptrEntries     = endingTaskStub->ptrEntries; //saved in stub when create
    4.35     
    4.36     /* Check if parent was waiting on this task */
    4.37 -   if(endingTaskStub->parentIsTask){
    4.38 -       VSsTaskStub* parent = (VSsTaskStub*) endingTaskStub->parent;
    4.39 -       parent->numChildTasks--;
    4.40 -       if(parent->isWaiting && parent->numChildTasks == 0){
    4.41 -           parent->isWaiting = FALSE;
    4.42 -           resume_slaveVP( parent->slaveAssignedTo, semEnv );
    4.43 +   if(endingTaskStub->parentIsTask)
    4.44 +    { VSsTaskStub* parent = (VSsTaskStub*) endingTaskStub->parent;
    4.45 +      parent->numChildTasks--;
    4.46 +      if(parent->isWaiting && parent->numChildTasks == 0)
    4.47 +       {
    4.48 +         parent->isWaiting = FALSE;
    4.49 +         resume_slaveVP( parent->slaveAssignedTo, semEnv );
    4.50         }
    4.51 -   } else {
    4.52 -       VSsThreadInfo* parent = (VSsThreadInfo*) endingTaskStub->parent;
    4.53 -       parent->numChildTasks--;
    4.54 -       if(parent->isWaiting && parent->numChildTasks == 0){
    4.55 -           parent->isWaiting = FALSE;
    4.56 -           resume_slaveVP( parent->slaveAssignedTo, semEnv );
    4.57 +    } 
    4.58 +   else 
    4.59 +    { VSsThreadInfo* parent = (VSsThreadInfo*) endingTaskStub->parent;
    4.60 +      parent->numChildTasks--;
    4.61 +      if(parent->isWaiting && parent->numChildTasks == 0)
    4.62 +       {
    4.63 +         parent->isWaiting = FALSE;
    4.64 +         resume_slaveVP( parent->slaveAssignedTo, semEnv );
    4.65         }
    4.66 -   }
    4.67 +    }
    4.68     
    4.69     /*The task's controlled arguments are processed one by one.
    4.70      *Processing an argument means getting arg-pointer's entry.
    4.71 @@ -549,7 +547,7 @@
    4.72   * separate tasks can send to the same receiver, and doing hash on the
    4.73   * receive task, so they will stack up.
    4.74   */
    4.75 -void
    4.76 +inline void
    4.77  handleSendTypeTo( VSsSemReq *semReq, VSsSemEnv *semEnv )
    4.78   { SlaveVP    *senderSlv, *receiverSlv;
    4.79     int32      *senderID, *receiverID;
    4.80 @@ -645,7 +643,7 @@
    4.81  /*Looks like can make single handler for both sends..
    4.82   */
    4.83  //TODO: combine both send handlers into single handler
    4.84 -void
    4.85 +inline void
    4.86  handleSendFromTo( VSsSemReq *semReq, VSsSemEnv *semEnv)
    4.87   { SlaveVP     *senderSlv, *receiverSlv;
    4.88     int32       *senderID, *receiverID;
    4.89 @@ -721,7 +719,7 @@
    4.90  //
    4.91  
    4.92  
    4.93 -void
    4.94 +inline void
    4.95  handleReceiveTypeTo( VSsSemReq *semReq, VSsSemEnv *semEnv)
    4.96   { SlaveVP    *senderSlv, *receiverSlv;
    4.97     int32      *receiverID;
    4.98 @@ -803,7 +801,7 @@
    4.99  
   4.100  /*
   4.101   */
   4.102 -void
   4.103 +inline void
   4.104  handleReceiveFromTo( VSsSemReq *semReq, VSsSemEnv *semEnv)
   4.105   { SlaveVP     *senderSlv, *receiverSlv;
   4.106     int32       *senderID,  *receiverID;
   4.107 @@ -867,8 +865,8 @@
   4.108   }
   4.109  
   4.110  //==========================================================================
   4.111 -void
   4.112 -handleTaskwait( VSsSemReq *semReq, SlaveVP *requestingSlv, VSsSemEnv *semEnv )
   4.113 +inline void
   4.114 +handleTaskwait( VSsSemReq *semReq, SlaveVP *requestingSlv, VSsSemEnv *semEnv)
   4.115   {
   4.116      VSsTaskStub* requestingTaskStub;
   4.117      

     5.1 --- a/VSs_Request_Handlers.h	Fri Jul 13 17:35:49 2012 +0200
     5.2 +++ b/VSs_Request_Handlers.h	Wed Aug 01 00:18:53 2012 -0700
     5.3 @@ -19,6 +19,17 @@
     5.4  inline void
     5.5  handleEndTask( VSsSemReq *semReq, VSsSemEnv *semEnv);
     5.6  inline void
     5.7 +handleSendTypeTo( VSsSemReq *semReq, VSsSemEnv *semEnv);
     5.8 +inline void
     5.9 +handleSendFromTo( VSsSemReq *semReq, VSsSemEnv *semEnv);
    5.10 +inline void
    5.11 +handleReceiveTypeTo( VSsSemReq *semReq, VSsSemEnv *semEnv);
    5.12 +inline void
    5.13 +handleReceiveFromTo( VSsSemReq *semReq, VSsSemEnv *semEnv);
    5.14 +inline void
    5.15 +handleTaskwait(VSsSemReq *semReq, SlaveVP *requestingSlv, VSsSemEnv *semEnv);
    5.16 +
    5.17 +inline void
    5.18  handleMalloc( VSsSemReq *semReq, SlaveVP *requestingSlv, VSsSemEnv *semEnv);
    5.19  inline void
    5.20  handleFree( VSsSemReq *semReq, SlaveVP *requestingSlv, VSsSemEnv *semEnv );

     6.1 --- a/__brch__default	Fri Jul 13 17:35:49 2012 +0200
     6.2 +++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.3 @@ -1,4 +0,0 @@
     6.4 -This branch is for the project structure defined Jan 2012..  the #includes reflect this directory structure.
     6.5 -
     6.6 -More importantly, the MC_shared  version of VMS requires a separat malloc implemeted by VMS code..  so this branch has modified the library to use the VMS-specific malloc.
     6.7 -

     7.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     7.2 +++ b/__brch__dev_expl_VP_and_DKU	Wed Aug 01 00:18:53 2012 -0700
     7.3 @@ -0,0 +1,4 @@
     7.4 +This branch is for the project structure defined Jan 2012..  the #includes reflect this directory structure.
     7.5 +
     7.6 +More importantly, the MC_shared  version of VMS requires a separat malloc implemeted by VMS code..  so this branch has modified the library to use the VMS-specific malloc.
     7.7 +