1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/.hgignore	Wed May 11 15:29:58 2011 +0200
     1.3 @@ -0,0 +1,1 @@
     1.4 +glob:.hgtags

     2.1 --- a/VCilk.h	Tue Nov 16 15:59:23 2010 +0100
     2.2 +++ b/VCilk.h	Wed May 11 15:29:58 2011 +0200
     2.3 @@ -1,178 +1,200 @@
     2.4 -/*
     2.5 - *  Copyright 2009 OpenSourceStewardshipFoundation.org
     2.6 - *  Licensed under GNU General Public License version 2
     2.7 - *
     2.8 - * Author: seanhalle@yahoo.com
     2.9 - *
    2.10 - */
    2.11 -
    2.12 -#ifndef _VCilk_H
    2.13 -#define	_VCilk_H
    2.14 -
    2.15 -#include "VMS/Queue_impl/PrivateQueue.h"
    2.16 -#include "VMS/Hash_impl/PrivateHash.h"
    2.17 -#include "VMS/VMS.h"
    2.18 -
    2.19 -
    2.20 -
    2.21 -/*This header defines everything specific to the VCilk semantic plug-in
    2.22 - */
    2.23 -
    2.24 -//===========================================================================
    2.25 -#define NUM_STRUCS_IN_SEM_ENV 1000
    2.26 -
    2.27 -//===========================================================================
    2.28 -typedef struct _VCilkSemReq   VCilkSemReq;
    2.29 -typedef void  (*PtrToAtomicFn )   ( void * ); //executed atomically in master
    2.30 -
    2.31 -//===========================================================================
    2.32 -
    2.33 -
    2.34 -/*Semantic layer-specific data sent inside a request from lib called in app
    2.35 - * to request handler called in MasterLoop
    2.36 - */
    2.37 -enum VCilkReqType
    2.38 - {
    2.39 -   syncReq = 1,
    2.40 -   mallocReq,
    2.41 -   freeReq,
    2.42 -   singleton,
    2.43 -   atomic,
    2.44 -   trans_start,
    2.45 -   trans_end
    2.46 - };
    2.47 -
    2.48 -struct _VCilkSemReq
    2.49 - { enum VCilkReqType    reqType;
    2.50 -   VirtProcr           *requestingPr;
    2.51 -   
    2.52 -   int32                sizeToMalloc;
    2.53 -   void                *ptrToFree;
    2.54 -   
    2.55 -   VirtProcrFnPtr       fnPtr;
    2.56 -   void                *initData;
    2.57 -   int32                coreToSpawnOnto;
    2.58 -
    2.59 -   int32              singletonID;
    2.60 -   void              *endJumpPt;
    2.61 -
    2.62 -   PtrToAtomicFn      fnToExecInMaster;
    2.63 -   void              *dataForFn;
    2.64 -
    2.65 -   int32              transID;
    2.66 - }
    2.67 -/* VCilkSemReq */;
    2.68 -
    2.69 -typedef struct
    2.70 - {
    2.71 -   VirtProcr      *VPCurrentlyExecuting;
    2.72 -   PrivQueueStruc *waitingVPQ;
    2.73 - }
    2.74 -VCilkTrans;
    2.75 -
    2.76 -typedef struct
    2.77 - {
    2.78 -   PrivQueueStruc **readyVPQs;
    2.79 -   HashTable       *commHashTbl;
    2.80 -   int32            numVirtPr;
    2.81 -   int32            nextCoreToGetNewPr;
    2.82 -   int32            primitiveStartTime;
    2.83 -
    2.84 -                       //fix limit on num with dynArray
    2.85 -   int32            singletonHasBeenExecutedFlags[NUM_STRUCS_IN_SEM_ENV];
    2.86 -   VCilkTrans       transactionStrucs[NUM_STRUCS_IN_SEM_ENV];
    2.87 - }
    2.88 -VCilkSemEnv;
    2.89 -
    2.90 -typedef struct _TransListElem TransListElem;
    2.91 -struct _TransListElem
    2.92 - {
    2.93 -   int32          transID;
    2.94 -   TransListElem *nextTrans;
    2.95 - };
    2.96 -//TransListElem
    2.97 -
    2.98 -typedef struct
    2.99 - {
   2.100 -   int32          syncPending;
   2.101 -   int32          numLiveChildren;
   2.102 -   VirtProcr     *parentPr;
   2.103 -   
   2.104 -   int32          highestTransEntered;
   2.105 -   TransListElem *lastTransEntered;
   2.106 - }
   2.107 -VCilkSemData;
   2.108 -
   2.109 -//===========================================================================
   2.110 -
   2.111 -void
   2.112 -VCilk__create_seed_procr_and_do_work( VirtProcrFnPtr fn, void *initData );
   2.113 -
   2.114 -int32
   2.115 -VCilk__giveMinWorkUnitCycles( float32 percentOverhead );
   2.116 -
   2.117 -void inline
   2.118 -VCilk__start_primitive();
   2.119 -
   2.120 -int32 inline
   2.121 -VCilk__end_primitive_and_give_cycles();
   2.122 -
   2.123 -int32
   2.124 -VCilk__giveIdealNumWorkUnits();
   2.125 -
   2.126 -//=======================
   2.127 -
   2.128 -void
   2.129 -VCilk__init();
   2.130 -
   2.131 -void
   2.132 -VCilk__cleanup_at_end_of_shutdown();
   2.133 -
   2.134 -//=======================
   2.135 -
   2.136 -void inline
   2.137 -VCilk__spawn( int32  coreToSpawnOnto, VirtProcrFnPtr  fnPtr,
   2.138 -              void  *initData,        VirtProcr      *creatingPr );
   2.139 -
   2.140 -int32
   2.141 -VCilk__give_number_of_cores_to_spawn_onto();
   2.142 -
   2.143 -void
   2.144 -VCilk__sync( VirtProcr *animatingPr );
   2.145 -
   2.146 -void *
   2.147 -VCilk__malloc( int32 sizeToMalloc, VirtProcr *animPr );
   2.148 -
   2.149 -void
   2.150 -VCilk__free( void *ptrToFree, VirtProcr *animPr );
   2.151 -
   2.152 -void
   2.153 -VCilk__dissipate_procr( VirtProcr *procrToDissipate );
   2.154 -
   2.155 -
   2.156 -//======================= Concurrency Stuff ======================
   2.157 -void
   2.158 -VCilk__start_singleton( int32 singletonID, void *endSingletonLabelAddr,
   2.159 -                        VirtProcr *animPr );
   2.160 -
   2.161 -void
   2.162 -VCilk__animate_short_fn_in_isolation( PtrToAtomicFn ptrToFnToExecInMaster,
   2.163 -                                      void *data, VirtProcr *animPr );
   2.164 -
   2.165 -void
   2.166 -VCilk__start_transaction( int32 transactionID, VirtProcr *animPr );
   2.167 -
   2.168 -void
   2.169 -VCilk__end_transaction( int32 transactionID, VirtProcr *animPr );
   2.170 -
   2.171 -
   2.172 -//=========================  Internal use only  =============================
   2.173 -void
   2.174 -VCilk__Request_Handler( VirtProcr *requestingPr, void *_semEnv );
   2.175 -
   2.176 -VirtProcr *
   2.177 -VCilk__schedule_virt_procr( void *_semEnv, int coreNum );
   2.178 -
   2.179 -
   2.180 -#endif	/* _VCilk_H */
   2.181 -
   2.182 +/*
   2.183 + *  Copyright 2009 OpenSourceStewardshipFoundation.org
   2.184 + *  Licensed under GNU General Public License version 2
   2.185 + *
   2.186 + * Author: seanhalle@yahoo.com
   2.187 + *
   2.188 + */
   2.189 +
   2.190 +#ifndef _VCilk_H
   2.191 +#define	_VCilk_H
   2.192 +
   2.193 +#include "VMS/Queue_impl/PrivateQueue.h"
   2.194 +#include "VMS/Hash_impl/PrivateHash.h"
   2.195 +#include "VMS/VMS.h"
   2.196 +
   2.197 +
   2.198 +
   2.199 +/*This header defines everything specific to the VCilk semantic plug-in
   2.200 + */
   2.201 +
   2.202 +//===========================================================================
   2.203 +#define NUM_STRUCS_IN_SEM_ENV 1000
   2.204 +
   2.205 +//===========================================================================
   2.206 +typedef struct _VCilkSemReq   VCilkSemReq;
   2.207 +typedef void  (*PtrToAtomicFn )   ( void * ); //executed atomically in master
   2.208 +
   2.209 +//===========================================================================
   2.210 +
   2.211 +
   2.212 +/*WARNING: assembly hard-codes position of endInstrAddr as first field
   2.213 + */
   2.214 +typedef struct
   2.215 + {
   2.216 +   void           *endInstrAddr;
   2.217 +   int32           hasBeenStarted;
   2.218 +   int32           hasFinished;
   2.219 +   PrivQueueStruc *waitQ;
   2.220 + }
   2.221 +VCilkSingleton;
   2.222 +
   2.223 +/*Semantic layer-specific data sent inside a request from lib called in app
   2.224 + * to request handler called in MasterLoop
   2.225 + */
   2.226 +enum VCilkReqType
   2.227 + {
   2.228 +   syncReq = 1,
   2.229 +   mallocReq,
   2.230 +   freeReq,
   2.231 +   singleton_fn_start,
   2.232 +   singleton_fn_end,
   2.233 +   singleton_data_start,
   2.234 +   singleton_data_end,
   2.235 +   atomic,
   2.236 +   trans_start,
   2.237 +   trans_end
   2.238 + };
   2.239 +
   2.240 +struct _VCilkSemReq
   2.241 + { enum VCilkReqType    reqType;
   2.242 +   VirtProcr           *requestingPr;
   2.243 +   
   2.244 +   int32                sizeToMalloc;
   2.245 +   void                *ptrToFree;
   2.246 +   
   2.247 +   VirtProcrFnPtr       fnPtr;
   2.248 +   void                *initData;
   2.249 +   int32                coreToSpawnOnto;
   2.250 +
   2.251 +   int32              singletonID;
   2.252 +   VCilkSingleton     **singletonPtrAddr;
   2.253 +
   2.254 +   PtrToAtomicFn      fnToExecInMaster;
   2.255 +   void              *dataForFn;
   2.256 +
   2.257 +   int32              transID;
   2.258 + }
   2.259 +/* VCilkSemReq */;
   2.260 +
   2.261 +typedef struct
   2.262 + {
   2.263 +   VirtProcr      *VPCurrentlyExecuting;
   2.264 +   PrivQueueStruc *waitingVPQ;
   2.265 + }
   2.266 +VCilkTrans;
   2.267 +
   2.268 +typedef struct
   2.269 + {
   2.270 +   PrivQueueStruc **readyVPQs;
   2.271 +   HashTable       *commHashTbl;
   2.272 +   int32            numVirtPr;
   2.273 +   int32            nextCoreToGetNewPr;
   2.274 +   int32            primitiveStartTime;
   2.275 +
   2.276 +                       //fix limit on num with dynArray
   2.277 +   VCilkSingleton     fnSingletons[NUM_STRUCS_IN_SEM_ENV];
   2.278 +   VCilkTrans       transactionStrucs[NUM_STRUCS_IN_SEM_ENV];
   2.279 + }
   2.280 +VCilkSemEnv;
   2.281 +
   2.282 +typedef struct _TransListElem TransListElem;
   2.283 +struct _TransListElem
   2.284 + {
   2.285 +   int32          transID;
   2.286 +   TransListElem *nextTrans;
   2.287 + };
   2.288 +//TransListElem
   2.289 +
   2.290 +typedef struct
   2.291 + {
   2.292 +   int32          syncPending;
   2.293 +   int32          numLiveChildren;
   2.294 +   VirtProcr     *parentPr;
   2.295 +   
   2.296 +   int32          highestTransEntered;
   2.297 +   TransListElem *lastTransEntered;
   2.298 + }
   2.299 +VCilkSemData;
   2.300 +
   2.301 +//===========================================================================
   2.302 +
   2.303 +void
   2.304 +VCilk__create_seed_procr_and_do_work( VirtProcrFnPtr fn, void *initData );
   2.305 +
   2.306 +int32
   2.307 +VCilk__giveMinWorkUnitCycles( float32 percentOverhead );
   2.308 +
   2.309 +void inline
   2.310 +VCilk__start_primitive();
   2.311 +
   2.312 +int32 inline
   2.313 +VCilk__end_primitive_and_give_cycles();
   2.314 +
   2.315 +int32
   2.316 +VCilk__giveIdealNumWorkUnits();
   2.317 +
   2.318 +//=======================
   2.319 +
   2.320 +void
   2.321 +VCilk__init();
   2.322 +
   2.323 +void
   2.324 +VCilk__cleanup_at_end_of_shutdown();
   2.325 +
   2.326 +//=======================
   2.327 +
   2.328 +void inline
   2.329 +VCilk__spawn( int32  coreToSpawnOnto, VirtProcrFnPtr  fnPtr,
   2.330 +              void  *initData,        VirtProcr      *creatingPr );
   2.331 +
   2.332 +int32
   2.333 +VCilk__give_number_of_cores_to_spawn_onto();
   2.334 +
   2.335 +void
   2.336 +VCilk__sync( VirtProcr *animatingPr );
   2.337 +
   2.338 +void *
   2.339 +VCilk__malloc( int32 sizeToMalloc, VirtProcr *animPr );
   2.340 +
   2.341 +void
   2.342 +VCilk__free( void *ptrToFree, VirtProcr *animPr );
   2.343 +
   2.344 +void
   2.345 +VCilk__dissipate_procr( VirtProcr *procrToDissipate );
   2.346 +
   2.347 +
   2.348 +//======================= Concurrency Stuff ======================
   2.349 +void
   2.350 +VCilk__start_fn_singleton( int32 singletonID, VirtProcr *animPr );
   2.351 +
   2.352 +void
   2.353 +VCilk__end_fn_singleton( int32 singletonID, VirtProcr *animPr );
   2.354 +
   2.355 +void
   2.356 +VCilk__start_data_singleton( VCilkSingleton **singeltonAddr, VirtProcr *animPr );
   2.357 +
   2.358 +void
   2.359 +VCilk__end_data_singleton( VCilkSingleton **singletonAddr, VirtProcr *animPr );
   2.360 +
   2.361 +void
   2.362 +VCilk__animate_short_fn_in_isolation( PtrToAtomicFn ptrToFnToExecInMaster,
   2.363 +                                      void *data, VirtProcr *animPr );
   2.364 +
   2.365 +void
   2.366 +VCilk__start_transaction( int32 transactionID, VirtProcr *animPr );
   2.367 +
   2.368 +void
   2.369 +VCilk__end_transaction( int32 transactionID, VirtProcr *animPr );
   2.370 +
   2.371 +
   2.372 +//=========================  Internal use only  =============================
   2.373 +void
   2.374 +VCilk__Request_Handler( VirtProcr *requestingPr, void *_semEnv );
   2.375 +
   2.376 +VirtProcr *
   2.377 +VCilk__schedule_virt_procr( void *_semEnv, int coreNum );
   2.378 +
   2.379 +
   2.380 +#endif	/* _VCilk_H */
   2.381 +

     3.1 --- a/VCilk_PluginFns.c	Tue Nov 16 15:59:23 2010 +0100
     3.2 +++ b/VCilk_PluginFns.c	Wed May 11 15:29:58 2011 +0200
     3.3 @@ -1,467 +1,555 @@
     3.4 -/*
     3.5 - * Copyright 2010  OpenSourceCodeStewardshipFoundation
     3.6 - *
     3.7 - * Licensed under BSD
     3.8 - */
     3.9 -
    3.10 -#include <stdio.h>
    3.11 -#include <stdlib.h>
    3.12 -
    3.13 -#include "VMS/Queue_impl/PrivateQueue.h"
    3.14 -#include "VCilk.h"
    3.15 -
    3.16 -
    3.17 -
    3.18 -//===========================================================================
    3.19 -void inline
    3.20 -handleSync( VirtProcr *requestingPr, VCilkSemEnv *semEnv );
    3.21 -
    3.22 -void inline
    3.23 -handleMalloc( VCilkSemReq *semReq, VirtProcr *requestingPr,
    3.24 -              VCilkSemEnv *semEnv );
    3.25 -void inline
    3.26 -handleFree( VCilkSemReq *semReq, VirtProcr *requestingPr,
    3.27 -            VCilkSemEnv *semEnv );
    3.28 -void inline
    3.29 -handleDissipate( VirtProcr *requestingPr, VCilkSemEnv *semEnv );
    3.30 -
    3.31 -void inline
    3.32 -handleSpawn( VMSReqst *req, VirtProcr *requestingPr, VCilkSemEnv *semEnv );
    3.33 -
    3.34 -void inline
    3.35 -dispatchSemReq( VMSReqst *req, VirtProcr *requestingPr, VCilkSemEnv *semEnv);
    3.36 -
    3.37 -void inline
    3.38 -handleTransEnd( VCilkSemReq *semReq, VirtProcr *requestingPr,
    3.39 -                VCilkSemEnv*semEnv);
    3.40 -void inline
    3.41 -handleTransStart( VCilkSemReq *semReq, VirtProcr *requestingPr,
    3.42 -                  VCilkSemEnv *semEnv );
    3.43 -void inline
    3.44 -handleAtomic( VCilkSemReq *semReq, VirtProcr *requestingPr,
    3.45 -              VCilkSemEnv *semEnv);
    3.46 -void inline
    3.47 -handleSingleton( VCilkSemReq *semReq, VirtProcr *requestingPr,
    3.48 -                 VCilkSemEnv *semEnv );
    3.49 -
    3.50 -void inline
    3.51 -resume_procr( VirtProcr *procr, VCilkSemEnv *semEnv );
    3.52 -
    3.53 -//===========================================================================
    3.54 -
    3.55 -
    3.56 -//============================== Scheduler ==================================
    3.57 -//
    3.58 -/*For VCilk, scheduling a slave simply takes the next work-unit off the
    3.59 - * ready-to-go work-unit queue and assigns it to the slaveToSched.
    3.60 - *If the ready-to-go work-unit queue is empty, then nothing to schedule
    3.61 - * to the slave -- return FALSE to let Master loop know scheduling that
    3.62 - * slave failed.
    3.63 - */
    3.64 -VirtProcr *
    3.65 -VCilk__schedule_virt_procr( void *_semEnv, int coreNum )
    3.66 - { VirtProcr   *schedPr;
    3.67 -   VCilkSemEnv *semEnv;
    3.68 -
    3.69 -   semEnv = (VCilkSemEnv *)_semEnv;
    3.70 -
    3.71 -   schedPr = readPrivQ( semEnv->readyVPQs[coreNum] );
    3.72 -      //Note, using a non-blocking queue -- it returns NULL if queue empty
    3.73 -
    3.74 -   return( schedPr );
    3.75 - }
    3.76 -
    3.77 -
    3.78 -//===========================  Request Handler  =============================
    3.79 -//
    3.80 -/*Will get requests to send, to receive, and to create new processors.
    3.81 - * Upon send, check the hash to see if a receive is waiting.
    3.82 - * Upon receive, check hash to see if a send has already happened.
    3.83 - * When other is not there, put in.  When other is there, the comm.
    3.84 - *  completes, which means the receiver P gets scheduled and
    3.85 - *  picks up right after the receive request.  So make the work-unit
    3.86 - *  and put it into the queue of work-units ready to go.
    3.87 - * Other request is create a new Processor, with the function to run in the
    3.88 - *  Processor, and initial data.
    3.89 - */
    3.90 -void
    3.91 -VCilk__Request_Handler( VirtProcr *requestingPr, void *_semEnv )
    3.92 - { VCilkSemEnv *semEnv;
    3.93 -   VMSReqst    *req;
    3.94 -   VCilkSemReq *semReq;
    3.95 - 
    3.96 -   //============================= MEASUREMENT STUFF ========================
    3.97 -   #ifdef MEAS__TIME_PLUGIN
    3.98 -   int32 startStamp, endStamp;
    3.99 -   saveLowTimeStampCountInto( startStamp );
   3.100 -   #endif
   3.101 -   //========================================================================
   3.102 -
   3.103 -   semEnv = (VCilkSemEnv *)_semEnv;
   3.104 -
   3.105 -   req = VMS__take_next_request_out_of( requestingPr );
   3.106 -   
   3.107 -   while( req != NULL )
   3.108 -    {
   3.109 -      switch( req->reqType )
   3.110 -       { case semantic:     dispatchSemReq( req, requestingPr, semEnv );
   3.111 -            break;
   3.112 -         case createReq:    //create request has to come as a VMS request,
   3.113 -                            // to allow MasterLoop to do stuff before gets
   3.114 -                            // here, and maybe also stuff after all requests
   3.115 -                            // done -- however, can still attach semantic
   3.116 -                            // req data to req.
   3.117 -                            handleSpawn(          req, requestingPr, semEnv);
   3.118 -            break;
   3.119 -         case dissipate:    handleDissipate(           requestingPr, semEnv);
   3.120 -            break;
   3.121 -         case VMSSemantic:  VMS__handle_VMSSemReq(req, requestingPr, semEnv,
   3.122 -                                                              &resume_procr);
   3.123 -            break;
   3.124 -         default:
   3.125 -            break;
   3.126 -       }
   3.127 -      
   3.128 -      DoneHandlingReqst:
   3.129 -
   3.130 -      req = VMS__take_next_request_out_of( requestingPr );
   3.131 -    } //while( req != NULL )
   3.132 -
   3.133 -   //============================= MEASUREMENT STUFF ========================
   3.134 -   #ifdef MEAS__TIME_PLUGIN
   3.135 -   saveLowTimeStampCountInto( endStamp );
   3.136 -   addIntervalToHist( startStamp, endStamp, _VMSMasterEnv->pluginLowTimeHist );
   3.137 -   addIntervalToHist( startStamp, endStamp, _VMSMasterEnv->pluginHighTimeHist );
   3.138 -   #endif
   3.139 -   //========================================================================
   3.140 - }
   3.141 -
   3.142 -void inline
   3.143 -dispatchSemReq( VMSReqst *req, VirtProcr *reqPr, VCilkSemEnv *semEnv )
   3.144 - { VCilkSemReq *semReq;
   3.145 -
   3.146 -   semReq = VMS__take_sem_reqst_from(req);
   3.147 -
   3.148 -   if( semReq == NULL ) return;
   3.149 -   switch( semReq->reqType )
   3.150 -    {
   3.151 -      case syncReq:         handleSync(                 reqPr, semEnv );
   3.152 -         break;
   3.153 -      case mallocReq:       handleMalloc(       semReq, reqPr, semEnv );
   3.154 -         break;
   3.155 -      case freeReq:         handleFree(         semReq, reqPr, semEnv );
   3.156 -         break;
   3.157 -      case singleton:       handleSingleton(    semReq, reqPr, semEnv );
   3.158 -         break;
   3.159 -      case atomic:          handleAtomic(       semReq, reqPr, semEnv );
   3.160 -         break;
   3.161 -      case trans_start:     handleTransStart(   semReq, reqPr, semEnv );
   3.162 -         break;
   3.163 -      case trans_end:       handleTransEnd(     semReq, reqPr, semEnv );
   3.164 -         break;
   3.165 -    }
   3.166 -   //NOTE: semantic request data strucs allocated on stack in VCilk Lib calls
   3.167 - }
   3.168 -
   3.169 -
   3.170 -
   3.171 -//=========================== Request Handlers ==============================
   3.172 -void inline
   3.173 -resume_procr( VirtProcr *procr, VCilkSemEnv *semEnv )
   3.174 - {
   3.175 -   writePrivQ( procr, semEnv->readyVPQs[ procr->coreAnimatedBy] );
   3.176 - }
   3.177 -
   3.178 -
   3.179 -
   3.180 -
   3.181 -/* check if list of live children is empty.
   3.182 - * If yes, then resume.
   3.183 - * If no, then set sync-pending flag.
   3.184 - */
   3.185 -void
   3.186 -handleSync( VirtProcr *requestingPr, VCilkSemEnv *semEnv )
   3.187 - {
   3.188 -   if(((VCilkSemData *)(requestingPr->semanticData))->numLiveChildren  == 0 )
   3.189 -    { //no live children to wait for
   3.190 -      resume_procr( requestingPr, semEnv );
   3.191 -    }
   3.192 -   else
   3.193 -    {
   3.194 -      ((VCilkSemData *)(requestingPr->semanticData))->syncPending = TRUE;
   3.195 -    }
   3.196 - }
   3.197 -
   3.198 -/*
   3.199 - */
   3.200 -void
   3.201 -handleMalloc( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.202 -              VCilkSemEnv *semEnv )
   3.203 - { void *ptr;
   3.204 -   
   3.205 -   ptr = VMS__malloc( semReq->sizeToMalloc );
   3.206 -   requestingPr->dataRetFromReq = ptr;
   3.207 -   resume_procr( requestingPr, semEnv );
   3.208 - }
   3.209 -
   3.210 -/*
   3.211 - */
   3.212 -void inline
   3.213 -handleFree( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.214 -            VCilkSemEnv *semEnv )
   3.215 - {
   3.216 -   VMS__free( semReq->ptrToFree );
   3.217 -   resume_procr( requestingPr, semEnv );
   3.218 - }
   3.219 -
   3.220 -
   3.221 -//============================== VMS requests ===============================
   3.222 -/*Re-use this in the entry-point fn
   3.223 - */
   3.224 -inline VirtProcr *
   3.225 -VCilk__create_procr_helper( VirtProcrFnPtr fnPtr, void  *initData,
   3.226 -   VirtProcr *requestingPr, VCilkSemEnv *semEnv,  int32  coreToScheduleOnto )
   3.227 - { VirtProcr    *newPr;
   3.228 -   VCilkSemData *semData;
   3.229 -
   3.230 -      //This is running in master, so use internal version
   3.231 -   newPr = VMS__create_procr( fnPtr, initData );
   3.232 -
   3.233 -   semData = VMS__malloc( sizeof(VCilkSemData) );
   3.234 -
   3.235 -   semData->numLiveChildren = 0;
   3.236 -   semData->parentPr        = requestingPr;
   3.237 -   semData->syncPending     = FALSE;
   3.238 -   
   3.239 -   semData->highestTransEntered = -1;
   3.240 -   semData->lastTransEntered    = NULL;
   3.241 -
   3.242 -   newPr->semanticData = semData;
   3.243 -
   3.244 -   /* increase the number of live children of requester.
   3.245 -    */
   3.246 -   if( requestingPr != NULL ) //NULL when creating seed procr
   3.247 -     ((VCilkSemData *)(requestingPr->semanticData))->numLiveChildren +=1;
   3.248 -
   3.249 -   semEnv->numVirtPr += 1;
   3.250 -
   3.251 -   //=================== Assign new processor to a core =====================
   3.252 -   #ifdef SEQUENTIAL
   3.253 -   newPr->coreAnimatedBy = 0;
   3.254 -
   3.255 -   #else
   3.256 -
   3.257 -   if(coreToScheduleOnto < 0 || coreToScheduleOnto >= NUM_CORES )
   3.258 -    {    //out-of-range, so round-robin assignment
   3.259 -      newPr->coreAnimatedBy = semEnv->nextCoreToGetNewPr;
   3.260 -
   3.261 -      if( semEnv->nextCoreToGetNewPr >= NUM_CORES - 1 )
   3.262 -          semEnv->nextCoreToGetNewPr  = 0;
   3.263 -      else
   3.264 -          semEnv->nextCoreToGetNewPr += 1;
   3.265 -    }
   3.266 -   else //core num in-range, so use it
   3.267 -    { newPr->coreAnimatedBy = coreToScheduleOnto;
   3.268 -    }
   3.269 -   #endif
   3.270 -   //========================================================================
   3.271 -
   3.272 -   return newPr;
   3.273 - }
   3.274 -
   3.275 -
   3.276 -void inline
   3.277 -handleSpawn( VMSReqst *req, VirtProcr *requestingPr, VCilkSemEnv *semEnv )
   3.278 - { VCilkSemReq *semReq;
   3.279 -   VirtProcr    *newPr;
   3.280 -
   3.281 -   semReq = VMS__take_sem_reqst_from( req );
   3.282 -
   3.283 -   newPr = VCilk__create_procr_helper( semReq->fnPtr, semReq->initData,
   3.284 -                             requestingPr, semEnv, semReq->coreToSpawnOnto );
   3.285 -
   3.286 -      //For VPThread, caller needs ptr to created processor returned to it
   3.287 -   requestingPr->dataRetFromReq = newPr;
   3.288 -
   3.289 -   resume_procr( newPr,        semEnv );
   3.290 -   resume_procr( requestingPr, semEnv );
   3.291 - }
   3.292 -
   3.293 -
   3.294 -
   3.295 -/*get parentVP & remove dissipator from parent's live children.
   3.296 - *If this was last live child, check "sync pending" flag
   3.297 - *-- if set, then resume the parentVP.
   3.298 - */
   3.299 -void inline
   3.300 -handleDissipate( VirtProcr *requestingPr, VCilkSemEnv *semEnv )
   3.301 - {
   3.302 -   VirtProcr *
   3.303 -   parentPr = ((VCilkSemData *)
   3.304 -               (requestingPr->semanticData))->parentPr;
   3.305 -   if( parentPr == NULL ) //means this is seed processor being dissipated
   3.306 -    { //Just act normally, except don't deal with parent
   3.307 -      // VMS__Free is implemented to ignore requests to free data from
   3.308 -      // outside VMS, so all this processor's non-VMS allocated data will
   3.309 -      // remain and be cleaned up outside
   3.310 -    }
   3.311 -   else
   3.312 -    {
   3.313 -      ((VCilkSemData *)(parentPr->semanticData))->numLiveChildren -= 1;
   3.314 -      if( ((VCilkSemData *)
   3.315 -            (parentPr->semanticData))->numLiveChildren <= 0 )
   3.316 -       { //this was last live child of parent
   3.317 -         if( ((VCilkSemData *)
   3.318 -               (parentPr->semanticData))->syncPending == TRUE )
   3.319 -          { //was waiting for last child to dissipate, so resume it
   3.320 -            ((VCilkSemData *)
   3.321 -              (parentPr->semanticData))->syncPending = FALSE;
   3.322 -            resume_procr( parentPr, semEnv );
   3.323 -          }
   3.324 -       }
   3.325 -    }
   3.326 -
   3.327 -   VMS__free( requestingPr->semanticData );
   3.328 -   
   3.329 -       //Now do normal dissipate
   3.330 -   
   3.331 -       //call VMS to free_all AppVP state -- stack and so on
   3.332 -   VMS__dissipate_procr( requestingPr );
   3.333 -
   3.334 -   semEnv->numVirtPr -= 1;
   3.335 -   if( semEnv->numVirtPr == 0 )
   3.336 -    {    //no more work, so shutdown
   3.337 -      VMS__shutdown();
   3.338 -    }
   3.339 - }
   3.340 -
   3.341 -
   3.342 -//=============================== Atomic ====================================
   3.343 -//
   3.344 -/*Uses ID as index into array of flags.  If flag already set, resumes from
   3.345 - * end-label.  Else, sets flag and resumes normally.
   3.346 - */
   3.347 -void inline
   3.348 -handleSingleton( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.349 -                 VCilkSemEnv *semEnv )
   3.350 - {
   3.351 -   if( semEnv->singletonHasBeenExecutedFlags[ semReq->singletonID ] )
   3.352 -      requestingPr->nextInstrPt = semReq->endJumpPt;
   3.353 -   else
   3.354 -      semEnv->singletonHasBeenExecutedFlags[ semReq->singletonID ] = TRUE;
   3.355 -
   3.356 -   resume_procr( requestingPr, semEnv );
   3.357 - }
   3.358 -
   3.359 -
   3.360 -/*This executes the function in the masterVP, take the function
   3.361 - * pointer out of the request and call it, then resume the VP.
   3.362 - */
   3.363 -void inline
   3.364 -handleAtomic( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.365 -              VCilkSemEnv *semEnv )
   3.366 - {
   3.367 -   semReq->fnToExecInMaster( semReq->dataForFn );
   3.368 -   resume_procr( requestingPr, semEnv );
   3.369 - }
   3.370 -
   3.371 -/*First, it looks at the VP's semantic data, to see the highest transactionID
   3.372 - * that VP
   3.373 - * already has entered.  If the current ID is not larger, it throws an
   3.374 - * exception stating a bug in the code.
   3.375 - *Otherwise it puts the current ID
   3.376 - * there, and adds the ID to a linked list of IDs entered -- the list is
   3.377 - * used to check that exits are properly ordered.
   3.378 - *Next it is uses transactionID as index into an array of transaction
   3.379 - * structures.
   3.380 - *If the "VP_currently_executing" field is non-null, then put requesting VP
   3.381 - * into queue in the struct.  (At some point a holder will request
   3.382 - * end-transaction, which will take this VP from the queue and resume it.)
   3.383 - *If NULL, then write requesting into the field and resume.
   3.384 - */
   3.385 -void inline
   3.386 -handleTransStart( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.387 -                  VCilkSemEnv *semEnv )
   3.388 - { VCilkSemData *semData;
   3.389 -   TransListElem *nextTransElem;
   3.390 -
   3.391 -      //check ordering of entering transactions is correct
   3.392 -   semData = requestingPr->semanticData;
   3.393 -   if( semData->highestTransEntered > semReq->transID )
   3.394 -    {    //throw VMS exception, which shuts down VMS.
   3.395 -      VMS__throw_exception( "transID smaller than prev", requestingPr, NULL);
   3.396 -    }
   3.397 -      //add this trans ID to the list of transactions entered -- check when
   3.398 -      // end a transaction
   3.399 -   semData->highestTransEntered = semReq->transID;
   3.400 -   nextTransElem = VMS__malloc( sizeof(TransListElem) );
   3.401 -   nextTransElem->transID = semReq->transID;
   3.402 -   nextTransElem->nextTrans = semData->lastTransEntered;
   3.403 -   semData->lastTransEntered = nextTransElem;
   3.404 -
   3.405 -      //get the structure for this transaction ID
   3.406 -   VCilkTrans *
   3.407 -   transStruc = &(semEnv->transactionStrucs[ semReq->transID ]);
   3.408 -
   3.409 -   if( transStruc->VPCurrentlyExecuting == NULL )
   3.410 -    {
   3.411 -      transStruc->VPCurrentlyExecuting = requestingPr;
   3.412 -      resume_procr( requestingPr, semEnv );
   3.413 -    }
   3.414 -   else
   3.415 -    {    //note, might make future things cleaner if save request with VP and
   3.416 -         // add this trans ID to the linked list when gets out of queue.
   3.417 -         // but don't need for now, and lazy..
   3.418 -      writePrivQ( requestingPr, transStruc->waitingVPQ );
   3.419 -    }
   3.420 - }
   3.421 -
   3.422 -
   3.423 -/*Use the trans ID to get the transaction structure from the array.
   3.424 - *Look at VP_currently_executing to be sure it's same as requesting VP.
   3.425 - * If different, throw an exception, stating there's a bug in the code.
   3.426 - *Next, take the first element off the list of entered transactions.
   3.427 - * Check to be sure the ending transaction is the same ID as the next on
   3.428 - * the list.  If not, incorrectly nested so throw an exception.
   3.429 - *
   3.430 - *Next, get from the queue in the structure.
   3.431 - *If it's empty, set VP_currently_executing field to NULL and resume
   3.432 - * requesting VP.
   3.433 - *If get somethine, set VP_currently_executing to the VP from the queue, then
   3.434 - * resume both.
   3.435 - */
   3.436 -void inline
   3.437 -handleTransEnd( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.438 -                VCilkSemEnv *semEnv )
   3.439 - { VCilkSemData    *semData;
   3.440 -   VirtProcr     *waitingPr;
   3.441 -   VCilkTrans      *transStruc;
   3.442 -   TransListElem *lastTrans;
   3.443 -
   3.444 -   transStruc = &(semEnv->transactionStrucs[ semReq->transID ]);
   3.445 -
   3.446 -      //make sure transaction ended in same VP as started it.
   3.447 -   if( transStruc->VPCurrentlyExecuting != requestingPr )
   3.448 -    {
   3.449 -      VMS__throw_exception( "trans ended in diff VP", requestingPr, NULL );
   3.450 -    }
   3.451 -
   3.452 -      //make sure nesting is correct -- last ID entered should == this ID
   3.453 -   semData = requestingPr->semanticData;
   3.454 -   lastTrans = semData->lastTransEntered;
   3.455 -   if( lastTrans->transID != semReq->transID )
   3.456 -    {
   3.457 -      VMS__throw_exception( "trans incorrectly nested", requestingPr, NULL );
   3.458 -    }
   3.459 -
   3.460 -   semData->lastTransEntered = semData->lastTransEntered->nextTrans;
   3.461 -
   3.462 -
   3.463 -   waitingPr = readPrivQ( transStruc->waitingVPQ );
   3.464 -   transStruc->VPCurrentlyExecuting = waitingPr;
   3.465 -
   3.466 -   if( waitingPr != NULL )
   3.467 -      resume_procr( waitingPr, semEnv );
   3.468 -
   3.469 -   resume_procr( requestingPr, semEnv );
   3.470 - }
   3.471 +/*
   3.472 + * Copyright 2010  OpenSourceCodeStewardshipFoundation
   3.473 + *
   3.474 + * Licensed under BSD
   3.475 + */
   3.476 +
   3.477 +#include <stdio.h>
   3.478 +#include <stdlib.h>
   3.479 +
   3.480 +#include "VMS/Queue_impl/PrivateQueue.h"
   3.481 +#include "VCilk.h"
   3.482 +
   3.483 +
   3.484 +
   3.485 +//===========================================================================
   3.486 +void inline
   3.487 +handleSync( VirtProcr *requestingPr, VCilkSemEnv *semEnv );
   3.488 +
   3.489 +void inline
   3.490 +handleMalloc( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.491 +              VCilkSemEnv *semEnv );
   3.492 +void inline
   3.493 +handleFree( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.494 +            VCilkSemEnv *semEnv );
   3.495 +void inline
   3.496 +handleDissipate( VirtProcr *requestingPr, VCilkSemEnv *semEnv );
   3.497 +
   3.498 +void inline
   3.499 +handleSpawn( VMSReqst *req, VirtProcr *requestingPr, VCilkSemEnv *semEnv );
   3.500 +
   3.501 +void inline
   3.502 +dispatchSemReq( VMSReqst *req, VirtProcr *requestingPr, VCilkSemEnv *semEnv);
   3.503 +
   3.504 +void inline
   3.505 +handleTransEnd( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.506 +                VCilkSemEnv*semEnv);
   3.507 +void inline
   3.508 +handleTransStart( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.509 +                  VCilkSemEnv *semEnv );
   3.510 +void inline
   3.511 +handleAtomic( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.512 +              VCilkSemEnv *semEnv);
   3.513 +inline void
   3.514 +handleStartFnSingleton( VCilkSemReq *semReq, VirtProcr *reqstingPr,
   3.515 +                      VCilkSemEnv *semEnv );
   3.516 +inline void
   3.517 +handleEndFnSingleton( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.518 +                    VCilkSemEnv *semEnv );
   3.519 +inline void
   3.520 +handleStartDataSingleton( VCilkSemReq *semReq, VirtProcr *reqstingPr,
   3.521 +                      VCilkSemEnv *semEnv );
   3.522 +inline void
   3.523 +handleEndDataSingleton( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.524 +                    VCilkSemEnv *semEnv );
   3.525 +
   3.526 +void inline
   3.527 +resume_procr( VirtProcr *procr, VCilkSemEnv *semEnv );
   3.528 +
   3.529 +//===========================================================================
   3.530 +
   3.531 +
   3.532 +//============================== Scheduler ==================================
   3.533 +//
   3.534 +/*For VCilk, scheduling a slave simply takes the next work-unit off the
   3.535 + * ready-to-go work-unit queue and assigns it to the slaveToSched.
   3.536 + *If the ready-to-go work-unit queue is empty, then nothing to schedule
   3.537 + * to the slave -- return FALSE to let Master loop know scheduling that
   3.538 + * slave failed.
   3.539 + */
   3.540 +VirtProcr *
   3.541 +VCilk__schedule_virt_procr( void *_semEnv, int coreNum )
   3.542 + { VirtProcr   *schedPr;
   3.543 +   VCilkSemEnv *semEnv;
   3.544 +
   3.545 +   semEnv = (VCilkSemEnv *)_semEnv;
   3.546 +
   3.547 +   schedPr = readPrivQ( semEnv->readyVPQs[coreNum] );
   3.548 +      //Note, using a non-blocking queue -- it returns NULL if queue empty
   3.549 +
   3.550 +   return( schedPr );
   3.551 + }
   3.552 +
   3.553 +
   3.554 +//===========================  Request Handler  =============================
   3.555 +//
   3.556 +/*Will get requests to send, to receive, and to create new processors.
   3.557 + * Upon send, check the hash to see if a receive is waiting.
   3.558 + * Upon receive, check hash to see if a send has already happened.
   3.559 + * When other is not there, put in.  When other is there, the comm.
   3.560 + *  completes, which means the receiver P gets scheduled and
   3.561 + *  picks up right after the receive request.  So make the work-unit
   3.562 + *  and put it into the queue of work-units ready to go.
   3.563 + * Other request is create a new Processor, with the function to run in the
   3.564 + *  Processor, and initial data.
   3.565 + */
   3.566 +void
   3.567 +VCilk__Request_Handler( VirtProcr *requestingPr, void *_semEnv )
   3.568 + { VCilkSemEnv *semEnv;
   3.569 +   VMSReqst    *req;
   3.570 +   VCilkSemReq *semReq;
   3.571 + 
   3.572 +
   3.573 +   semEnv = (VCilkSemEnv *)_semEnv;
   3.574 +
   3.575 +   req = VMS__take_next_request_out_of( requestingPr );
   3.576 +   
   3.577 +   while( req != NULL )
   3.578 +    {
   3.579 +      switch( req->reqType )
   3.580 +       { case semantic:     dispatchSemReq( req, requestingPr, semEnv );
   3.581 +            break;
   3.582 +         case createReq:    //create request has to come as a VMS request,
   3.583 +                            // to allow MasterLoop to do stuff before gets
   3.584 +                            // here, and maybe also stuff after all requests
   3.585 +                            // done -- however, can still attach semantic
   3.586 +                            // req data to req.
   3.587 +                            handleSpawn(          req, requestingPr, semEnv);
   3.588 +            break;
   3.589 +         case dissipate:    handleDissipate(           requestingPr, semEnv);
   3.590 +            break;
   3.591 +         case VMSSemantic:  VMS__handle_VMSSemReq(req, requestingPr, semEnv,
   3.592 +                                                              &resume_procr);
   3.593 +            break;
   3.594 +         default:
   3.595 +            break;
   3.596 +       }
   3.597 +      
   3.598 +      DoneHandlingReqst:
   3.599 +
   3.600 +      req = VMS__take_next_request_out_of( requestingPr );
   3.601 +    } //while( req != NULL )
   3.602 + }
   3.603 +
   3.604 +void inline
   3.605 +dispatchSemReq( VMSReqst *req, VirtProcr *reqPr, VCilkSemEnv *semEnv )
   3.606 + { VCilkSemReq *semReq;
   3.607 +
   3.608 +   semReq = VMS__take_sem_reqst_from(req);
   3.609 +
   3.610 +   if( semReq == NULL ) return;
   3.611 +   switch( semReq->reqType )
   3.612 +    {
   3.613 +      case syncReq:         handleSync(                 reqPr, semEnv );
   3.614 +         break;
   3.615 +      case mallocReq:       handleMalloc(       semReq, reqPr, semEnv );
   3.616 +         break;
   3.617 +      case freeReq:         handleFree(         semReq, reqPr, semEnv );
   3.618 +         break;
   3.619 +      case singleton_fn_start:  handleStartFnSingleton(semReq, reqPr, semEnv);
   3.620 +         break;
   3.621 +      case singleton_fn_end:    handleEndFnSingleton(  semReq, reqPr, semEnv);
   3.622 +         break;
   3.623 +      case singleton_data_start:handleStartDataSingleton(semReq,reqPr,semEnv);
   3.624 +         break;
   3.625 +      case singleton_data_end:  handleEndDataSingleton(semReq, reqPr, semEnv);
   3.626 +         break;
   3.627 +      case atomic:          handleAtomic(       semReq, reqPr, semEnv );
   3.628 +         break;
   3.629 +      case trans_start:     handleTransStart(   semReq, reqPr, semEnv );
   3.630 +         break;
   3.631 +      case trans_end:       handleTransEnd(     semReq, reqPr, semEnv );
   3.632 +         break;
   3.633 +    }
   3.634 +   //NOTE: semantic request data strucs allocated on stack in VCilk Lib calls
   3.635 + }
   3.636 +
   3.637 +
   3.638 +
   3.639 +//=========================== Request Handlers ==============================
   3.640 +void inline
   3.641 +resume_procr( VirtProcr *procr, VCilkSemEnv *semEnv )
   3.642 + {
   3.643 +   writePrivQ( procr, semEnv->readyVPQs[ procr->coreAnimatedBy] );
   3.644 + }
   3.645 +
   3.646 +
   3.647 +
   3.648 +
   3.649 +/* check if list of live children is empty.
   3.650 + * If yes, then resume.
   3.651 + * If no, then set sync-pending flag.
   3.652 + */
   3.653 +inline void
   3.654 +handleSync( VirtProcr *requestingPr, VCilkSemEnv *semEnv )
   3.655 + {
   3.656 +         Meas_startSync
   3.657 +   if(((VCilkSemData *)(requestingPr->semanticData))->numLiveChildren  == 0 )
   3.658 +    { //no live children to wait for
   3.659 +      resume_procr( requestingPr, semEnv );
   3.660 +    }
   3.661 +   else
   3.662 +    {
   3.663 +      ((VCilkSemData *)(requestingPr->semanticData))->syncPending = TRUE;
   3.664 +    }
   3.665 +         Meas_endSync
   3.666 + }
   3.667 +
   3.668 +/*
   3.669 + */
   3.670 +inline void
   3.671 +handleMalloc( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.672 +              VCilkSemEnv *semEnv )
   3.673 + { void *ptr;
   3.674 +   
   3.675 +   ptr = VMS__malloc( semReq->sizeToMalloc );
   3.676 +   requestingPr->dataRetFromReq = ptr;
   3.677 +   resume_procr( requestingPr, semEnv );
   3.678 + }
   3.679 +
   3.680 +/*
   3.681 + */
   3.682 +void inline
   3.683 +handleFree( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.684 +            VCilkSemEnv *semEnv )
   3.685 + {
   3.686 +   VMS__free( semReq->ptrToFree );
   3.687 +   resume_procr( requestingPr, semEnv );
   3.688 + }
   3.689 +
   3.690 +
   3.691 +//============================== VMS requests ===============================
   3.692 +/*Re-use this in the entry-point fn
   3.693 + */
   3.694 +inline VirtProcr *
   3.695 +VCilk__create_procr_helper( VirtProcrFnPtr fnPtr, void  *initData,
   3.696 +   VirtProcr *requestingPr, VCilkSemEnv *semEnv,  int32  coreToScheduleOnto )
   3.697 + { VirtProcr    *newPr;
   3.698 +   VCilkSemData *semData;
   3.699 +
   3.700 +      //This is running in master, so use internal version
   3.701 +   newPr = VMS__create_procr( fnPtr, initData );
   3.702 +
   3.703 +   semData = VMS__malloc( sizeof(VCilkSemData) );
   3.704 +
   3.705 +   semData->numLiveChildren = 0;
   3.706 +   semData->parentPr        = requestingPr;
   3.707 +   semData->syncPending     = FALSE;
   3.708 +   
   3.709 +   semData->highestTransEntered = -1;
   3.710 +   semData->lastTransEntered    = NULL;
   3.711 +
   3.712 +   newPr->semanticData = semData;
   3.713 +
   3.714 +   /* increase the number of live children of requester.
   3.715 +    */
   3.716 +   if( requestingPr != NULL ) //NULL when creating seed procr
   3.717 +     ((VCilkSemData *)(requestingPr->semanticData))->numLiveChildren +=1;
   3.718 +
   3.719 +   semEnv->numVirtPr += 1;
   3.720 +
   3.721 +   //=================== Assign new processor to a core =====================
   3.722 +   #ifdef SEQUENTIAL
   3.723 +   newPr->coreAnimatedBy = 0;
   3.724 +
   3.725 +   #else
   3.726 +
   3.727 +   if(coreToScheduleOnto < 0 || coreToScheduleOnto >= NUM_CORES )
   3.728 +    {    //out-of-range, so round-robin assignment
   3.729 +      newPr->coreAnimatedBy = semEnv->nextCoreToGetNewPr;
   3.730 +
   3.731 +      if( semEnv->nextCoreToGetNewPr >= NUM_CORES - 1 )
   3.732 +          semEnv->nextCoreToGetNewPr  = 0;
   3.733 +      else
   3.734 +          semEnv->nextCoreToGetNewPr += 1;
   3.735 +    }
   3.736 +   else //core num in-range, so use it
   3.737 +    { newPr->coreAnimatedBy = coreToScheduleOnto;
   3.738 +    }
   3.739 +   #endif
   3.740 +   //========================================================================
   3.741 +
   3.742 +   return newPr;
   3.743 + }
   3.744 +
   3.745 +
   3.746 +void inline
   3.747 +handleSpawn( VMSReqst *req, VirtProcr *requestingPr, VCilkSemEnv *semEnv )
   3.748 + { VCilkSemReq *semReq;
   3.749 +   VirtProcr    *newPr;
   3.750 +
   3.751 +         Meas_startSpawn
   3.752 +   semReq = VMS__take_sem_reqst_from( req );
   3.753 +
   3.754 +   newPr = VCilk__create_procr_helper( semReq->fnPtr, semReq->initData,
   3.755 +                             requestingPr, semEnv, semReq->coreToSpawnOnto );
   3.756 +
   3.757 +      //For VPThread, caller needs ptr to created processor returned to it
   3.758 +   requestingPr->dataRetFromReq = newPr;
   3.759 +
   3.760 +   resume_procr( newPr,        semEnv );
   3.761 +   resume_procr( requestingPr, semEnv );
   3.762 +         Meas_endSpawn
   3.763 + }
   3.764 +
   3.765 +
   3.766 +
   3.767 +/*get parentVP & remove dissipator from parent's live children.
   3.768 + *If this was last live child, check "sync pending" flag
   3.769 + *-- if set, then resume the parentVP.
   3.770 + */
   3.771 +void inline
   3.772 +handleDissipate( VirtProcr *requestingPr, VCilkSemEnv *semEnv )
   3.773 + {
   3.774 +   VirtProcr *
   3.775 +   parentPr = ((VCilkSemData *)
   3.776 +               (requestingPr->semanticData))->parentPr;
   3.777 +   if( parentPr == NULL ) //means this is seed processor being dissipated
   3.778 +    { //Just act normally, except don't deal with parent
   3.779 +      // VMS__Free is implemented to ignore requests to free data from
   3.780 +      // outside VMS, so all this processor's non-VMS allocated data will
   3.781 +      // remain and be cleaned up outside
   3.782 +    }
   3.783 +   else
   3.784 +    {
   3.785 +      ((VCilkSemData *)(parentPr->semanticData))->numLiveChildren -= 1;
   3.786 +      if( ((VCilkSemData *)
   3.787 +            (parentPr->semanticData))->numLiveChildren <= 0 )
   3.788 +       { //this was last live child of parent
   3.789 +         if( ((VCilkSemData *)
   3.790 +               (parentPr->semanticData))->syncPending == TRUE )
   3.791 +          { //was waiting for last child to dissipate, so resume it
   3.792 +            ((VCilkSemData *)
   3.793 +              (parentPr->semanticData))->syncPending = FALSE;
   3.794 +            resume_procr( parentPr, semEnv );
   3.795 +          }
   3.796 +       }
   3.797 +    }
   3.798 +
   3.799 +   VMS__free( requestingPr->semanticData );
   3.800 +   
   3.801 +       //Now do normal dissipate
   3.802 +   
   3.803 +       //call VMS to free_all AppVP state -- stack and so on
   3.804 +   VMS__dissipate_procr( requestingPr );
   3.805 +
   3.806 +   semEnv->numVirtPr -= 1;
   3.807 +   if( semEnv->numVirtPr == 0 )
   3.808 +    {    //no more work, so shutdown
   3.809 +      VMS__shutdown();
   3.810 +    }
   3.811 + }
   3.812 +
   3.813 +
   3.814 +//=============================== Atomic ====================================
   3.815 +//
   3.816 +/*Uses ID as index into array of flags.  If flag already set, resumes from
   3.817 + * end-label.  Else, sets flag and resumes normally.
   3.818 + */
   3.819 +void inline
   3.820 +handleStartSingleton_helper( VCilkSingleton *singleton, VirtProcr *reqstingPr,
   3.821 +                             VCilkSemEnv    *semEnv )
   3.822 + {
   3.823 +   if( singleton->hasFinished )
   3.824 +    {    //the code that sets the flag to true first sets the end instr addr
   3.825 +      reqstingPr->dataRetFromReq = singleton->endInstrAddr;
   3.826 +      resume_procr( reqstingPr, semEnv );
   3.827 +      return;
   3.828 +    }
   3.829 +   else if( singleton->hasBeenStarted )
   3.830 +    {    //singleton is in-progress in a diff slave, so wait for it to finish
   3.831 +      writePrivQ(reqstingPr, singleton->waitQ );
   3.832 +      return;
   3.833 +    }
   3.834 +   else
   3.835 +    {    //hasn't been started, so this is the first attempt at the singleton
   3.836 +      singleton->hasBeenStarted = TRUE;
   3.837 +      reqstingPr->dataRetFromReq = 0x0;
   3.838 +      resume_procr( reqstingPr, semEnv );
   3.839 +      return;
   3.840 +    }
   3.841 + }
   3.842 +void inline
   3.843 +handleStartFnSingleton( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.844 +                      VCilkSemEnv *semEnv )
   3.845 + { VCilkSingleton *singleton;
   3.846 +
   3.847 +   singleton = &(semEnv->fnSingletons[ semReq->singletonID ]);
   3.848 +   handleStartSingleton_helper( singleton, requestingPr, semEnv );
   3.849 + }
   3.850 +void inline
   3.851 +handleStartDataSingleton( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.852 +                      VCilkSemEnv *semEnv )
   3.853 + { VCilkSingleton *singleton;
   3.854 +
   3.855 +   if( *(semReq->singletonPtrAddr) == NULL )
   3.856 +    { singleton                 = VMS__malloc( sizeof(VCilkSingleton) );
   3.857 +      singleton->waitQ          = makeVMSPrivQ();
   3.858 +      singleton->endInstrAddr   = 0x0;
   3.859 +      singleton->hasBeenStarted = FALSE;
   3.860 +      singleton->hasFinished    = FALSE;
   3.861 +      *(semReq->singletonPtrAddr)  = singleton;
   3.862 +    }
   3.863 +   else
   3.864 +      singleton = *(semReq->singletonPtrAddr);
   3.865 +   handleStartSingleton_helper( singleton, requestingPr, semEnv );
   3.866 + }
   3.867 +
   3.868 +
   3.869 +void inline
   3.870 +handleEndSingleton_helper( VCilkSingleton *singleton, VirtProcr *requestingPr,
   3.871 +                           VCilkSemEnv    *semEnv )
   3.872 + { PrivQueueStruc *waitQ;
   3.873 +   int32           numWaiting, i;
   3.874 +   VirtProcr      *resumingPr;
   3.875 +
   3.876 +   if( singleton->hasFinished )
   3.877 +    { //by definition, only one slave should ever be able to run end singleton
   3.878 +      // so if this is true, is an error
   3.879 +      //VMS__throw_exception( "singleton code ran twice", requestingPr, NULL);
   3.880 +    }
   3.881 +
   3.882 +   singleton->hasFinished = TRUE;
   3.883 +   waitQ = singleton->waitQ;
   3.884 +   numWaiting = numInPrivQ( waitQ );
   3.885 +   for( i = 0; i < numWaiting; i++ )
   3.886 +    {    //they will resume inside start singleton, then jmp to end singleton
   3.887 +      resumingPr = readPrivQ( waitQ );
   3.888 +      resumingPr->dataRetFromReq = singleton->endInstrAddr;
   3.889 +      resume_procr( resumingPr, semEnv );
   3.890 +    }
   3.891 +
   3.892 +   resume_procr( requestingPr, semEnv );
   3.893 +
   3.894 +}
   3.895 +void inline
   3.896 +handleEndFnSingleton( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.897 +                        VCilkSemEnv *semEnv )
   3.898 + {
   3.899 +   VCilkSingleton   *singleton;
   3.900 +
   3.901 +   singleton = &(semEnv->fnSingletons[ semReq->singletonID ]);
   3.902 +   handleEndSingleton_helper( singleton, requestingPr, semEnv );
   3.903 +  }
   3.904 +void inline
   3.905 +handleEndDataSingleton( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.906 +                        VCilkSemEnv *semEnv )
   3.907 + {
   3.908 +   VCilkSingleton   *singleton;
   3.909 +
   3.910 +   singleton = *(semReq->singletonPtrAddr);
   3.911 +   handleEndSingleton_helper( singleton, requestingPr, semEnv );
   3.912 +  }
   3.913 +
   3.914 +
   3.915 +/*This executes the function in the masterVP, take the function
   3.916 + * pointer out of the request and call it, then resume the VP.
   3.917 + */
   3.918 +void inline
   3.919 +handleAtomic( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.920 +              VCilkSemEnv *semEnv )
   3.921 + {
   3.922 +   semReq->fnToExecInMaster( semReq->dataForFn );
   3.923 +   resume_procr( requestingPr, semEnv );
   3.924 + }
   3.925 +
   3.926 +/*First, it looks at the VP's semantic data, to see the highest transactionID
   3.927 + * that VP
   3.928 + * already has entered.  If the current ID is not larger, it throws an
   3.929 + * exception stating a bug in the code.
   3.930 + *Otherwise it puts the current ID
   3.931 + * there, and adds the ID to a linked list of IDs entered -- the list is
   3.932 + * used to check that exits are properly ordered.
   3.933 + *Next it is uses transactionID as index into an array of transaction
   3.934 + * structures.
   3.935 + *If the "VP_currently_executing" field is non-null, then put requesting VP
   3.936 + * into queue in the struct.  (At some point a holder will request
   3.937 + * end-transaction, which will take this VP from the queue and resume it.)
   3.938 + *If NULL, then write requesting into the field and resume.
   3.939 + */
   3.940 +void inline
   3.941 +handleTransStart( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.942 +                  VCilkSemEnv *semEnv )
   3.943 + { VCilkSemData *semData;
   3.944 +   TransListElem *nextTransElem;
   3.945 +
   3.946 +      //check ordering of entering transactions is correct
   3.947 +   semData = requestingPr->semanticData;
   3.948 +   if( semData->highestTransEntered > semReq->transID )
   3.949 +    {    //throw VMS exception, which shuts down VMS.
   3.950 +      VMS__throw_exception( "transID smaller than prev", requestingPr, NULL);
   3.951 +    }
   3.952 +      //add this trans ID to the list of transactions entered -- check when
   3.953 +      // end a transaction
   3.954 +   semData->highestTransEntered = semReq->transID;
   3.955 +   nextTransElem = VMS__malloc( sizeof(TransListElem) );
   3.956 +   nextTransElem->transID = semReq->transID;
   3.957 +   nextTransElem->nextTrans = semData->lastTransEntered;
   3.958 +   semData->lastTransEntered = nextTransElem;
   3.959 +
   3.960 +      //get the structure for this transaction ID
   3.961 +   VCilkTrans *
   3.962 +   transStruc = &(semEnv->transactionStrucs[ semReq->transID ]);
   3.963 +
   3.964 +   if( transStruc->VPCurrentlyExecuting == NULL )
   3.965 +    {
   3.966 +      transStruc->VPCurrentlyExecuting = requestingPr;
   3.967 +      resume_procr( requestingPr, semEnv );
   3.968 +    }
   3.969 +   else
   3.970 +    {    //note, might make future things cleaner if save request with VP and
   3.971 +         // add this trans ID to the linked list when gets out of queue.
   3.972 +         // but don't need for now, and lazy..
   3.973 +      writePrivQ( requestingPr, transStruc->waitingVPQ );
   3.974 +    }
   3.975 + }
   3.976 +
   3.977 +
   3.978 +/*Use the trans ID to get the transaction structure from the array.
   3.979 + *Look at VP_currently_executing to be sure it's same as requesting VP.
   3.980 + * If different, throw an exception, stating there's a bug in the code.
   3.981 + *Next, take the first element off the list of entered transactions.
   3.982 + * Check to be sure the ending transaction is the same ID as the next on
   3.983 + * the list.  If not, incorrectly nested so throw an exception.
   3.984 + *
   3.985 + *Next, get from the queue in the structure.
   3.986 + *If it's empty, set VP_currently_executing field to NULL and resume
   3.987 + * requesting VP.
   3.988 + *If get somethine, set VP_currently_executing to the VP from the queue, then
   3.989 + * resume both.
   3.990 + */
   3.991 +void inline
   3.992 +handleTransEnd( VCilkSemReq *semReq, VirtProcr *requestingPr,
   3.993 +                VCilkSemEnv *semEnv )
   3.994 + { VCilkSemData    *semData;
   3.995 +   VirtProcr     *waitingPr;
   3.996 +   VCilkTrans      *transStruc;
   3.997 +   TransListElem *lastTrans;
   3.998 +
   3.999 +   transStruc = &(semEnv->transactionStrucs[ semReq->transID ]);
  3.1000 +
  3.1001 +      //make sure transaction ended in same VP as started it.
  3.1002 +   if( transStruc->VPCurrentlyExecuting != requestingPr )
  3.1003 +    {
  3.1004 +      VMS__throw_exception( "trans ended in diff VP", requestingPr, NULL );
  3.1005 +    }
  3.1006 +
  3.1007 +      //make sure nesting is correct -- last ID entered should == this ID
  3.1008 +   semData = requestingPr->semanticData;
  3.1009 +   lastTrans = semData->lastTransEntered;
  3.1010 +   if( lastTrans->transID != semReq->transID )
  3.1011 +    {
  3.1012 +      VMS__throw_exception( "trans incorrectly nested", requestingPr, NULL );
  3.1013 +    }
  3.1014 +
  3.1015 +   semData->lastTransEntered = semData->lastTransEntered->nextTrans;
  3.1016 +
  3.1017 +
  3.1018 +   waitingPr = readPrivQ( transStruc->waitingVPQ );
  3.1019 +   transStruc->VPCurrentlyExecuting = waitingPr;
  3.1020 +
  3.1021 +   if( waitingPr != NULL )
  3.1022 +      resume_procr( waitingPr, semEnv );
  3.1023 +
  3.1024 +   resume_procr( requestingPr, semEnv );
  3.1025 + }

     4.1 --- a/VCilk__DESIGN_NOTES.txt	Tue Nov 16 15:59:23 2010 +0100
     4.2 +++ b/VCilk__DESIGN_NOTES.txt	Wed May 11 15:29:58 2011 +0200
     4.3 @@ -1,28 +1,28 @@
     4.4 -
     4.5 -
     4.6 -
     4.7 - the design:
     4.8 -
     4.9 -Only has the spawn and sync calls, nothing else.
    4.10 -
    4.11 -For spawn, creates a new VP
    4.12 -
    4.13 -For sync, waits for all VPs created by itself to dissipate.
    4.14 -
    4.15 -To implement these, in request handler:
    4.16 -For spawn:
    4.17 - create new virtual processor
    4.18 - In requester, add newly created to the list of live children
    4.19 - In newly created, add pointer to requester, as the parentVP
    4.20 -
    4.21 -For Dissipate:
    4.22 - remove dissipator from its parent's list of live children.
    4.23 - If this was last in list, check "sync pending" flag 
    4.24 - -- if set, then resume the parentVP.
    4.25 -
    4.26 -For Sync:
    4.27 - check if list of live of children is empty.  
    4.28 - If yes, then resume.  
    4.29 - If no, then set sync-pending flag and remain suspended
    4.30 -
    4.31 -That's it.  Quick and simple,
    4.32 +
    4.33 
    4.34 +
    4.35 
    4.36 +
    4.37 
    4.38 + the design:
    4.39 
    4.40 +
    4.41 
    4.42 +Only has the spawn and sync calls, nothing else.
    4.43 
    4.44 +
    4.45 
    4.46 +For spawn, creates a new VP
    4.47 
    4.48 +
    4.49 
    4.50 +For sync, waits for all VPs created by itself to dissipate.
    4.51 
    4.52 +
    4.53 
    4.54 +To implement these, in request handler:
    4.55 
    4.56 +For spawn:
    4.57 
    4.58 + create new virtual processor
    4.59 
    4.60 + In requester, add newly created to the list of live children
    4.61 
    4.62 + In newly created, add pointer to requester, as the parentVP
    4.63 
    4.64 +
    4.65 
    4.66 +For Dissipate:
    4.67 
    4.68 + remove dissipator from its parent's list of live children.
    4.69 
    4.70 + If this was last in list, check "sync pending" flag 
    4.71 
    4.72 + -- if set, then resume the parentVP.
    4.73 
    4.74 +
    4.75 
    4.76 +For Sync:
    4.77 
    4.78 + check if list of live of children is empty.  
    4.79 
    4.80 + If yes, then resume.  
    4.81 
    4.82 + If no, then set sync-pending flag and remain suspended
    4.83 
    4.84 +
    4.85 
    4.86 +That's it.  Quick and simple,
    4.87 

     5.1 --- a/VCilk_lib.c	Tue Nov 16 15:59:23 2010 +0100
     5.2 +++ b/VCilk_lib.c	Wed May 11 15:29:58 2011 +0200
     5.3 @@ -1,429 +1,539 @@
     5.4 -/*
     5.5 - * Copyright 2010  OpenSourceCodeStewardshipFoundation
     5.6 - *
     5.7 - * Licensed under BSD
     5.8 - */
     5.9 -
    5.10 -#include <stdio.h>
    5.11 -#include <stdlib.h>
    5.12 -
    5.13 -#include "VMS/VMS.h"
    5.14 -#include "VCilk.h"
    5.15 -#include "VMS/Queue_impl/PrivateQueue.h"
    5.16 -#include "VMS/Hash_impl/PrivateHash.h"
    5.17 -
    5.18 -
    5.19 -//==========================================================================
    5.20 -
    5.21 -void
    5.22 -VCilk__init();
    5.23 -
    5.24 -void
    5.25 -VCilk__init_Seq();
    5.26 -
    5.27 -void
    5.28 -VCilk__init_Helper();
    5.29 -//==========================================================================
    5.30 -
    5.31 -
    5.32 -/*TODO: Q: dealing with library f()s and DKU vs WT vs FoR
    5.33 - * (still want to do FoR, with time-lines as syntax, could be super cool)
    5.34 - * A: thinking pin the coreLoops for all of BLIS -- let Master arbitrate
    5.35 - * among library, DKU, WT, FoR -- all the patterns in terms of virtual
    5.36 - * processors (or equivalently work-units), so Master picks which virt procr
    5.37 - * from which portions of app (DKU, WT, FoR) onto which sched slots
    5.38 - *Might even do hierarchy of masters -- group of sched slots for each core
    5.39 - * has its own master, that keeps generated work local
    5.40 - * single-reader-single-writer sync everywhere -- no atomic primitives (but
    5.41 - * memory fences on architectures that need them)
    5.42 - * Might have the different schedulers talk to each other, to negotiate
    5.43 - * larger-grain sharing of resources, according to predicted critical
    5.44 - * path, and expansion of work
    5.45 - */
    5.46 -
    5.47 -
    5.48 -
    5.49 -//===========================================================================
    5.50 -
    5.51 -
    5.52 -/*These are the library functions *called in the application*
    5.53 - * 
    5.54 - *There's a pattern for the outside sequential code to interact with the
    5.55 - * VMS_HW code.
    5.56 - *The VMS_HW system is inside a boundary..  every VCilk system is in its
    5.57 - * own directory that contains the functions for each of the processor types.
    5.58 - * One of the processor types is the "seed" processor that starts the
    5.59 - * cascade of creating all the processors that do the work.
    5.60 - *So, in the directory is a file called "EntryPoint.c" that contains the
    5.61 - * function, named appropriately to the work performed, that the outside
    5.62 - * sequential code calls.  This function follows a pattern:
    5.63 - *1) it calls VCilk__init()
    5.64 - *2) it creates the initial data for the seed processor, which is passed
    5.65 - *    in to the function
    5.66 - *3) it creates the seed VCilk processor, with the data to start it with.
    5.67 - *4) it calls startVCilkThenWaitUntilWorkDone
    5.68 - *5) it gets the returnValue from the transfer struc and returns that
    5.69 - *    from the function
    5.70 - *
    5.71 - *For now, a new VCilk system has to be created via VCilk__init every
    5.72 - * time an entry point function is called -- later, might add letting the
    5.73 - * VCilk system be created once, and let all the entry points just reuse
    5.74 - * it -- want to be as simple as possible now, and see by using what makes
    5.75 - * sense for later..
    5.76 - */
    5.77 -
    5.78 -
    5.79 -
    5.80 -//===========================================================================
    5.81 -
    5.82 -/*This is the "border crossing" function -- the thing that crosses from the
    5.83 - * outside world, into the VMS_HW world.  It initializes and starts up the
    5.84 - * VMS system, then creates one processor from the specified function and
    5.85 - * puts it into the readyQ.  From that point, that one function is resp.
    5.86 - * for creating all the other processors, that then create others, and so
    5.87 - * forth.
    5.88 - *When all the processors, including the seed, have dissipated, then this
    5.89 - * function returns.  The results will have been written by side-effect via
    5.90 - * pointers read from, or written into initData.
    5.91 - *
    5.92 - *NOTE: no Threads should exist in the outside program that might touch
    5.93 - * any of the data reachable from initData passed in to here
    5.94 - */
    5.95 -void
    5.96 -VCilk__create_seed_procr_and_do_work( VirtProcrFnPtr fnPtr, void *initData )
    5.97 - { VCilkSemEnv *semEnv;
    5.98 -   VirtProcr *seedPr;
    5.99 -
   5.100 -   #ifdef SEQUENTIAL
   5.101 -   VCilk__init_Seq();  //debug sequential exe
   5.102 -   #else
   5.103 -   VCilk__init();      //normal multi-thd
   5.104 -   #endif
   5.105 -   semEnv = _VMSMasterEnv->semanticEnv;
   5.106 -
   5.107 -      //VCilk starts with one processor, which is put into initial environ,
   5.108 -      // and which then calls create() to create more, thereby expanding work
   5.109 -   seedPr = VCilk__create_procr_helper( fnPtr, initData, NULL, semEnv, -1 );
   5.110 -   resume_procr( seedPr, semEnv );
   5.111 -
   5.112 -   #ifdef SEQUENTIAL
   5.113 -   VMS__start_the_work_then_wait_until_done_Seq();  //debug sequential exe
   5.114 -   #else
   5.115 -   VMS__start_the_work_then_wait_until_done();      //normal multi-thd
   5.116 -   #endif
   5.117 -
   5.118 -   VCilk__cleanup_at_end_of_shutdown();
   5.119 - }
   5.120 -
   5.121 -
   5.122 -int32 inline
   5.123 -VCilk__giveMinWorkUnitCycles( float32 percentOverhead )
   5.124 - {
   5.125 -   return MIN_WORK_UNIT_CYCLES;
   5.126 - }
   5.127 -
   5.128 -int32
   5.129 -VCilk__giveIdealNumWorkUnits()
   5.130 - {
   5.131 -   return NUM_SCHED_SLOTS * NUM_CORES;
   5.132 - }
   5.133 -
   5.134 -/*To measure how long a primitive operation takes, when calculating number of
   5.135 - * sub-tasks to divide into.
   5.136 - * For now, use TSC -- later, make these two macros with assembly that first
   5.137 - * saves jump point, and second jumps back several times to get reliable time
   5.138 - */
   5.139 -void inline
   5.140 -VCilk__start_primitive()
   5.141 - { //int32 *saveAddr;
   5.142 -   //saveAddr = &(((VCilkSemEnv *)(_VMSMasterEnv->semanticEnv))->primitiveStartTime);
   5.143 -   saveLowTimeStampCountInto( (((VCilkSemEnv *)
   5.144 -                        (_VMSMasterEnv->semanticEnv))->primitiveStartTime) );
   5.145 - }
   5.146 -
   5.147 -/*Just quick and dirty for now -- make reliable later
   5.148 - * will want this to jump back several times -- to be sure cache is warm
   5.149 - * because don't want comm time included in calc-time measurement -- and
   5.150 - * also to throw out any "weird" values due to OS interrupt or TSC rollover
   5.151 - */
   5.152 -int32 inline
   5.153 -VCilk__end_primitive_and_give_cycles()
   5.154 - { int32 endTime, startTime;
   5.155 -   //TODO: fix by repeating time-measurement
   5.156 -   saveLowTimeStampCountInto( endTime );
   5.157 -   startTime = ( (VCilkSemEnv *)
   5.158 -                 (_VMSMasterEnv->semanticEnv))->primitiveStartTime;
   5.159 -   return (endTime - startTime);
   5.160 - }
   5.161 -
   5.162 -//===========================================================================
   5.163 -//
   5.164 -/*Initializes all the data-structures for a VCilk system -- but doesn't
   5.165 - * start it running yet!
   5.166 - *
   5.167 - *This and its callees run in main thread outside VMS
   5.168 - * 
   5.169 - *This sets up the semantic layer over the VMS system
   5.170 - *
   5.171 - *First, calls VMS_Setup, then creates own environment, making it ready
   5.172 - * for creating the seed processor and then starting the work.
   5.173 - */
   5.174 -void
   5.175 -VCilk__init()
   5.176 - {
   5.177 -   VMS__init();
   5.178 -      //masterEnv, a global var, now is partially set up by init_VMS
   5.179 -
   5.180 -   VCilk__init_Helper();
   5.181 - }
   5.182 -
   5.183 -void
   5.184 -VCilk__init_Seq()
   5.185 - {
   5.186 -   VMS__init_Seq();
   5.187 -      //masterEnv, a global var, now is partially set up by init_VMS
   5.188 -
   5.189 -   VCilk__init_Helper();
   5.190 - }
   5.191 -
   5.192 -/*Runs in main thread before VMS system starts
   5.193 - */
   5.194 -void
   5.195 -VCilk__init_Helper()
   5.196 - { VCilkSemEnv     *semanticEnv;
   5.197 -   PrivQueueStruc **readyVPQs;
   5.198 -   int              coreIdx;
   5.199 - 
   5.200 -      //Hook up the semantic layer's plug-ins to the Master virt procr
   5.201 -   _VMSMasterEnv->requestHandler = &VCilk__Request_Handler;
   5.202 -   _VMSMasterEnv->slaveScheduler = &VCilk__schedule_virt_procr;
   5.203 -
   5.204 -      //create the semantic layer's environment (all its data) and add to
   5.205 -      // the master environment
   5.206 -   semanticEnv = VMS__malloc( sizeof( VCilkSemEnv ) );
   5.207 -   _VMSMasterEnv->semanticEnv = semanticEnv;
   5.208 -
   5.209 -      //create the ready queue, hash tables used for pairing send to receive
   5.210 -      // and so forth
   5.211 -      //TODO: add hash tables for pairing sends with receives, and
   5.212 -      // initialize the data ownership system
   5.213 -   readyVPQs = VMS__malloc( NUM_CORES * sizeof(PrivQueueStruc *) );
   5.214 -
   5.215 -   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
   5.216 -    {
   5.217 -      readyVPQs[ coreIdx ] = makeVMSPrivQ();
   5.218 -    }
   5.219 -   
   5.220 -   semanticEnv->readyVPQs = readyVPQs;
   5.221 -   
   5.222 -   semanticEnv->nextCoreToGetNewPr = 0;
   5.223 -
   5.224 -   //TODO: bug -- turn these arrays into dyn arrays to eliminate limit
   5.225 -   //semanticEnv->singletonHasBeenExecutedFlags = makeDynArrayInfo( );
   5.226 -   //semanticEnv->transactionStrucs = makeDynArrayInfo( );
   5.227 -   //something like: setHighestIdx( dynArrayInfo, NUM_STRUCS_IN_SEM_ENV )
   5.228 -   int32 i;
   5.229 -   for( i = 0; i < NUM_STRUCS_IN_SEM_ENV; i++ )
   5.230 -    {
   5.231 -      semanticEnv->singletonHasBeenExecutedFlags[i] = FALSE;
   5.232 -      semanticEnv->transactionStrucs[i].waitingVPQ  = makeVMSPrivQ();
   5.233 -    }
   5.234 -
   5.235 - }
   5.236 -
   5.237 -
   5.238 -/*Runs in main thread, outside VMS
   5.239 - *Frees any memory allocated by VCilk__init() then calls VMS's cleanup
   5.240 - */
   5.241 -void
   5.242 -VCilk__cleanup_at_end_of_shutdown()
   5.243 - { VCilkSemEnv *semanticEnv;
   5.244 - 
   5.245 -   semanticEnv = _VMSMasterEnv->semanticEnv;
   5.246 -
   5.247 -   /*
   5.248 -   int32 coreIdx;
   5.249 -   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
   5.250 -    {
   5.251 -      VMS__free( semanticEnv->readyVPQs[coreIdx]->startOfData );
   5.252 -      VMS__free( semanticEnv->readyVPQs[coreIdx] );
   5.253 -    }
   5.254 -   VMS__free( semanticEnv->readyVPQs );
   5.255 -   
   5.256 -   VMS__free( _VMSMasterEnv->semanticEnv );
   5.257 -    */
   5.258 -   VMS__cleanup_at_end_of_shutdown();
   5.259 - }
   5.260 -
   5.261 -
   5.262 -//===========================================================================
   5.263 -
   5.264 -
   5.265 -/*Spawn involves allocating mem as well as creating processor which itself
   5.266 - * allocates, so has to be done inside master
   5.267 - */
   5.268 -void inline
   5.269 -VCilk__spawn( int32  coreToSpawnOnto, VirtProcrFnPtr  fnPtr,
   5.270 -              void  *initData,        VirtProcr      *requestingPr )
   5.271 - { VCilkSemReq reqData;
   5.272 -
   5.273 -      //the semantic request data is on the stack and disappears when this
   5.274 -      // call returns -- it's guaranteed to remain in the VP's stack for as
   5.275 -      // long as the VP is suspended.
   5.276 -   reqData.reqType         = 0; //know it's type because in a VMS create req
   5.277 -   reqData.coreToSpawnOnto = coreToSpawnOnto;
   5.278 -   reqData.fnPtr           = fnPtr;
   5.279 -   reqData.initData        = initData;
   5.280 -   reqData.requestingPr    = requestingPr;
   5.281 -
   5.282 -   VMS__send_create_procr_req( &reqData, requestingPr );
   5.283 - } 
   5.284 -
   5.285 -
   5.286 -int32
   5.287 -VCilk__give_number_of_cores_to_spawn_onto()
   5.288 - {
   5.289 -   return NUM_CORES;
   5.290 - }
   5.291 -
   5.292 -
   5.293 -
   5.294 -/*This runs inside slave VP, so can't do any freeing -- have to do in plugin
   5.295 - */
   5.296 -void inline
   5.297 -VCilk__dissipate_procr( VirtProcr *procrToDissipate )
   5.298 - { 
   5.299 -
   5.300 -   VMS__send_dissipate_req( procrToDissipate );
   5.301 - }
   5.302 -
   5.303 -//===========================================================================
   5.304 -
   5.305 -void
   5.306 -VCilk__sync( VirtProcr *animPr )
   5.307 - { VCilkSemReq reqData;
   5.308 - 
   5.309 -   reqData.reqType      = syncReq;
   5.310 -   reqData.requestingPr = animPr;
   5.311 -
   5.312 -   VMS__send_sem_request( &reqData, animPr );
   5.313 - }
   5.314 -
   5.315 -
   5.316 -
   5.317 -void *
   5.318 -VCilk__malloc( int32 sizeToMalloc, VirtProcr *animPr )
   5.319 - { VCilkSemReq reqData;
   5.320 -
   5.321 -   reqData.reqType      = mallocReq;
   5.322 -   reqData.requestingPr = animPr;
   5.323 -   reqData.sizeToMalloc = sizeToMalloc;
   5.324 -
   5.325 -   VMS__send_sem_request( &reqData, animPr );
   5.326 -
   5.327 -   return animPr->dataRetFromReq;
   5.328 - }
   5.329 -
   5.330 -
   5.331 -/*Sends request to Master, which does the work of freeing
   5.332 - */
   5.333 -void
   5.334 -VCilk__free( void *ptrToFree, VirtProcr *animPr )
   5.335 - { VCilkSemReq reqData;
   5.336 -
   5.337 -   reqData.reqType      = freeReq;
   5.338 -   reqData.requestingPr = animPr;
   5.339 -   reqData.ptrToFree    = ptrToFree;
   5.340 -
   5.341 -   VMS__send_sem_request( &reqData, animPr );
   5.342 - }
   5.343 -
   5.344 -//===========================================================================
   5.345 -
   5.346 -/*Uses ID as index into array of flags.  If flag already set, resumes from
   5.347 - * end-label.  Else, sets flag and resumes normally.
   5.348 - */
   5.349 -void
   5.350 -VCilk__start_singleton( int32 singletonID, void *endSingletonLabelAddr,
   5.351 -                      VirtProcr *animPr )
   5.352 - {
   5.353 -   VCilkSemReq  reqData;
   5.354 -
   5.355 -      //
   5.356 -   reqData.reqType     = singleton;
   5.357 -   reqData.singletonID = singletonID;
   5.358 -   reqData.endJumpPt   = endSingletonLabelAddr;
   5.359 -
   5.360 -   VMS__send_sem_request( &reqData, animPr );
   5.361 - }
   5.362 -
   5.363 -/*This executes the function in the masterVP, so it executes in isolation
   5.364 - * from any other copies -- only one copy of the function can ever execute
   5.365 - * at a time.
   5.366 - *
   5.367 - *It suspends to the master, and the request handler takes the function
   5.368 - * pointer out of the request and calls it, then resumes the VP.
   5.369 - *Only very short functions should be called this way -- for longer-running
   5.370 - * isolation, use transaction-start and transaction-end, which run the code
   5.371 - * between as work-code.
   5.372 - */
   5.373 -void
   5.374 -VCilk__animate_short_fn_in_isolation( PtrToAtomicFn ptrToFnToExecInMaster,
   5.375 -                                    void *data, VirtProcr *animPr )
   5.376 - {
   5.377 -   VCilkSemReq  reqData;
   5.378 -
   5.379 -      //
   5.380 -   reqData.reqType          = atomic;
   5.381 -   reqData.fnToExecInMaster = ptrToFnToExecInMaster;
   5.382 -   reqData.dataForFn        = data;
   5.383 -
   5.384 -   VMS__send_sem_request( &reqData, animPr );
   5.385 - }
   5.386 -
   5.387 -
   5.388 -/*This suspends to the master.
   5.389 - *First, it looks at the VP's data, to see the highest transactionID that VP
   5.390 - * already has entered.  If the current ID is not larger, it throws an
   5.391 - * exception stating a bug in the code.  Otherwise it puts the current ID
   5.392 - * there, and adds the ID to a linked list of IDs entered -- the list is
   5.393 - * used to check that exits are properly ordered.
   5.394 - *Next it is uses transactionID as index into an array of transaction
   5.395 - * structures.
   5.396 - *If the "VP_currently_executing" field is non-null, then put requesting VP
   5.397 - * into queue in the struct.  (At some point a holder will request
   5.398 - * end-transaction, which will take this VP from the queue and resume it.)
   5.399 - *If NULL, then write requesting into the field and resume.
   5.400 - */
   5.401 -void
   5.402 -VCilk__start_transaction( int32 transactionID, VirtProcr *animPr )
   5.403 - {
   5.404 -   VCilkSemReq  reqData;
   5.405 -
   5.406 -      //
   5.407 -   reqData.reqType     = trans_start;
   5.408 -   reqData.transID     = transactionID;
   5.409 -
   5.410 -   VMS__send_sem_request( &reqData, animPr );
   5.411 - }
   5.412 -
   5.413 -/*This suspends to the master, then uses transactionID as index into an
   5.414 - * array of transaction structures.
   5.415 - *It looks at VP_currently_executing to be sure it's same as requesting VP.
   5.416 - * If different, throws an exception, stating there's a bug in the code.
   5.417 - *Next it looks at the queue in the structure.
   5.418 - *If it's empty, it sets VP_currently_executing field to NULL and resumes.
   5.419 - *If something in, gets it, sets VP_currently_executing to that VP, then
   5.420 - * resumes both.
   5.421 - */
   5.422 -void
   5.423 -VCilk__end_transaction( int32 transactionID, VirtProcr *animPr )
   5.424 - {
   5.425 -   VCilkSemReq  reqData;
   5.426 -
   5.427 -      //
   5.428 -   reqData.reqType     = trans_end;
   5.429 -   reqData.transID     = transactionID;
   5.430 -
   5.431 -   VMS__send_sem_request( &reqData, animPr );
   5.432 - }
   5.433 +/*
   5.434 + * Copyright 2010  OpenSourceCodeStewardshipFoundation
   5.435 + *
   5.436 + * Licensed under BSD
   5.437 + */
   5.438 +
   5.439 +#include <stdio.h>
   5.440 +#include <stdlib.h>
   5.441 +
   5.442 +#include "VMS/VMS.h"
   5.443 +#include "VCilk.h"
   5.444 +#include "VMS/Queue_impl/PrivateQueue.h"
   5.445 +#include "VMS/Hash_impl/PrivateHash.h"
   5.446 +
   5.447 +
   5.448 +//==========================================================================
   5.449 +
   5.450 +void
   5.451 +VCilk__init();
   5.452 +
   5.453 +void
   5.454 +VCilk__init_Seq();
   5.455 +
   5.456 +void
   5.457 +VCilk__init_Helper();
   5.458 +//==========================================================================
   5.459 +
   5.460 +
   5.461 +/*TODO: Q: dealing with library f()s and DKU vs WT vs FoR
   5.462 + * (still want to do FoR, with time-lines as syntax, could be super cool)
   5.463 + * A: thinking pin the coreLoops for all of BLIS -- let Master arbitrate
   5.464 + * among library, DKU, WT, FoR -- all the patterns in terms of virtual
   5.465 + * processors (or equivalently work-units), so Master picks which virt procr
   5.466 + * from which portions of app (DKU, WT, FoR) onto which sched slots
   5.467 + *Might even do hierarchy of masters -- group of sched slots for each core
   5.468 + * has its own master, that keeps generated work local
   5.469 + * single-reader-single-writer sync everywhere -- no atomic primitives (but
   5.470 + * memory fences on architectures that need them)
   5.471 + * Might have the different schedulers talk to each other, to negotiate
   5.472 + * larger-grain sharing of resources, according to predicted critical
   5.473 + * path, and expansion of work
   5.474 + */
   5.475 +
   5.476 +
   5.477 +
   5.478 +//===========================================================================
   5.479 +
   5.480 +
   5.481 +/*These are the library functions *called in the application*
   5.482 + * 
   5.483 + *There's a pattern for the outside sequential code to interact with the
   5.484 + * VMS_HW code.
   5.485 + *The VMS_HW system is inside a boundary..  every VCilk system is in its
   5.486 + * own directory that contains the functions for each of the processor types.
   5.487 + * One of the processor types is the "seed" processor that starts the
   5.488 + * cascade of creating all the processors that do the work.
   5.489 + *So, in the directory is a file called "EntryPoint.c" that contains the
   5.490 + * function, named appropriately to the work performed, that the outside
   5.491 + * sequential code calls.  This function follows a pattern:
   5.492 + *1) it calls VCilk__init()
   5.493 + *2) it creates the initial data for the seed processor, which is passed
   5.494 + *    in to the function
   5.495 + *3) it creates the seed VCilk processor, with the data to start it with.
   5.496 + *4) it calls startVCilkThenWaitUntilWorkDone
   5.497 + *5) it gets the returnValue from the transfer struc and returns that
   5.498 + *    from the function
   5.499 + *
   5.500 + *For now, a new VCilk system has to be created via VCilk__init every
   5.501 + * time an entry point function is called -- later, might add letting the
   5.502 + * VCilk system be created once, and let all the entry points just reuse
   5.503 + * it -- want to be as simple as possible now, and see by using what makes
   5.504 + * sense for later..
   5.505 + */
   5.506 +
   5.507 +
   5.508 +
   5.509 +//===========================================================================
   5.510 +
   5.511 +/*This is the "border crossing" function -- the thing that crosses from the
   5.512 + * outside world, into the VMS_HW world.  It initializes and starts up the
   5.513 + * VMS system, then creates one processor from the specified function and
   5.514 + * puts it into the readyQ.  From that point, that one function is resp.
   5.515 + * for creating all the other processors, that then create others, and so
   5.516 + * forth.
   5.517 + *When all the processors, including the seed, have dissipated, then this
   5.518 + * function returns.  The results will have been written by side-effect via
   5.519 + * pointers read from, or written into initData.
   5.520 + *
   5.521 + *NOTE: no Threads should exist in the outside program that might touch
   5.522 + * any of the data reachable from initData passed in to here
   5.523 + */
   5.524 +void
   5.525 +VCilk__create_seed_procr_and_do_work( VirtProcrFnPtr fnPtr, void *initData )
   5.526 + { VCilkSemEnv *semEnv;
   5.527 +   VirtProcr *seedPr;
   5.528 +
   5.529 +   #ifdef SEQUENTIAL
   5.530 +   VCilk__init_Seq();  //debug sequential exe
   5.531 +   #else
   5.532 +   VCilk__init();      //normal multi-thd
   5.533 +   #endif
   5.534 +   semEnv = _VMSMasterEnv->semanticEnv;
   5.535 +
   5.536 +      //VCilk starts with one processor, which is put into initial environ,
   5.537 +      // and which then calls create() to create more, thereby expanding work
   5.538 +   seedPr = VCilk__create_procr_helper( fnPtr, initData, NULL, semEnv, -1 );
   5.539 +   resume_procr( seedPr, semEnv );
   5.540 +
   5.541 +   #ifdef SEQUENTIAL
   5.542 +   VMS__start_the_work_then_wait_until_done_Seq();  //debug sequential exe
   5.543 +   #else
   5.544 +   VMS__start_the_work_then_wait_until_done();      //normal multi-thd
   5.545 +   #endif
   5.546 +
   5.547 +   VCilk__cleanup_at_end_of_shutdown();
   5.548 + }
   5.549 +
   5.550 +
   5.551 +int32 inline
   5.552 +VCilk__giveMinWorkUnitCycles( float32 percentOverhead )
   5.553 + {
   5.554 +   return MIN_WORK_UNIT_CYCLES;
   5.555 + }
   5.556 +
   5.557 +int32
   5.558 +VCilk__giveIdealNumWorkUnits()
   5.559 + {
   5.560 +   return NUM_SCHED_SLOTS * NUM_CORES;
   5.561 + }
   5.562 +
   5.563 +/*To measure how long a primitive operation takes, when calculating number of
   5.564 + * sub-tasks to divide into.
   5.565 + * For now, use TSC -- later, make these two macros with assembly that first
   5.566 + * saves jump point, and second jumps back several times to get reliable time
   5.567 + */
   5.568 +void inline
   5.569 +VCilk__start_primitive()
   5.570 + { //int32 *saveAddr;
   5.571 +   //saveAddr = &(((VCilkSemEnv *)(_VMSMasterEnv->semanticEnv))->primitiveStartTime);
   5.572 +   saveLowTimeStampCountInto( (((VCilkSemEnv *)
   5.573 +                        (_VMSMasterEnv->semanticEnv))->primitiveStartTime) );
   5.574 + }
   5.575 +
   5.576 +/*Just quick and dirty for now -- make reliable later
   5.577 + * will want this to jump back several times -- to be sure cache is warm
   5.578 + * because don't want comm time included in calc-time measurement -- and
   5.579 + * also to throw out any "weird" values due to OS interrupt or TSC rollover
   5.580 + */
   5.581 +int32 inline
   5.582 +VCilk__end_primitive_and_give_cycles()
   5.583 + { int32 endTime, startTime;
   5.584 +   //TODO: fix by repeating time-measurement
   5.585 +   saveLowTimeStampCountInto( endTime );
   5.586 +   startTime = ( (VCilkSemEnv *)
   5.587 +                 (_VMSMasterEnv->semanticEnv))->primitiveStartTime;
   5.588 +   return (endTime - startTime);
   5.589 + }
   5.590 +
   5.591 +//===========================================================================
   5.592 +//
   5.593 +/*Initializes all the data-structures for a VCilk system -- but doesn't
   5.594 + * start it running yet!
   5.595 + *
   5.596 + *This and its callees run in main thread outside VMS
   5.597 + * 
   5.598 + *This sets up the semantic layer over the VMS system
   5.599 + *
   5.600 + *First, calls VMS_Setup, then creates own environment, making it ready
   5.601 + * for creating the seed processor and then starting the work.
   5.602 + */
   5.603 +void
   5.604 +VCilk__init()
   5.605 + {
   5.606 +   VMS__init();
   5.607 +      //masterEnv, a global var, now is partially set up by init_VMS
   5.608 +
   5.609 +   VCilk__init_Helper();
   5.610 + }
   5.611 +
   5.612 +void
   5.613 +VCilk__init_Seq()
   5.614 + {
   5.615 +   VMS__init_Seq();
   5.616 +      //masterEnv, a global var, now is partially set up by init_VMS
   5.617 +
   5.618 +   VCilk__init_Helper();
   5.619 + }
   5.620 +
   5.621 +/*Runs in main thread before VMS system starts
   5.622 + */
   5.623 +void
   5.624 +VCilk__init_Helper()
   5.625 + { VCilkSemEnv     *semanticEnv;
   5.626 +   PrivQueueStruc **readyVPQs;
   5.627 +   int              coreIdx;
   5.628 + 
   5.629 +      //Hook up the semantic layer's plug-ins to the Master virt procr
   5.630 +   _VMSMasterEnv->requestHandler = &VCilk__Request_Handler;
   5.631 +   _VMSMasterEnv->slaveScheduler = &VCilk__schedule_virt_procr;
   5.632 +
   5.633 +      //create the semantic layer's environment (all its data) and add to
   5.634 +      // the master environment
   5.635 +   semanticEnv = VMS__malloc( sizeof( VCilkSemEnv ) );
   5.636 +   _VMSMasterEnv->semanticEnv = semanticEnv;
   5.637 +
   5.638 +      //create the ready queue, hash tables used for pairing send to receive
   5.639 +      // and so forth
   5.640 +      //TODO: add hash tables for pairing sends with receives, and
   5.641 +      // initialize the data ownership system
   5.642 +   readyVPQs = VMS__malloc( NUM_CORES * sizeof(PrivQueueStruc *) );
   5.643 +
   5.644 +   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
   5.645 +    {
   5.646 +      readyVPQs[ coreIdx ] = makeVMSPrivQ();
   5.647 +    }
   5.648 +   
   5.649 +   semanticEnv->readyVPQs = readyVPQs;
   5.650 +   
   5.651 +   semanticEnv->nextCoreToGetNewPr = 0;
   5.652 +
   5.653 +   //TODO: bug -- turn these arrays into dyn arrays to eliminate limit
   5.654 +   //semanticEnv->singletonHasBeenExecutedFlags = makeDynArrayInfo( );
   5.655 +   //semanticEnv->transactionStrucs = makeDynArrayInfo( );
   5.656 +   //something like: setHighestIdx( dynArrayInfo, NUM_STRUCS_IN_SEM_ENV )
   5.657 +   int32 i;
   5.658 +   for( i = 0; i < NUM_STRUCS_IN_SEM_ENV; i++ )
   5.659 +    {
   5.660 +      semanticEnv->fnSingletons[i].endInstrAddr      = NULL;
   5.661 +      semanticEnv->fnSingletons[i].hasBeenStarted    = FALSE;
   5.662 +      semanticEnv->fnSingletons[i].hasFinished       = FALSE;
   5.663 +      semanticEnv->fnSingletons[i].waitQ             = makeVMSPrivQ();
   5.664 +      semanticEnv->transactionStrucs[i].waitingVPQ   = makeVMSPrivQ();
   5.665 +    }
   5.666 +
   5.667 + }
   5.668 +
   5.669 +
   5.670 +/*Runs in main thread, outside VMS
   5.671 + *Frees any memory allocated by VCilk__init() then calls VMS's cleanup
   5.672 + */
   5.673 +void
   5.674 +VCilk__cleanup_at_end_of_shutdown()
   5.675 + { VCilkSemEnv *semanticEnv;
   5.676 + 
   5.677 +   semanticEnv = _VMSMasterEnv->semanticEnv;
   5.678 +
   5.679 +   /*
   5.680 +   int32 coreIdx;
   5.681 +   for( coreIdx = 0; coreIdx < NUM_CORES; coreIdx++ )
   5.682 +    {
   5.683 +      VMS__free( semanticEnv->readyVPQs[coreIdx]->startOfData );
   5.684 +      VMS__free( semanticEnv->readyVPQs[coreIdx] );
   5.685 +    }
   5.686 +   VMS__free( semanticEnv->readyVPQs );
   5.687 +   
   5.688 +   VMS__free( _VMSMasterEnv->semanticEnv );
   5.689 +    */
   5.690 +   VMS__cleanup_at_end_of_shutdown();
   5.691 + }
   5.692 +
   5.693 +
   5.694 +//===========================================================================
   5.695 +
   5.696 +
   5.697 +/*Spawn involves allocating mem as well as creating processor which itself
   5.698 + * allocates, so has to be done inside master
   5.699 + */
   5.700 +void inline
   5.701 +VCilk__spawn( int32  coreToSpawnOnto, VirtProcrFnPtr  fnPtr,
   5.702 +              void  *initData,        VirtProcr      *requestingPr )
   5.703 + { VCilkSemReq reqData;
   5.704 +
   5.705 +      //the semantic request data is on the stack and disappears when this
   5.706 +      // call returns -- it's guaranteed to remain in the VP's stack for as
   5.707 +      // long as the VP is suspended.
   5.708 +   reqData.reqType         = 0; //know it's type because in a VMS create req
   5.709 +   reqData.coreToSpawnOnto = coreToSpawnOnto;
   5.710 +   reqData.fnPtr           = fnPtr;
   5.711 +   reqData.initData        = initData;
   5.712 +   reqData.requestingPr    = requestingPr;
   5.713 +
   5.714 +   VMS__send_create_procr_req( &reqData, requestingPr );
   5.715 + } 
   5.716 +
   5.717 +
   5.718 +int32
   5.719 +VCilk__give_number_of_cores_to_spawn_onto()
   5.720 + {
   5.721 +   return NUM_CORES;
   5.722 + }
   5.723 +
   5.724 +
   5.725 +
   5.726 +/*This runs inside slave VP, so can't do any freeing -- have to do in plugin
   5.727 + */
   5.728 +void inline
   5.729 +VCilk__dissipate_procr( VirtProcr *procrToDissipate )
   5.730 + { 
   5.731 +
   5.732 +   VMS__send_dissipate_req( procrToDissipate );
   5.733 + }
   5.734 +
   5.735 +//===========================================================================
   5.736 +
   5.737 +void
   5.738 +VCilk__sync( VirtProcr *animPr )
   5.739 + { VCilkSemReq reqData;
   5.740 + 
   5.741 +   reqData.reqType      = syncReq;
   5.742 +   reqData.requestingPr = animPr;
   5.743 +
   5.744 +   VMS__send_sem_request( &reqData, animPr );
   5.745 + }
   5.746 +
   5.747 +
   5.748 +
   5.749 +void *
   5.750 +VCilk__malloc( int32 sizeToMalloc, VirtProcr *animPr )
   5.751 + { VCilkSemReq reqData;
   5.752 +
   5.753 +   reqData.reqType      = mallocReq;
   5.754 +   reqData.requestingPr = animPr;
   5.755 +   reqData.sizeToMalloc = sizeToMalloc;
   5.756 +
   5.757 +   VMS__send_sem_request( &reqData, animPr );
   5.758 +
   5.759 +   return animPr->dataRetFromReq;
   5.760 + }
   5.761 +
   5.762 +
   5.763 +/*Sends request to Master, which does the work of freeing
   5.764 + */
   5.765 +void
   5.766 +VCilk__free( void *ptrToFree, VirtProcr *animPr )
   5.767 + { VCilkSemReq reqData;
   5.768 +
   5.769 +   reqData.reqType      = freeReq;
   5.770 +   reqData.requestingPr = animPr;
   5.771 +   reqData.ptrToFree    = ptrToFree;
   5.772 +
   5.773 +   VMS__send_sem_request( &reqData, animPr );
   5.774 + }
   5.775 +
   5.776 +//===========================================================================
   5.777 +//
   5.778 +/*A function singleton is a function whose body executes exactly once, on a
   5.779 + * single core, no matter how many times the fuction is called and no
   5.780 + * matter how many cores or the timing of cores calling it.
   5.781 + *
   5.782 + *A data singleton is a ticket attached to data.  That ticket can be used
   5.783 + * to get the data through the function exactly once, no matter how many
   5.784 + * times the data is given to the function, and no matter the timing of
   5.785 + * trying to get the data through from different cores.
   5.786 + */
   5.787 +
   5.788 +/*Fn singleton uses ID as index into array of singleton structs held in the
   5.789 + * semantic environment.
   5.790 + */
   5.791 +void
   5.792 +VCilk__start_fn_singleton( int32 singletonID,   VirtProcr *animPr )
   5.793 + {
   5.794 +   VCilkSemReq  reqData;
   5.795 +
   5.796 +      //
   5.797 +   reqData.reqType     = singleton_fn_start;
   5.798 +   reqData.singletonID = singletonID;
   5.799 +
   5.800 +   VMS__send_sem_request( &reqData, animPr );
   5.801 +   if( animPr->dataRetFromReq ) //will be 0 or addr of label in end singleton
   5.802 +    {
   5.803 +      asm volatile("movl         %0,      %%eax;  \
   5.804 +                    jmp                  *%%eax"  \
   5.805 +      /* outputs */ :                             \
   5.806 +      /* inputs  */ : "g"(animPr->dataRetFromReq) \
   5.807 +      /* clobber */ : "memory", "%eax", "%ebx", "%ecx", "%edx","%edi","%esi"\
   5.808 +                   );
   5.809 +    }
   5.810 + }
   5.811 +
   5.812 +/*Data singleton hands addr of loc holding a pointer to a singleton struct.
   5.813 + * The start_data_singleton makes the structure and puts its addr into the
   5.814 + * location.
   5.815 + */
   5.816 +void
   5.817 +VCilk__start_data_singleton( VCilkSingleton **singletonAddr,  VirtProcr *animPr )
   5.818 + {
   5.819 +   VCilkSemReq  reqData;
   5.820 +
   5.821 +   if( *singletonAddr && (*singletonAddr)->hasFinished )
   5.822 +      goto JmpToEndSingleton;
   5.823 +      //
   5.824 +   reqData.reqType       = singleton_data_start;
   5.825 +   reqData.singletonPtrAddr = singletonAddr;
   5.826 +
   5.827 +   VMS__send_sem_request( &reqData, animPr );
   5.828 +   if( animPr->dataRetFromReq ) //either 0 or end singleton's return addr
   5.829 +    {    //Assembly code changes the return addr on the stack to the one
   5.830 +         // saved into the singleton by the end-singleton-fn
   5.831 +         //The return addr is at 0x4(%%ebp)
   5.832 +      JmpToEndSingleton:
   5.833 +      asm volatile("movl        %0,      %%eax;   \
   5.834 +                    movl    (%%eax),     %%ebx;   \
   5.835 +                    movl    (%%ebx),     %%eax;   \
   5.836 +                    movl     %%eax,  0x4(%%ebp);" \
   5.837 +      /* outputs */ :                             \
   5.838 +      /* inputs  */ : "m"(singletonAddr) \
   5.839 +      /* clobber */ : "memory", "%eax", "%ebx", "%ecx", "%edx","%edi","%esi"\
   5.840 +                   );
   5.841 +    }
   5.842 +   //now, simply return
   5.843 +   //will exit either from the start singleton call or the end-singleton call
   5.844 + }
   5.845 +
   5.846 +/*Uses ID as index into array of flags.  If flag already set, resumes from
   5.847 + * end-label.  Else, sets flag and resumes normally.
   5.848 + *
   5.849 + *Note, this call cannot be inlined because the instr addr at the label
   5.850 + * inside is shared by all invocations of a given singleton ID.
   5.851 + */
   5.852 +void
   5.853 +VCilk__end_fn_singleton( int32 singletonID, VirtProcr *animPr )
   5.854 + {
   5.855 +   VCilkSemReq  reqData;
   5.856 +
   5.857 +      //don't need this addr until after at least one singleton has reached
   5.858 +      // this function
   5.859 +   VCilkSemEnv *semEnv = VMS__give_sem_env_for( animPr );
   5.860 +   semEnv->fnSingletons[ singletonID].endInstrAddr = &&EndSingletonInstrAddr;
   5.861 +
   5.862 +   reqData.reqType     = singleton_fn_end;
   5.863 +   reqData.singletonID = singletonID;
   5.864 +
   5.865 +   VMS__send_sem_request( &reqData, animPr );
   5.866 +
   5.867 +EndSingletonInstrAddr:
   5.868 +   return;
   5.869 + }
   5.870 +
   5.871 +void
   5.872 +VCilk__end_data_singleton(  VCilkSingleton **singletonPtrAddr, VirtProcr *animPr )
   5.873 + {
   5.874 +   VCilkSemReq  reqData;
   5.875 +
   5.876 +      //don't need this addr until after singleton struct has reached
   5.877 +      // this function for first time
   5.878 +      //do assembly that saves the return addr of this fn call into the
   5.879 +      // data singleton -- that data-singleton can only be given to exactly
   5.880 +      // one instance in the code of this function.  However, can use this
   5.881 +      // function in different places for different data-singletons.
   5.882 +//   (*(singletonAddr))->endInstrAddr =  &&EndDataSingletonInstrAddr;
   5.883 +
   5.884 +         //Assembly code takes the return addr off the stack and saves
   5.885 +         // into the singleton.  The first field in the singleton is the
   5.886 +         // "endInstrAddr" field, and the return addr is at 0x4(%%ebp)
   5.887 +      asm volatile("movl 0x4(%%ebp),     %%eax;   \
   5.888 +                    movl        %0,      %%ebx;   \
   5.889 +                    movl    (%%ebx),     %%ecx;   \
   5.890 +                    movl     %%eax,     (%%ecx);" \
   5.891 +      /* outputs */ :                             \
   5.892 +      /* inputs  */ : "m"(singletonPtrAddr) \
   5.893 +      /* clobber */ : "memory", "%eax", "%ebx", "%ecx", "%edx","%edi","%esi"\
   5.894 +                   );
   5.895 +
   5.896 +   reqData.reqType          = singleton_data_end;
   5.897 +   reqData.singletonPtrAddr = singletonPtrAddr;
   5.898 +
   5.899 +   VMS__send_sem_request( &reqData, animPr );
   5.900 + }
   5.901 +
   5.902 +/*This executes the function in the masterVP, so it executes in isolation
   5.903 + * from any other copies -- only one copy of the function can ever execute
   5.904 + * at a time.
   5.905 + *
   5.906 + *It suspends to the master, and the request handler takes the function
   5.907 + * pointer out of the request and calls it, then resumes the VP.
   5.908 + *Only very short functions should be called this way -- for longer-running
   5.909 + * isolation, use transaction-start and transaction-end, which run the code
   5.910 + * between as work-code.
   5.911 + */
   5.912 +void
   5.913 +VCilk__animate_short_fn_in_isolation( PtrToAtomicFn ptrToFnToExecInMaster,
   5.914 +                                    void *data, VirtProcr *animPr )
   5.915 + {
   5.916 +   VCilkSemReq  reqData;
   5.917 +
   5.918 +      //
   5.919 +   reqData.reqType          = atomic;
   5.920 +   reqData.fnToExecInMaster = ptrToFnToExecInMaster;
   5.921 +   reqData.dataForFn        = data;
   5.922 +
   5.923 +   VMS__send_sem_request( &reqData, animPr );
   5.924 + }
   5.925 +
   5.926 +
   5.927 +/*This suspends to the master.
   5.928 + *First, it looks at the VP's data, to see the highest transactionID that VP
   5.929 + * already has entered.  If the current ID is not larger, it throws an
   5.930 + * exception stating a bug in the code.  Otherwise it puts the current ID
   5.931 + * there, and adds the ID to a linked list of IDs entered -- the list is
   5.932 + * used to check that exits are properly ordered.
   5.933 + *Next it is uses transactionID as index into an array of transaction
   5.934 + * structures.
   5.935 + *If the "VP_currently_executing" field is non-null, then put requesting VP
   5.936 + * into queue in the struct.  (At some point a holder will request
   5.937 + * end-transaction, which will take this VP from the queue and resume it.)
   5.938 + *If NULL, then write requesting into the field and resume.
   5.939 + */
   5.940 +void
   5.941 +VCilk__start_transaction( int32 transactionID, VirtProcr *animPr )
   5.942 + {
   5.943 +   VCilkSemReq  reqData;
   5.944 +
   5.945 +      //
   5.946 +   reqData.reqType     = trans_start;
   5.947 +   reqData.transID     = transactionID;
   5.948 +
   5.949 +   VMS__send_sem_request( &reqData, animPr );
   5.950 + }
   5.951 +
   5.952 +/*This suspends to the master, then uses transactionID as index into an
   5.953 + * array of transaction structures.
   5.954 + *It looks at VP_currently_executing to be sure it's same as requesting VP.
   5.955 + * If different, throws an exception, stating there's a bug in the code.
   5.956 + *Next it looks at the queue in the structure.
   5.957 + *If it's empty, it sets VP_currently_executing field to NULL and resumes.
   5.958 + *If something in, gets it, sets VP_currently_executing to that VP, then
   5.959 + * resumes both.
   5.960 + */
   5.961 +void
   5.962 +VCilk__end_transaction( int32 transactionID, VirtProcr *animPr )
   5.963 + {
   5.964 +   VCilkSemReq  reqData;
   5.965 +
   5.966 +      //
   5.967 +   reqData.reqType     = trans_end;
   5.968 +   reqData.transID     = transactionID;
   5.969 +
   5.970 +   VMS__send_sem_request( &reqData, animPr );
   5.971 + }