1.1 --- a/CoreLoop.c	Wed Jul 07 13:15:54 2010 -0700
     1.2 +++ b/CoreLoop.c	Sat Jul 24 08:58:47 2010 -0700
     1.3 @@ -8,9 +8,10 @@
     1.4  #include "VMS.h"
     1.5  #include "Queue_impl/BlockingQueue.h"
     1.6  
     1.7 -#include <windows.h>
     1.8  #include <stdio.h>
     1.9  #include <time.h>
    1.10 +#include <pthread.h>
    1.11 +#include <sched.h>
    1.12  
    1.13  
    1.14  /*This is the loop that runs in the PThread pinned to each core
    1.15 @@ -20,18 +21,35 @@
    1.16   * write the slave's "Done" flag and repeat.
    1.17   */
    1.18  //pthread_create requires ptr to func that takes void * and returns void *
    1.19 -DWORD WINAPI
    1.20 -coreLoop( LPVOID paramsIn )
    1.21 +void *
    1.22 +coreLoop( void *paramsIn )
    1.23   {   
    1.24     ThdParams      *coreLoopThdParams;
    1.25     VirtProcr      *currPr;
    1.26     CASQueueStruc  *workQ;
    1.27 +   unsigned long   coreMask;  //has 1 in bit positions of allowed cores
    1.28 +   int             errorCode;
    1.29     
    1.30        // Get the communication queues out of the param passed in
    1.31     coreLoopThdParams = (ThdParams *)paramsIn;
    1.32 -      //Winblows requires pinning thd to core inside thread-function
    1.33 +
    1.34 +      //wait until signalled that setup is complete
    1.35 +   pthread_mutex_lock( _VMSMasterEnv->suspend_mutex );
    1.36 +   while( !(_VMSMasterEnv->setupComplete) )
    1.37 +    {
    1.38 +      pthread_cond_wait( _VMSMasterEnv->suspend_cond,
    1.39 +                         _VMSMasterEnv->suspend_mutex );
    1.40 +    }
    1.41 +   pthread_mutex_unlock( _VMSMasterEnv->suspend_mutex );
    1.42 +
    1.43 +      //set thread affinity
    1.44 +      //Linux requires pinning thd to core inside thread-function
    1.45        //Designate a core by a 1 in bit-position corresponding to the core
    1.46 -   SetThreadAffinityMask(GetCurrentThread(),1 << coreLoopThdParams->coreNum);
    1.47 +   coreMask = 1 << coreLoopThdParams->coreNum
    1.48 +   errorCode =
    1.49 +   pthread_setaffinity_np( pthread_self(), sizeof(coreMask), coreMask);
    1.50 +   
    1.51 +   if(errorCode){ printf("\nset affinity failure\n"); exit(); }
    1.52  
    1.53     
    1.54        //Save addr of "end core loop" label - jump to it to shut down coreloop
    1.55 @@ -57,7 +75,7 @@
    1.56     currPr = (VirtProcr *) readCASQ( workQ );
    1.57  
    1.58  //   printf("core %d loop procr addr: %d\n", coreLoopThdParams->coreNum, \
    1.59 -       (int)currPr ); fflush(stdin);
    1.60 +//       (int)currPr ); fflush(stdin);
    1.61     currPr->coreLoopStartPt = &&CoreLoopStartPt;  //to be sure.(GCC specific)
    1.62     
    1.63     currPr->coreAnimatedBy  = coreLoopThdParams->coreNum;
    1.64 @@ -99,5 +117,5 @@
    1.65  
    1.66        //jmp to here when want to shut down the VMS system
    1.67     EndCoreLoop:
    1.68 -   ExitThread( 0 );
    1.69 +   pthread_exit( NULL );
    1.70   }

     2.1 --- a/VMS.c	Wed Jul 07 13:15:54 2010 -0700
     2.2 +++ b/VMS.c	Sat Jul 24 08:58:47 2010 -0700
     2.3 @@ -81,27 +81,33 @@
     2.4  
     2.5     //========================================================================
     2.6     //                      Create the Threads
     2.7 -   int coreIdx;
     2.8 +   int coreIdx, retCode;
     2.9 +   #define thdAttrs NULL
    2.10  
    2.11 +   _VMSMasterEnv->setupComplete = 0;
    2.12 +   _VMSMasterEnv->suspend_mutex = PTHREAD_MUTEX_INITIALIZER;
    2.13 +   _VMSMasterEnv->suspend_cond  = PTHREAD_COND_INITIALIZER;
    2.14 +
    2.15 +      //Need the threads to be created suspended, and wait for a signal
    2.16 +      // before proceeding -- gives time after creating to initialize other
    2.17 +      // stuff before the coreLoops set off.
    2.18 +   
    2.19     //Make params given to the win threads that animate the core loops
    2.20     for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
    2.21      { coreLoopThdParams[coreIdx]          = malloc( sizeof(ThdParams) );
    2.22        coreLoopThdParams[coreIdx]->coreNum = coreIdx;
    2.23  
    2.24 -         //make the core loop threads, born in suspended state
    2.25 -      coreLoopThdHandles[ coreIdx ] =
    2.26 -         CreateThread ( NULL, // Security attributes
    2.27 -                        0, // Stack size
    2.28 -                        coreLoop,
    2.29 -                        coreLoopThdParams[coreIdx],
    2.30 -                        CREATE_SUSPENDED,
    2.31 -                        &(coreLoopThdIds[coreIdx])
    2.32 -                       );
    2.33 +      retCode =
    2.34 +      pthread_create( &(coreLoopThdHandles[coreIdx]), 
    2.35 +                        thdAttrs,
    2.36 +                       &coreLoop,
    2.37 +               (void *)(coreLoopThdParams[coreIdx]) );
    2.38 +      if(!retCode){printf("ERROR creating thread: %d\n", retCode); exit();}
    2.39      }
    2.40  
    2.41 +
    2.42   }
    2.43  
    2.44 -
    2.45  void
    2.46  create_sched_slots( MasterEnv *masterEnv )
    2.47   { SchedSlot  **schedSlots, **filledSlots;
    2.48 @@ -132,37 +138,39 @@
    2.49   { int coreIdx;
    2.50        //Start the core loops running
    2.51  //===========================================================================
    2.52 -   LARGE_INTEGER stPerfCount, endPerfCount, countFreq;
    2.53 +   TSCount  startCount, endCount;
    2.54     unsigned long long count = 0, freq = 0;
    2.55 -   double runTime;
    2.56 +   double   runTime;
    2.57  
    2.58 -   QueryPerformanceCounter( &stPerfCount );
    2.59 -
    2.60 -      //start them running
    2.61 -   for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
    2.62 -    {    //Create the threads
    2.63 -      ResumeThread( coreLoopThdHandles[coreIdx] ); //starts thread
    2.64 -    }
    2.65 -
    2.66 +      startCount = getTSCount();
    2.67 +   
    2.68 +      //tell the core loop threads that setup is complete
    2.69 +      //get lock, to lock out any threads still starting up -- they'll see
    2.70 +      // that setupComplete is true before entering while loop, and so never
    2.71 +      // wait on the condition
    2.72 +   pthread_mutex_lock(     _VMSMasterEnv->suspend_mutex );
    2.73 +   _VMSMasterEnv->setupComplete = 1;
    2.74 +   pthread_mutex_unlock(   _VMSMasterEnv->suspend_mutex );
    2.75 +   pthread_cond_broadcast( _VMSMasterEnv->suspend_cond );
    2.76 +   
    2.77 +   
    2.78        //wait for all to complete
    2.79     for( coreIdx=0; coreIdx < NUM_CORES; coreIdx++ )
    2.80      {
    2.81 -      WaitForSingleObject(coreLoopThdHandles[coreIdx], INFINITE);
    2.82 +      pthread_join( coreLoopThdHandles[coreIdx], NULL );
    2.83      }
    2.84 -
    2.85 +   
    2.86        //NOTE: do not clean up VMS env here -- semantic layer has to have
    2.87        // a chance to clean up its environment first, then do a call to free
    2.88        // the Master env and rest of VMS locations
    2.89  
    2.90 -   QueryPerformanceCounter( &endPerfCount );
    2.91 -   count = endPerfCount.QuadPart - stPerfCount.QuadPart;
    2.92  
    2.93 -   QueryPerformanceFrequency( &countFreq );
    2.94 -   freq = countFreq.QuadPart;
    2.95 -   runTime = (double)count / (double)freq;
    2.96 +      endCount = getTSCount();
    2.97 +      count = endCount - startCount;
    2.98  
    2.99 -   printf("\n Time startup to shutdown: %f\n", runTime);
   2.100 -   fflush( stdin );
   2.101 +      runTime = (double)count / (double)TSCOUNT_FREQ;
   2.102 +
   2.103 +      printf("\n Time startup to shutdown: %f\n", runTime); fflush( stdin );
   2.104   }
   2.105  
   2.106  

     3.1 --- a/VMS.h	Wed Jul 07 13:15:54 2010 -0700
     3.2 +++ b/VMS.h	Sat Jul 24 08:58:47 2010 -0700
     3.3 @@ -8,11 +8,11 @@
     3.4  
     3.5  #ifndef _VMS_H
     3.6  #define	_VMS_H
     3.7 +#define __USE_GNU
     3.8  
     3.9  #include "VMS_primitive_data_types.h"
    3.10  #include "Queue_impl/BlockingQueue.h"
    3.11 -#include <windows.h>
    3.12 -#include <winbase.h>
    3.13 +#include "pthread.h"
    3.14  
    3.15     //This value is the number of hardware threads in the shared memory
    3.16     // machine
    3.17 @@ -108,34 +108,36 @@
    3.18     int         stillRunning;
    3.19     
    3.20     VirtProcr  *masterVirtPr;
    3.21 +
    3.22     void       *semanticEnv;
    3.23     void       *OSEventStruc;    //for future, when add I/O to BLIS
    3.24  
    3.25     void       *coreLoopShutDownPt; //addr to jump to to shut down a coreLoop
    3.26 +
    3.27 +   int             setupComplete;
    3.28 +   pthread_mutex_t suspend_mutex;
    3.29 +   pthread_cond_t  suspend_cond;
    3.30   }
    3.31  MasterEnv;
    3.32  
    3.33  
    3.34  //==========================================================
    3.35  
    3.36 -DWORD WINAPI coreLoop( LPVOID paramsIn );
    3.37 -//void * coreLoop( void *paramsIn );  //standard PThreads fn prototype
    3.38 +void * coreLoop( void *paramsIn );  //standard PThreads fn prototype
    3.39  void masterLoop( void *initData, VirtProcr *masterPr );
    3.40  
    3.41  
    3.42  //=====================  Global Vars ===================
    3.43  
    3.44  
    3.45 -HANDLE      coreLoopThdHandles[ NUM_CORES ];  //windows handle to thread
    3.46 +pthread_t   coreLoopThdHandles[ NUM_CORES ];  //pthread's virt-procr state
    3.47  ThdParams   *coreLoopThdParams[ NUM_CORES ];
    3.48 -DWORD           coreLoopThdIds[ NUM_CORES ];
    3.49  
    3.50 -   //TODO: Debug: figure out if need to be volatile or not
    3.51  volatile MasterEnv      *_VMSMasterEnv;
    3.52  
    3.53     //workQ is global, static, and volatile so that core loop has its location
    3.54     // hard coded, and reloads every time through the loop -- that way don't
    3.55 -   // need to save any regs used by core loop (will see if this really works)
    3.56 +   // need to save any regs used by core loop
    3.57  volatile CASQueueStruc  *_VMSWorkQ;
    3.58  
    3.59  //==========================
    3.60 @@ -195,6 +197,10 @@
    3.61  
    3.62  typedef unsigned long long TSCount;
    3.63  
    3.64 +   //Frequency of TS counts
    3.65 +   //TODO: change freq for each machine
    3.66 +#define TSCOUNT_FREQ 3180000000
    3.67 +
    3.68  #define saveTimeStampCountInto(low, high) \
    3.69     asm volatile("RDTSC;                   \
    3.70                   movl %%eax, %0;          \