# HG changeset patch
# User Merten Sach <msach@mailbox.tu-berlin.de>
# Date 1321899363 -3600
# Node ID c8995a602b4663d7d98ccf6e4be43a550dea003f
# Parent  535c119ba09046657913be8b65ce09785c65cfa6
mallocing hw_event

diff -r 535c119ba090 -r c8995a602b46 .hgignore
--- a/.hgignore	Fri Oct 28 06:56:35 2011 -0700
+++ b/.hgignore	Mon Nov 21 19:16:03 2011 +0100
@@ -7,3 +7,4 @@
 c-ray-mt
 *.ppm
 *.o
+*~
diff -r 535c119ba090 -r c8995a602b46 Makefile
--- a/Makefile	Fri Oct 28 06:56:35 2011 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,51 +0,0 @@
-obj = 	\
-	src/VPThread_lib/VMS/Histogram/Histogram.o \
-	src/VPThread_lib/VMS/Histogram/FloatHist.o \
-	src/VPThread_lib/VMS/CoreLoop.o \
-	src/VPThread_lib/VMS/VMS.o \
-	src/VPThread_lib/VMS/MasterLoop.o \
-	src/VPThread_lib/VMS/Queue_impl/PrivateQueue.o \
-	src/VPThread_lib/VMS/Hash_impl/PrivateHash.o \
-	src/VPThread_lib/VMS/DynArray/DynArray.o \
-	src/VPThread_lib/VPThread_PluginFns.o \
-	src/VPThread_lib/VPThread_lib.o \
-	src/VPThread_lib/VMS/Histogram/DblHist.o \
-	src/VPThread_lib/VPThread.o \
-	src/VPThread_lib/VMS/probes.o \
-	src/VPThread_lib/VMS/ProcrContext.o \
-	src/VPThread_lib/VPThread_Request_Handlers.o \
-	src/VPThread_lib/VPThread_helper.o \
-	src/VPThread_lib/VMS/Hash_impl/MurmurHash2.o \
-	src/VPThread_lib/VMS/vmalloc.o \
-	src/VPThread_lib/VMS/contextSwitch.o \
-	src/VPThread_lib/VMS/Queue_impl/BlockingQueue.o \
-	src/VPThread_lib/VMS/vutilities.o \
-	src/Application/main.o
-
-bin = task_size_vs_exe_time
-
-NUM_CORES=4
-
-CC = gcc
-CFLAGS = -m64 -ffast-math -fwrapv -fno-omit-frame-pointer -O3 -D VPTHREAD -D APPLICATION=C-RAY -D NUM_CORES=$(NUM_CORES) -g -Wall
-
-$(bin): $(obj)
-	$(CC) -o $@ $(obj) -lm -lpthread
-	
-%.o : %.c
-	$(CC) -c $(CFLAGS) -o $@ $<
-
-.PHONY: clean
-clean:
-	rm -f $(obj) $(bin)
-
-.PHONY: install
-install:
-	cp $(bin) /usr/local/bin/$(bin)
-
-.PHONY: uninstall
-uninstall:
-	rm -f /usr/local/bin/$(bin)
-
-
-# $@ Name des Targets
diff -r 535c119ba090 -r c8995a602b46 src/Application/main.c
--- a/src/Application/main.c	Fri Oct 28 06:56:35 2011 -0700
+++ b/src/Application/main.c	Mon Nov 21 19:16:03 2011 +0100
@@ -8,9 +8,14 @@
 #include <ctype.h>
 #include <errno.h>
 #include <pthread.h>
+#include <unistd.h>
 #include "VPThread_lib/VPThread.h"
 #include "VPThread_lib/VMS/Queue_impl/PrivateQueue.h"
 
+#include <linux/perf_event.h>
+#include <linux/prctl.h>
+#include <sys/syscall.h>
+
 #undef DEBUG
 //#define DEBUG
 
@@ -61,9 +66,9 @@
 	"Usage: malloc_test [options]\n"
 	"  Spwans a number of threads and allocates memory.\n\n"
 	"Options:\n"
-	"  -t <num>   how many threads to use (default: 1)\n"
-	"  -m <num>   repeat workload and sync operation <m> times\n"
-        "  -n <num>   size of workload, repeat <n> times\n"     
+	"  -t <num>   how many threads to use (default: 1). This is internaly multiplied by the number of cores.\n"
+	"  -o <num>   repeat workload and sync operation <m> times\n"
+        "  -i <num>   size of workload, repeat <n> times\n"     
 	"  -h         this help screen\n\n"
 };
 
@@ -100,19 +105,16 @@
    VPThread__mutex_unlock(barr->mutex, animatingPr);
  }
 
+struct perfData{
+    uint64 cycles;
+    uint64 instructions;
+};
 
-
-union timeStamp{
-    uint32 highLow[2];
-    uint64 total;
-};
+typedef struct perfData measurement_t;
 
 struct input_t{
     struct barrier_t* barrier;
     uint64  totalWorkCycles;
-    uint64  workPlusMutexCycles;
-    union timeStamp startTime;
-    union timeStamp endTime;
 };
 
 
@@ -123,11 +125,24 @@
 int repetitions, workload_size, num_threads;
 size_t chunk_size = 0;
 
+int cycles_counter_fd[NUM_CORES];
 
 //======================== App Code =========================
 /*
  * Workload
  */
+
+#define saveCyclesAndInstrs(core,cycles) do{     \
+   int cycles_fd = cycles_counter_fd[core];             \
+   int nread;                                           \
+                                                        \
+   nread = read(cycles_fd,&(cycles),sizeof(cycles));    \
+   if(nread<=0){                                         \
+       perror("Error reading cycles counter");          \
+       cycles = 0;                                      \
+   }                                                    \
+} while (0) //macro magic for scoping
+
 void work(void* input, VirtProcr* animatingPr)
  {
    int n,m;
@@ -136,35 +151,48 @@
    unsigned int workspace1;
    double workspace2;
    int32 privateMutex = VPThread__make_mutex(animatingPr);
-
-   saveTimeStampCountInto(in->startTime.highLow[0], in->startTime.highLow[1]);
+   
+   int cpuid = sched_getcpu();
+    
    for(m=0; m<repetitions; m++)
     {
-      int32 stamp_startWorkload, stamp_endWorkload;
-      saveLowTimeStampCountInto( stamp_startWorkload );
+       
+      //measure inner workload to determine synchronisation overhead by subtraction
+      measurement_t startWorkload, endWorkload;
+      saveCyclesAndInstrs(cpuid,startWorkload.cycles);
+
+      //workload
       for(n=0; n<workload_size; n++)
        {
          workspace1 += (workspace1 + 32)/2;
          workspace2 += (workspace2 + 23.2)/1.4;
        }
-      saveLowTimeStampCountInto( stamp_endWorkload );
-      int32 numCycles = stamp_endWorkload-stamp_startWorkload;
-      if( numCycles < 100000000 ) totalWorkCycles += numCycles;//sanity check
+      
+    
+      //measure end of inner workload
+      saveCyclesAndInstrs(cpuid,endWorkload.cycles);
+      uint64 numCycles = endWorkload.cycles - startWorkload.cycles;
+      
+      
+     if( numCycles < 100000000 ) totalWorkCycles += numCycles;//sanity check
 
       VPThread__mutex_lock(privateMutex, animatingPr);
-      //access queue
+      //lock access to switch to different tast
       VPThread__mutex_unlock(privateMutex, animatingPr);
     }
 
-   saveTimeStampCountInto( in->endTime.highLow[0], in->endTime.highLow[1] );
    in->totalWorkCycles = totalWorkCycles;
-   in->workPlusMutexCycles = in->endTime.total - in->startTime.total;
+   printf("Cycles: %lu on CPU %lu\n", totalWorkCycles, cpuid);
+   
+   
+   //Wait for all threads to end
    barrier_wait(in->barrier, animatingPr);
    
    //Shutdown worker
    VPThread__dissipate_thread(animatingPr);
-//   printf("%d", workspace1);  //Should never execute!  VMS bug if does
-//   printf("%f", workspace2);
+   
+   printf("%d", workspace1);  //This is to prevent gcc from optimizing out the
+   printf("%f", workspace2);  //two workspace variables
  }
 
 /* this is run after the VMS is set up*/
@@ -174,29 +202,89 @@
    struct input_t input[num_threads];
    struct barrier_t barr;
    barrier_init(&barr, num_threads+1, animatingPr);
+   
+    //setup performance counters
+    struct perf_event_attr* hw_event;
+    hw_event = VMS__malloc(sizeof(struct perf_event_attr));
+    memset(hw_event,0,sizeof(hw_event));
+        hw_event->type = PERF_TYPE_HARDWARE;
+        hw_event->size = sizeof(hw_event);
+        hw_event->disabled = 0;
+        hw_event->freq = 0;
+        hw_event->inherit = 1; /* children inherit it   */
+        hw_event->pinned = 1; /* must always be on PMU */
+        hw_event->exclusive = 0; /* only group on PMU     */
+        hw_event->exclude_user = 0; /* don't count user      */
+        hw_event->exclude_kernel = 1; /* ditto kernel          */
+        hw_event->exclude_hv = 1; /* ditto hypervisor      */
+        hw_event->exclude_idle = 1; /* don't count when idle */
+        hw_event->mmap = 0; /* include mmap data     */
+        hw_event->comm = 0; /* include comm data     */
 
-   for(i=0; i<num_threads; i++)
-    { input[i].barrier = &barr;
+
+    for( i = 0; i < NUM_CORES; i++ )
+    {
+        hw_event->config = PERF_COUNT_HW_CPU_CYCLES; //cycles
+        cycles_counter_fd[i] = syscall(__NR_perf_event_open, hw_event,
+                0,//pid_t pid, 
+                i,//int cpu, 
+                -1,//int group_fd,
+                0//unsigned long flags
+        );
+        if (cycles_counter_fd[i]<0){
+            fprintf(stderr,"On core %d: ",i);
+            perror("Failed to open cycles counter");
+        }
     }
 
-   union timeStamp startBenchTime, endBenchTime;
-   uint64 lastThreadFinishTime = 0L;
-   saveTimeStampCountInto(startBenchTime.highLow[0], startBenchTime.highLow[1]);
+   //Count on all CPUs
+   int cycles_counter_main_fd = syscall(__NR_perf_event_open, hw_event,
+                0,//pid_t pid, 
+                -1,//int cpu, 
+                -1,//int group_fd,
+                0//unsigned long flags
+        );
+    if (cycles_counter_main_fd<0){
+        fprintf(stderr,"On core %d: ",i);
+        perror("Failed to open cycles counter");
+    } 
+   
+   //prepare input
    for(i=0; i<num_threads; i++)
-    { VPThread__create_thread((VirtProcrFnPtr)work, (void*)&input[i], animatingPr);
+    { 
+       input[i].barrier = &barr;
     }
+    
+   //save cycles before execution of threads to get longest runtime
+   measurement_t startBenchTime, endBenchTime;
+   int nread = read(cycles_counter_main_fd,&(startBenchTime.cycles),
+                sizeof(startBenchTime.cycles));
+   if(nread<0){                                         
+       perror("Error reading cycles counter");
+   }
+   
+   //create all threads
+   for(i=0; i<num_threads; i++)
+    { VPThread__create_thread((VirtProcrFnPtr)work, (void*)&input[i], animatingPr);}
+   //wait for all threads to finish
    barrier_wait(&barr, animatingPr);
-   saveTimeStampCountInto(endBenchTime.highLow[0], endBenchTime.highLow[1]);
+  
+   
+   //longest thread  measurement
+   nread = read(cycles_counter_main_fd,&(endBenchTime.cycles),
+                sizeof(endBenchTime.cycles));
+   if(nread<0){                                         
+       perror("Error reading cycles counter");
+   }
 
-
-   for(i=0; i<num_threads; i++)
-    { printf("WorkCycles: %d\n",input[i].totalWorkCycles);
-      printf("Work + Sync Cycles: %lu\n", input[i].workPlusMutexCycles);
-      if(input[i].endTime.total > lastThreadFinishTime)
-         lastThreadFinishTime = input[i].endTime.total;
+   uint64_t overallWorkCycles = 0;
+   for(i=0; i<num_threads; i++){ 
+       printf("WorkCycles: %d\n",input[i].totalWorkCycles);
+       overallWorkCycles += input[i].totalWorkCycles;
     }
-   printf("Time inside Barrier: %lu\n", endBenchTime.total-startBenchTime.total);
-   printf("Longest Span: %lu\n", lastThreadFinishTime-startBenchTime.total);
+   
+   printf("Sum across threads of work cycles: %lu\n", overallWorkCycles);
+   printf("Total Execution: %lu\n", endBenchTime.cycles-startBenchTime.cycles);
 
    //======================================================
 
@@ -226,19 +314,20 @@
                   fprintf(stderr, "invalid number of threads specified: %d\n", num_threads);
                   return EXIT_FAILURE;
                 }
+               num_threads *= NUM_CORES;
             break;
-            case 'n':
+            case 'i':
                if(!isdigit(argv[++i][0]))
                 {
-                  fputs("-n must be followed by a number\n", stderr);
+                  fputs("-i must be followed by a number\n", stderr);
                   return EXIT_FAILURE;
                 }
                repetitions = atoi(argv[i]);
 				break;
-            case 'm':
+            case 'o':
                if(!isdigit(argv[++i][0]))
                 {
-                  fputs("-m must be followed by a number (workload size)\n", stderr);
+                  fputs("-o must be followed by a number (workload size)\n", stderr);
                   return EXIT_FAILURE;
                 }
                workload_size = atoi(argv[i]);