Mercurial > cgi-bin > hgwebdir.cgi > VMS > VMS_Implementations > VMS_impls > VMS__MC_shared_impl

annotate VMS.h @ 134:a9b72021f053

find changesets by author, revision, files, or words in the commit message
Distributed memory management w/o free requests working
author Merten Sach <msach@mailbox.tu-berlin.de>
date Fri, 16 Sep 2011 16:19:24 +0200
parents dbfc8382d546
children 0b49fd35afc1
rev   line source
Me@42 1 /*
Me@42 2 * Copyright 2009 OpenSourceStewardshipFoundation.org
Me@42 3 * Licensed under GNU General Public License version 2
Me@42 4 *
Me@42 5 * Author: seanhalle@yahoo.com
Me@42 6 *
Me@42 7 */
Me@42 8 #ifndef _VMS_H
Me@42 9 #define _VMS_H
msach@76 10 #define _GNU_SOURCE
Me@42 11
msach@134 12 #include <pthread.h>
msach@134 13 #include <sys/time.h>
msach@134 14
Me@42 15 #include "VMS_primitive_data_types.h"
Me@55 16 #include "Queue_impl/PrivateQueue.h"
Me@42 17 #include "Histogram/Histogram.h"
Me@50 18 #include "DynArray/DynArray.h"
Me@50 19 #include "Hash_impl/PrivateHash.h"
Me@50 20 #include "vmalloc.h"
msach@134 21 #include "requests.h"
Me@50 22
Me@50 23 //=============================== Debug ===================================
Me@55 24 //
msach@76 25 //When SEQUENTIAL is defined, VMS does sequential exe in the main thread
msach@76 26 // It still does co-routines and all the mechanisms are the same, it just
msach@76 27 // has only a single thread and animates VPs one at a time
Me@45 28 //#define SEQUENTIAL
Me@42 29
Me@55 30 //#define USE_WORK_STEALING
Me@55 31
msach@76 32 //turns on the probe-instrumentation in the application -- when not
msach@76 33 // defined, the calls to the probe functions turn into comments
Me@52 34 #define STATS__ENABLE_PROBES
Me@60 35 //#define TURN_ON_DEBUG_PROBES
Me@52 36
msach@76 37 //These defines turn types of bug messages on and off
msach@76 38 // be sure debug messages are un-commented (next block of defines)
Me@68 39 #define dbgAppFlow TRUE /* Top level flow of application code -- general*/
Me@60 40 #define dbgProbes FALSE /* for issues inside probes themselves*/
Me@60 41 #define dbgB2BMaster FALSE /* in coreloop, back to back master VPs*/
Me@60 42 #define dbgRqstHdlr FALSE /* in request handler code*/
Me@52 43
msach@76 44 //Comment or un- the substitute half to turn on/off types of debug message
Me@55 45 #define DEBUG( bool, msg) \
Me@68 46 // if( bool){ printf(msg); fflush(stdin);}
Me@55 47 #define DEBUG1( bool, msg, param) \
Me@60 48 // if(bool){printf(msg, param); fflush(stdin);}
Me@55 49 #define DEBUG2( bool, msg, p1, p2) \
Me@60 50 // if(bool) {printf(msg, p1, p2); fflush(stdin);}
Me@45 51
msach@70 52 #define ERROR(msg) printf(msg);
msach@76 53 #define ERROR1(msg, param) printf(msg, param);
msach@70 54 #define ERROR2(msg, p1, p2) printf(msg, p1, p2);
Me@50 55
Me@50 56 //=========================== STATS =======================
Me@50 57
Me@45 58 //when MEAS__TIME_STAMP_SUSP is defined, causes code to be inserted and
Me@42 59 // compiled-in that saves the low part of the time stamp count just before
msach@76 60 // suspending a processor and just after resuming that processorsrc/VPThread_lib/VMS/VMS.h:322: warning: previous declaration of ‘VMS__create_procr’ was here. It is
Me@42 61 // saved into a field added to VirtProcr. Have to sanity-check for
Me@42 62 // rollover of low portion into high portion.
Me@65 63 //#define MEAS__TIME_STAMP_SUSP
Me@65 64 //#define MEAS__TIME_MASTER
Me@67 65 #define MEAS__TIME_PLUGIN
Me@67 66 #define MEAS__TIME_MALLOC
Me@68 67 //#define MEAS__TIME_MASTER_LOCK
Me@42 68 #define MEAS__NUM_TIMES_TO_RUN 100000
Me@42 69
Me@55 70 //For code that calculates normalization-offset between TSC counts of
Me@55 71 // different cores.
Me@45 72 #define NUM_TSC_ROUND_TRIPS 10
Me@45 73
Me@50 74
Me@50 75 //========================= Hardware related Constants =====================
Me@42 76 //This value is the number of hardware threads in the shared memory
Me@42 77 // machine
msach@69 78 #define NUM_CORES 8
Me@42 79
Me@55 80 // tradeoff amortizing master fixed overhead vs imbalance potential
Me@55 81 // when work-stealing, can make bigger, at risk of losing cache affinity
Me@55 82 #define NUM_SCHED_SLOTS 5
Me@42 83
Me@45 84 #define MIN_WORK_UNIT_CYCLES 20000
Me@45 85
Me@53 86 #define MASTERLOCK_RETRIES 10000
Me@42 87
Me@54 88 // stack size in virtual processors created
msach@78 89 #define VIRT_PROCR_STACK_SIZE 0x8000 /* 32K */
Me@42 90
Me@54 91 // memory for VMS__malloc
msach@134 92 #define MALLOC_ADDITIONAL_MEM_FROM_OS_SIZE 0x4000000 /* 64M */
Me@42 93
msach@78 94 #define CACHE_LINE 64
msach@79 95 #define PAGE_SIZE 4096
msach@78 96
Me@50 97
Me@50 98 //==============================
Me@42 99
Me@42 100 #define SUCCESS 0
Me@42 101
Me@55 102 #define writeVMSQ writePrivQ
Me@55 103 #define readVMSQ readPrivQ
Me@64 104 #define makeVMSQ makeVMSPrivQ
Me@55 105 #define numInVMSQ numInPrivQ
Me@55 106 #define VMSQueueStruc PrivQueueStruc
Me@42 107
Me@42 108
Me@50 109
Me@50 110 //===========================================================================
Me@50 111 typedef unsigned long long TSCount;
Me@50 112
Me@50 113 typedef struct _IntervalProbe IntervalProbe;
Me@55 114 typedef struct _GateStruc GateStruc;
Me@55 115
Me@42 116
Me@42 117 typedef VirtProcr * (*SlaveScheduler) ( void *, int ); //semEnv, coreIdx
Me@42 118 typedef void (*RequestHandler) ( VirtProcr *, void * ); //prWReqst, semEnv
Me@50 119 typedef void (*ResumePrFnPtr) ( VirtProcr *, void * );
Me@50 120
Me@50 121
Me@50 122 //==================== Core data structures ===================
Me@50 123
Me@120 124 /*Master Env is the only global variable -- has entry points for any other
Me@120 125 * data needed.
Me@62 126 */
Me@42 127 typedef struct
Me@42 128 {
Me@42 129 SlaveScheduler slaveScheduler;
Me@42 130 RequestHandler requestHandler;
Me@42 131
Me@42 132 SchedSlot ***allSchedSlots;
msach@127 133 VMSQueueStruc **readyToAnimateQs;
Me@42 134 VirtProcr **masterVPs;
Me@42 135
Me@42 136 void *semanticEnv;
Me@42 137 void *OSEventStruc; //for future, when add I/O to BLIS
Me@42 138
msach@73 139 void *coreLoopReturnPt;//addr to jump to to re-enter coreLoop
Me@42 140
Me@50 141 int32 setupComplete;
msach@71 142 volatile int32 masterLock;
msach@132 143
msach@132 144 MallocProlog *freeListHead[NUM_CORES];
msach@132 145 int32 amtOfOutstandingMem; //total currently allocated
Me@42 146
Me@50 147 int32 numMasterInARow[NUM_CORES];//detect back-to-back masterVP
msach@132 148 GateStruc *workStealingGates[NUM_CORES]; //concurrent work-steal
Me@55 149 int32 workStealingLock;
Me@55 150
msach@127 151 InterMasterReqst* interMasterRequestsFor[NUM_CORES];
msach@127 152 RequestHandler interPluginReqHdlr;
msach@127 153
msach@127 154 int32 numProcrsCreated; //gives ordering to processor creation
msach@132 155
msach@132 156 int32 currentMasterProcrID;
Me@50 157
Me@50 158 //=========== MEASUREMENT STUFF =============
msach@127 159 IntervalProbe **intervalProbes;
msach@127 160 PrivDynArrayInfo *dynIntervalProbesInfo;
msach@127 161 HashTable *probeNameHashTbl;
msach@127 162 int32 masterCreateProbeID;
msach@127 163 float64 createPtInSecs;
msach@127 164 Histogram **measHists;
msach@127 165 PrivDynArrayInfo *measHistsInfo;
Me@65 166 #ifdef MEAS__TIME_PLUGIN
msach@127 167 Histogram *reqHdlrLowTimeHist;
msach@127 168 Histogram *reqHdlrHighTimeHist;
Me@65 169 #endif
Me@65 170 #ifdef MEAS__TIME_MALLOC
msach@127 171 Histogram *mallocTimeHist;
msach@127 172 Histogram *freeTimeHist;
Me@65 173 #endif
Me@65 174 #ifdef MEAS__TIME_MASTER_LOCK
msach@127 175 Histogram *masterLockLowTimeHist;
msach@127 176 Histogram *masterLockHighTimeHist;
Me@65 177 #endif
Me@42 178 }
Me@42 179 MasterEnv;
Me@42 180
Me@55 181 //========================= Extra Stuff Data Strucs =======================
Me@54 182 typedef struct
Me@54 183 {
Me@42 184
Me@54 185 }
Me@54 186 VMSExcp;
Me@50 187
Me@55 188 struct _GateStruc
Me@55 189 {
Me@55 190 int32 gateClosed;
Me@55 191 int32 preGateProgress;
Me@55 192 int32 waitProgress;
Me@55 193 int32 exitProgress;
Me@55 194 };
Me@55 195 //GateStruc
Me@50 196
Me@50 197 //======================= OS Thread related ===============================
Me@42 198
msach@69 199 void * coreLoop( void *paramsIn ); //standard PThreads fn prototype
Me@42 200 void * coreLoop_Seq( void *paramsIn ); //standard PThreads fn prototype
Me@42 201 void masterLoop( void *initData, VirtProcr *masterPr );
Me@42 202
msach@69 203
Me@50 204 typedef struct
Me@50 205 {
Me@50 206 void *endThdPt;
Me@50 207 unsigned int coreNum;
Me@50 208 }
Me@50 209 ThdParams;
Me@42 210
Me@42 211 pthread_t coreLoopThdHandles[ NUM_CORES ]; //pthread's virt-procr state
Me@42 212 ThdParams *coreLoopThdParams [ NUM_CORES ];
Me@42 213 pthread_mutex_t suspendLock;
Me@42 214 pthread_cond_t suspend_cond;
Me@42 215
Me@50 216
msach@69 217
Me@50 218 //===================== Global Vars ===================
Me@50 219
Me@42 220 volatile MasterEnv *_VMSMasterEnv;
Me@42 221
Me@50 222
Me@50 223 //=========================== Function Prototypes =========================
Me@50 224
Me@53 225
Me@53 226 //========== Setup and shutdown ==========
Me@42 227 void
Me@42 228 VMS__init();
Me@42 229
Me@42 230 void
Me@42 231 VMS__init_Seq();
Me@42 232
Me@42 233 void
Me@42 234 VMS__start_the_work_then_wait_until_done();
Me@42 235
Me@42 236 void
Me@42 237 VMS__start_the_work_then_wait_until_done_Seq();
Me@42 238
msach@76 239 inline VirtProcr *
Me@42 240 VMS__create_procr( VirtProcrFnPtr fnPtr, void *initialData );
Me@42 241
Me@53 242 void
Me@53 243 VMS__dissipate_procr( VirtProcr *procrToDissipate );
Me@53 244
Me@50 245 //Use this to create processor inside entry point & other places outside
Me@50 246 // the VMS system boundary (IE, not run in slave nor Master)
Me@50 247 VirtProcr *
Me@50 248 VMS_ext__create_procr( VirtProcrFnPtr fnPtr, void *initialData );
Me@50 249
Me@53 250 void
Me@53 251 VMS_ext__dissipate_procr( VirtProcr *procrToDissipate );
Me@42 252
Me@50 253 void
Me@54 254 VMS__throw_exception( char *msgStr, VirtProcr *reqstPr, VMSExcp *excpData );
Me@54 255
Me@54 256 void
Me@53 257 VMS__shutdown();
Me@53 258
Me@53 259 void
Me@53 260 VMS__cleanup_at_end_of_shutdown();
Me@50 261
Me@64 262 void *
Me@64 263 VMS__give_sem_env_for( VirtProcr *animPr );
Me@64 264
Me@50 265
Me@50 266 //============== Request Related ===============
Me@50 267
Me@50 268 void
Me@50 269 VMS__suspend_procr( VirtProcr *callingPr );
Me@50 270
Me@42 271 inline void
Me@53 272 VMS__add_sem_request_in_mallocd_VMSReqst( void *semReqData, VirtProcr *callingPr );
Me@53 273
Me@53 274 inline void
Me@53 275 VMS__send_sem_request( void *semReqData, VirtProcr *callingPr );
Me@42 276
Me@42 277 void
Me@50 278 VMS__send_create_procr_req( void *semReqData, VirtProcr *reqstingPr );
Me@42 279
Me@53 280 void inline
Me@53 281 VMS__send_dissipate_req( VirtProcr *prToDissipate );
Me@53 282
Me@52 283 inline void
Me@52 284 VMS__send_VMSSem_request( void *semReqData, VirtProcr *callingPr );
Me@52 285
Me@120 286
Me@120 287 void inline
Me@120 288 VMS__send_inter_plugin_req( void *reqData, int32 targetMaster,
Me@120 289 VirtProcr *requestingMaster );
Me@120 290 void inline
Me@120 291 VMS__send_inter_VMSCore_req( InterVMSCoreReqst *reqData, int32 targetMaster,
Me@120 292 VirtProcr *requestingMaster );
Me@120 293
Me@42 294 VMSReqst *
Me@50 295 VMS__take_next_request_out_of( VirtProcr *procrWithReq );
Me@42 296
Me@42 297 inline void *
Me@42 298 VMS__take_sem_reqst_from( VMSReqst *req );
Me@42 299
msach@78 300 void inline
msach@78 301 VMS__handle_VMSSemReq( VMSReqst *req, VirtProcr *requestingPr, void *semEnv,
msach@78 302 ResumePrFnPtr resumePrFnPtr );
msach@78 303
Me@53 304 //======================== STATS ======================
Me@42 305
msach@76 306 //===== RDTSC wrapper ===== //Also runs with x86_64 code
Me@42 307
Me@42 308 #define saveTimeStampCountInto(low, high) \
Me@42 309 asm volatile("RDTSC; \
Me@42 310 movl %%eax, %0; \
Me@42 311 movl %%edx, %1;" \
Me@42 312 /* outputs */ : "=m" (low), "=m" (high)\
Me@42 313 /* inputs */ : \
Me@42 314 /* clobber */ : "%eax", "%edx" \
Me@42 315 );
Me@42 316
Me@42 317 #define saveLowTimeStampCountInto(low) \
Me@42 318 asm volatile("RDTSC; \
Me@42 319 movl %%eax, %0;" \
Me@42 320 /* outputs */ : "=m" (low) \
Me@42 321 /* inputs */ : \
Me@42 322 /* clobber */ : "%eax", "%edx" \
Me@42 323 );
Me@68 324
Me@68 325 //====================
Me@68 326 #define makeAMeasHist( idx, name, numBins, startVal, binWidth ) \
Me@68 327 makeHighestDynArrayIndexBeAtLeast( _VMSMasterEnv->measHistsInfo, idx ); \
Me@68 328 _VMSMasterEnv->measHists[idx] = \
Me@68 329 makeFixedBinHist( numBins, startVal, binWidth, name );
Me@68 330
Me@68 331
Me@68 332 #define MEAS__SUB_CREATE /*turn on/off subtraction of create from plugin*/
msach@69 333
msach@73 334 #ifdef VPTHREAD
msach@73 335
msach@69 336 //VPThread
Me@68 337 #define createHistIdx 1
Me@68 338 #define mutexLockHistIdx 2
Me@68 339 #define mutexUnlockHistIdx 3
Me@68 340 #define condWaitHistIdx 4
Me@68 341 #define condSignalHistIdx 5
Me@68 342
msach@76 343 #define MakeTheMeasHists() \
msach@73 344 _VMSMasterEnv->measHistsInfo = \
msach@73 345 makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
msach@84 346 makeAMeasHist( createHistIdx, "create", 250, 0, 100 ) \
msach@78 347 makeAMeasHist( mutexLockHistIdx, "mutex_lock", 50, 0, 100 ) \
msach@78 348 makeAMeasHist( mutexUnlockHistIdx, "mutex_unlock", 50, 0, 100 ) \
msach@78 349 makeAMeasHist( condWaitHistIdx, "cond_wait", 50, 0, 100 ) \
msach@78 350 makeAMeasHist( condSignalHistIdx, "cond_signal", 50, 0, 100 )
msach@73 351
msach@73 352 #endif
msach@73 353
msach@73 354
msach@73 355 #ifdef VCILK
msach@73 356
msach@69 357 //VCilk
msach@69 358 #define spawnHistIdx 1
msach@69 359 #define syncHistIdx 2
msach@69 360
msach@76 361 #define MakeTheMeasHists() \
msach@73 362 _VMSMasterEnv->measHistsInfo = \
msach@73 363 makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
msach@73 364 makeAMeasHist( spawnHistIdx, "Spawn", 50, 0, 200 ) \
msach@73 365 makeAMeasHist( syncHistIdx, "Sync", 50, 0, 200 )
msach@73 366
msach@73 367
msach@73 368 #endif
msach@73 369
msach@73 370 #ifdef SSR
msach@73 371
msach@69 372 //SSR
msach@69 373 #define SendFromToHistIdx 1
msach@69 374 #define SendOfTypeHistIdx 2
msach@69 375 #define ReceiveFromToHistIdx 3
msach@69 376 #define ReceiveOfTypeHistIdx 4
msach@69 377
msach@76 378 #define MakeTheMeasHists() \
msach@69 379 _VMSMasterEnv->measHistsInfo = \
msach@73 380 makePrivDynArrayOfSize( (void***)&(_VMSMasterEnv->measHists), 200); \
msach@73 381 makeAMeasHist( SendFromToHistIdx, "SendFromTo", 50, 0, 100 ) \
msach@73 382 makeAMeasHist( SendOfTypeHistIdx, "SendOfType", 50, 0, 100 ) \
msach@73 383 makeAMeasHist( ReceiveFromToHistIdx,"ReceiveFromTo", 50, 0, 100 ) \
msach@73 384 makeAMeasHist( ReceiveOfTypeHistIdx,"ReceiveOfType", 50, 0, 100 )
Me@68 385
msach@73 386 #endif
msach@69 387
msach@69 388 //===========================================================================
msach@69 389 //VPThread
msach@69 390
Me@68 391
Me@68 392 #define Meas_startCreate \
Me@68 393 int32 startStamp, endStamp; \
Me@68 394 saveLowTimeStampCountInto( startStamp ); \
Me@68 395
Me@68 396 #define Meas_endCreate \
Me@68 397 saveLowTimeStampCountInto( endStamp ); \
Me@68 398 addIntervalToHist( startStamp, endStamp, \
Me@68 399 _VMSMasterEnv->measHists[ createHistIdx ] );
Me@68 400
Me@68 401 #define Meas_startMutexLock \
Me@68 402 int32 startStamp, endStamp; \
Me@68 403 saveLowTimeStampCountInto( startStamp ); \
Me@68 404
Me@68 405 #define Meas_endMutexLock \
Me@68 406 saveLowTimeStampCountInto( endStamp ); \
Me@68 407 addIntervalToHist( startStamp, endStamp, \
Me@68 408 _VMSMasterEnv->measHists[ mutexLockHistIdx ] );
Me@68 409
Me@68 410 #define Meas_startMutexUnlock \
Me@68 411 int32 startStamp, endStamp; \
Me@68 412 saveLowTimeStampCountInto( startStamp ); \
Me@68 413
Me@68 414 #define Meas_endMutexUnlock \
Me@68 415 saveLowTimeStampCountInto( endStamp ); \
Me@68 416 addIntervalToHist( startStamp, endStamp, \
Me@68 417 _VMSMasterEnv->measHists[ mutexUnlockHistIdx ] );
Me@68 418
Me@68 419 #define Meas_startCondWait \
Me@68 420 int32 startStamp, endStamp; \
Me@68 421 saveLowTimeStampCountInto( startStamp ); \
Me@68 422
Me@68 423 #define Meas_endCondWait \
Me@68 424 saveLowTimeStampCountInto( endStamp ); \
Me@68 425 addIntervalToHist( startStamp, endStamp, \
Me@68 426 _VMSMasterEnv->measHists[ condWaitHistIdx ] );
Me@68 427
Me@68 428 #define Meas_startCondSignal \
Me@68 429 int32 startStamp, endStamp; \
Me@68 430 saveLowTimeStampCountInto( startStamp ); \
Me@68 431
Me@68 432 #define Meas_endCondSignal \
Me@68 433 saveLowTimeStampCountInto( endStamp ); \
Me@68 434 addIntervalToHist( startStamp, endStamp, \
Me@68 435 _VMSMasterEnv->measHists[ condSignalHistIdx ] );
Me@68 436
msach@69 437 //===========================================================================
msach@69 438 // VCilk
msach@69 439 #define Meas_startSpawn \
msach@69 440 int32 startStamp, endStamp; \
msach@69 441 saveLowTimeStampCountInto( startStamp ); \
msach@69 442
msach@69 443 #define Meas_endSpawn \
msach@69 444 saveLowTimeStampCountInto( endStamp ); \
msach@69 445 addIntervalToHist( startStamp, endStamp, \
msach@69 446 _VMSMasterEnv->measHists[ spawnHistIdx ] );
msach@69 447
msach@69 448 #define Meas_startSync \
msach@69 449 int32 startStamp, endStamp; \
msach@69 450 saveLowTimeStampCountInto( startStamp ); \
msach@69 451
msach@69 452 #define Meas_endSync \
msach@69 453 saveLowTimeStampCountInto( endStamp ); \
msach@69 454 addIntervalToHist( startStamp, endStamp, \
msach@69 455 _VMSMasterEnv->measHists[ syncHistIdx ] );
msach@69 456
msach@69 457 //===========================================================================
msach@69 458 // SSR
msach@69 459 #define Meas_startSendFromTo \
msach@69 460 int32 startStamp, endStamp; \
msach@69 461 saveLowTimeStampCountInto( startStamp ); \
msach@69 462
msach@69 463 #define Meas_endSendFromTo \
msach@69 464 saveLowTimeStampCountInto( endStamp ); \
msach@69 465 addIntervalToHist( startStamp, endStamp, \
msach@69 466 _VMSMasterEnv->measHists[ SendFromToHistIdx ] );
msach@69 467
msach@69 468 #define Meas_startSendOfType \
msach@69 469 int32 startStamp, endStamp; \
msach@69 470 saveLowTimeStampCountInto( startStamp ); \
msach@69 471
msach@69 472 #define Meas_endSendOfType \
msach@69 473 saveLowTimeStampCountInto( endStamp ); \
msach@69 474 addIntervalToHist( startStamp, endStamp, \
msach@69 475 _VMSMasterEnv->measHists[ SendOfTypeHistIdx ] );
msach@69 476
msach@69 477 #define Meas_startReceiveFromTo \
msach@69 478 int32 startStamp, endStamp; \
msach@69 479 saveLowTimeStampCountInto( startStamp ); \
msach@69 480
msach@69 481 #define Meas_endReceiveFromTo \
msach@69 482 saveLowTimeStampCountInto( endStamp ); \
msach@69 483 addIntervalToHist( startStamp, endStamp, \
msach@69 484 _VMSMasterEnv->measHists[ ReceiveFromToHistIdx ] );
msach@69 485
msach@69 486 #define Meas_startReceiveOfType \
msach@69 487 int32 startStamp, endStamp; \
msach@69 488 saveLowTimeStampCountInto( startStamp ); \
msach@69 489
msach@69 490 #define Meas_endReceiveOfType \
msach@69 491 saveLowTimeStampCountInto( endStamp ); \
msach@69 492 addIntervalToHist( startStamp, endStamp, \
msach@69 493 _VMSMasterEnv->measHists[ReceiveOfTypeHistIdx ] );
Me@68 494
Me@53 495 //=====
Me@42 496
Me@50 497 #include "probes.h"
Me@65 498 #include "vutilities.h"
Me@42 499
Me@42 500 #endif /* _VMS_H */
Me@42 501