cuda_ld_preload_example.c

Go to the documentation of this file.

/*
  Example of using LD_PRELOAD with the CUDA component.  
  Asim YarKhan

  This is designed to work with the simpleMultiGPU_no_counters binary
  in the PAPI CUDA component tests directory.  First trace the library
  calls in simpleMultiGPU_no_counters binary using ltrace.  Note in
  the ltrace output that the CUDA C APIs are different from the CUDA
  calls visible to nvcc. Then figure out appropriate place to attach
  the PAPI calls.  The initialization is attached to the first entry
  to cudaSetDevice.  Each cudaSetDevice is also used to setup the PAPI
  events for that device.  It was harder to figure out where to attach
  the PAPI_start.  After running some tests, I attached it to the 18th
  invocation of gettimeofday (kind of arbitrary! Sorry!).  The
  PAPI_stop was attached to the first invocation of cudaFreeHost.

*/

#define _GNU_SOURCE

#include <stdio.h>
#include <dlfcn.h>

#include "papi.h"

#define MAXDEVICES 5
int EventSet = PAPI_NULL;
int devseen[MAXDEVICES] = {0};

static void *dl1;
int (*PAPI_library_init_ptr)(int version); 
int (*PAPI_create_eventset_ptr)(int *EventSet); 
int (*PAPI_add_named_event_ptr)(int EventSet, char *EventName); 
int (*PAPI_start_ptr)(int EventSet); 
int (*PAPI_stop_ptr)(int EventSet, long long * values); 
int cudaSetDevice(int devnum, int n1, int n2, int n3, void *ptr1) 
{
    static int onetime = 0;
    int retval, retval_cudaSetDevice;
    //printf("cudaSetDevice wrapper %d\n", devnum);
    if ( onetime==0 ) {
        onetime=1;
        // Load the papi library dynamically and read the relevant functions
        dl1 = dlopen( "libpapi.so", RTLD_NOW | RTLD_GLOBAL );
        if ( dl1==NULL ) printf("Intercept cudaSetDevice: Cannot load libpapi.so\n");
        PAPI_library_init_ptr = dlsym( dl1, "PAPI_library_init" );
        PAPI_create_eventset_ptr = dlsym( dl1, "PAPI_create_eventset" );
        PAPI_add_named_event_ptr = dlsym( dl1, "PAPI_add_named_event" );
        PAPI_start_ptr = dlsym( dl1, "PAPI_start" );
        PAPI_stop_ptr = dlsym( dl1, "PAPI_stop" );
        // Start using PAPI
        printf("Intercept cudaSetDevice: Initializing PAPI on device %d\n", devnum);
        retval = (PAPI_library_init_ptr)( PAPI_VER_CURRENT );
        if( retval != PAPI_VER_CURRENT ) fprintf( stdout, "PAPI_library_init failed\n" );
        printf( "PAPI version: %d.%d.%d\n", PAPI_VERSION_MAJOR( PAPI_VERSION ), PAPI_VERSION_MINOR( PAPI_VERSION ), PAPI_VERSION_REVISION( PAPI_VERSION ) );
        retval = (PAPI_create_eventset_ptr)( &EventSet );
        if( retval != PAPI_OK ) fprintf( stdout, "PAPI_create_eventset failed\n" );
    }
    int (*original_function)(int devnum, int n1, int n2, int n3, void *ptr1);
    original_function = dlsym(RTLD_NEXT, "cudaSetDevice");
    retval_cudaSetDevice = (*original_function)( devnum, n1, n2, n3, ptr1 );
    if ( devseen[devnum]==0 ) {
        devseen[devnum]=1;
        char tmpEventName[120];
        printf("Intercept cudaSetDevice: Attaching events for device on device %d\n", devnum);
        snprintf( tmpEventName, 110, "cuda:::device:%d:%s", devnum, "inst_executed" );
        retval = (PAPI_add_named_event_ptr)( EventSet, tmpEventName );
        if (retval!=PAPI_OK) printf( "Could not add event %s\n", tmpEventName );
    }
    return retval_cudaSetDevice;
}


int gettimeofday(void *ptr1, void *ptr2)
{
    static int onetime = 0;
    onetime++;
    // printf("gettimeofday onetime %d\n", onetime);
    // Use above print statement to determine that the N-th gettime of day works
    if ( onetime==17 ) {
        printf("Intercept gettimeofday: Attaching PAPI_start to the %d th call to gettimeofday (this may need to be adjusted)\n", onetime);
        int retval = (PAPI_start_ptr)( EventSet );
        printf("Starting PAPI\n");
        if( retval!=PAPI_OK ) fprintf( stdout, "PAPI_start failed\n" );
    }
    int (*original_function)(void *ptr1, void *ptr2);
    original_function = dlsym(RTLD_NEXT, "gettimeofday");
    return (*original_function)(ptr1, ptr2);
}

int cudaFreeHost(void *ptr1, void *ptr2, int n1, int n2, void *ptr3) 
{
    static int onetime = 0;
    long long values[10];
    int retval, devnum;
    onetime++;
    if ( onetime==1 ) {
        printf("Intercept cudaFreeHost: Used to get PAPI results\n" );
        retval = (PAPI_stop_ptr)( EventSet, values );
        if( retval != PAPI_OK )  fprintf( stderr, "PAPI_stop failed\n" );
        for( devnum = 0; devnum < MAXDEVICES && devseen[devnum]==1  ; devnum++ )
            printf( "PAPI counterValue: cuda::device:%d:%s: %12lld \n", devnum, "inst_executed", values[devnum] );
    }
    int (*original_function)(void *ptr1, void *ptr2, int n1, int n2, void *ptr3);
    original_function = dlsym(RTLD_NEXT, "cudaFreeHost");
    return (*original_function)(ptr1, ptr2, n1, n2, ptr3);
}