Using TAU with charm++

(tested with 5.4 release 1)

Note that when TAU is built with charm++, only charm++ programs can use the resulting library. Additionally, no TAU calls can be made before the ConverseInit call.

When using TAU_COMPILER with charm++, the following change must be made to charm/src/conv-core/converse.h :

static __inline__ unsigned long long int rdtsc(void)
{
        unsigned long long int x;
#ifdef CMK_IA64
	__asm__ __volatile__("mov %0=ar.itc" : "=r"(x) :: "memory");
#else
        __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x));
#endif
        return x;
}

static __inline__ unsigned long long int rdtsc(void)
{
        unsigned long long int x;
#ifndef TAU_CHARM
#ifdef CMK_IA64
	__asm__ __volatile__("mov %0=ar.itc" : "=r"(x) :: "memory");
#else
        __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x));
#endif
#endif
        return x;
}

Using TAU with NAMD

1. Download and install PDT

     wget http://www.cs.uoregon.edu/research/paracomp/proj/pdtoolkit/Download/pdt_latest.tar.gz
     tar xzf pdt_latest.tar.gz
     cd pdtoolkit-3.3.1
     ./configure ; make ; make install 
   

2. Download and install TAU

     wget http://www.cs.uoregon.edu/research/paracomp/proj/tau/tauprofile/dist/tau_latest.tar.gz
     tar xzf tau_latest.tar.gz
     cd tau-2.14.3
     ./configure -pdt=<path to pdt> -charm=<path to charm>
   

   Note that TAU can be configure to use various options (-TRACE, -PROFILECALLPATH, etc) with -PROFILE as the default.
   
3. Apply the TAU NAMD 2.5 patch

   cd NAMD_2.5_Source
   patch < path/to/tau/examples/charm/namd-2.5.patch
   

   Alternatively see the manual patching instructions below.
4. Make the following changes to arch/Linux-i686-g++.arch (or your .arch file):
   • Add "include /include/Makefile"
   • Change

     "CXX = ..."

     to

     "CXX = $(TAU_COMPILER) -optTauSelectFile=<path to select.tau> ..."

   • Change

     "CC = ..."

     to

     "CC = $(TAU_COMPILER) -optTauSelectFile=<path to select.tau> ..."

   For example :

   include /home/amorris/tau2/include/Makefile
   
   NAMD_ARCH = Linux-i686
   CHARMARCH = net-linux
   
   CXX = $(TAU_COMPILER) -optTauSelectFile=/home/amorris/select.tau g++ -DSOCKLEN_T=socklen_t -DNO_STRSTREAM_H
   CXXOPTS = -O3 -march=pentiumpro -ffast-math -static
   CC = $(TAU_COMPILER) -optTauSelectFile=/home/amorris/select.tau gcc
   COPTS = -O3 -march=pentiumpro -ffast-math -static
   

5. We recommend that you set the environment variables PROFILEDIR and TRACEDIR

   export PROFILEDIR=/tmp
   export TRACEDIR=/tmp
   

6. After execution, you should file profile.X.X.X files in your $PROFILEDIR, view these wwith pprof or paraprof

1. Make the above change to converse.h
2. Create the selective instrumentation file. We have had success using the following select.tau file :

   BEGIN_EXCLUDE_LIST
   void Sequencer::thread#
   void Controller::thread#
   int NAMD_read_int#
   void PDBDATA::scan#
   Real PDBAtom::zcoor#
   Real PDBAtom::ycoor#
   Real PDBAtom::xcoor#
   Bool Molocule::is_hydrogen#
   END_EXCLUDE_LIST
   
   BEGIN_FILE_EXCLUDE_LIST
   *mainfunc.C
   *memusage.C
   *BackEnd.C
   END_FILE_EXCLUDE_LIST
   

3. Make the following changes to Makefile :

   Add $(TAU_COMPILER) to the link phase, as shown

   namd2:	$(INCDIR) $(DSTDIR) $(OBJS) $(LIBS)
   	$(MAKEBUILDINFO)
   	$(TAU_COMPILER) $(CHARMC) -verbose -ld++-option \
   	"$(COPTI)$(CHARMINC) $(COPTI)$(INCDIR) $(COPTI)$(SRCDIR) $(CXXOPTS)" \
   	-module NeighborLB -module commlib -language charm++ \
   	$(BUILDINFO).o \
   	$(OBJS) \
   	$(DPMTALIB) \
   	$(DPMELIB) \
   	$(TCLLIB) \
   	$(FFTLIB) \
   	$(PLUGINLIB) \
   	-lm -o namd2
   

4. Make the following changes to BackEnd.C
   • Add the following to the top of BackEnd.C:

     #include <TAU.h>
     extern "C" void Tau_create_top_level_timer_if_necessary(void);
     

   • Change BackEnd::init as follows:

     void BackEnd::init(int argc, char **argv) {
       ConverseInit(argc, argv, slave_init, 1, 1);  // calls slave_init on others
       TAU_PROFILE_SET_NODE(CmiMyPe());
       Tau_create_top_level_timer_if_necessary();
       TAU_PROFILE("BackEnd::init", "", TAU_DEFAULT);
       cpuTime_start = CmiCpuTimer();
       wallTime_start = CmiWallTimer();
       if ( CmiMyPe() ) {
         slave_init(argc, argv);  // for procs that call main
         TAU_PROFILE_EXIT("Calling ConverseExit()");
         ConverseExit();  // should never return
       }
       all_init(argc, argv);
     

5. Make the following changes to mainfunc.C
   • Add the following to the top of mainfun.C:

     #include <TAU.h>
     

   • Change main as follows:

     int main(int argc, char **argv) {
       BackEnd::init(argc,argv);
       TAU_PROFILE("int main(int, char**) C", " ", TAU_DEFAULT);
       ScriptTcl *script = new ScriptTcl;
       Node::Object()->setScript(script);
     
       ...
     
     
       TAU_PROFILE_EXIT("main exiting");
       BackEnd::exit();
       return 0;
     }
     

6. Change Controller.C as follows:
   • Add the following to the top of Controller.C:

     #include <TAU.h>
     

   • Make the following change to ::threadRun

     void Controller::threadRun(Controller* arg)
     {
       TAU_REGISTER_THREAD();
       arg->algorithm();
     }
     

   • Make the following change to ::terminate

     void Controller::terminate(void) {
       TAU_PROFILE_EXIT("Controller::terminate");
       BackEnd::awaken();
       CthFree(thread);
       CthSuspend();
     }
     

7. Change Sequencer.C as follows:
   • Add the following to the top of Sequencer.C:

     #include <TAU.h>
     

   • Make the following change to ::threadRun

     void Sequencer::threadRun(Sequencer* arg)
     {
       TAU_REGISTER_THREAD();
       arg->algorithm();
     }
     

   • Make the following change to ::terminate

     void Sequencer::terminate(void) {
       TAU_PROFILE_EXIT("Sequencer::terminate");
       CthFree(thread);
       CthSuspend();
     }