DataManager.h

Go to the documentation of this file.

#ifndef DATAMANAGER_H
#define DATAMANAGER_H

#include <vector>
#include <map>
#include <string>
#include "GenericTreeNode.h"
#include "ParallelGravity.decl.h"

#if CHARM_VERSION > 60401 && CMK_BALANCED_INJECTION_API
#include "ckBIconfig.h"
#endif


struct TreePieceDescriptor{
    TreePiece *treePiece;
        Tree::GenericTreeNode *root;

    TreePieceDescriptor(){}
    TreePieceDescriptor(TreePiece *tp_, GenericTreeNode *r){
        treePiece = tp_;
                root = r;
    }
};

#ifdef CUDA

struct UpdateParticlesStruct{
  CkCallback *cb;
  DataManager *dm;
  VariablePartData *buf;
  int size;
};


// store pointers to flattened buffers if !gpuFree 
struct PendingBuffers {
  CkVec<CudaMultipoleMoments> *moments;
  CkVec<CompactPartData> *particles;
  int chunk;
    CkCallback *cb;
};

#endif

class DataManager : public CBase_DataManager {
    friend class TreePiece;
        friend class OctTreeBuildPhaseIWorker;

    CProxy_TreePiece treePieces;

protected:

    std::vector<SFC::Key> boundaryKeys;

    std::vector<int> responsibleIndex;

    std::vector<int> particleCounts;

    // holds chare array indices of registered treepieces
    CkVec<TreePieceDescriptor> registeredTreePieces;
#ifdef CUDA
    //CkVec<int> registeredTreePieceIndices;
        int cumNumReplicatedNodes;
        int treePiecesDone;
        int savedChunk;
        int treePiecesDonePrefetch;
        int treePiecesDoneLocalComputation;
        // XXX - assumes that only one chunk can be on the gpu
        // at a given time
        int treePiecesDoneRemoteChunkComputation;
        int treePiecesWantParticlesBack;
        int treePiecesParticlesUpdated;
        int savedNumTotalParticles;
        int savedNumTotalNodes;
        // keeps track of buckets of particles that were
        // received during the prefetch and which were subsequently
        // shipped off to the gpu - XXX
        // not including cached particles in postfrefetch entities shipped to gpu
        // since it is hard to count their number given just the pointer to the cache entry
        // * either do not concern yourself with cached particles
        // * or for each entry, get key, find bucket node in CM, DM or TPs and get number
        // for now, the former

        std::map<NodeKey, int> cachedPartsOnGpu;
        // local particles that have been copied to the gpu
        //std::map<NodeKey, int> localPartsOnGpu;

        // can the gpu accept a chunk of remote particles/nodes?
        bool gpuFree;

        // This var will indicate if particle data has been loaded to the GPU
        bool gputransfer;
        CkCallback *localTransferCallback;

        PendingBuffers *currentChunkBuffers;
        // queue that stores all pending chunk transfers
        CkQ<PendingBuffers *> pendingChunkTransferQ;

        // last remote chunk's size in moments and particles
        int lastChunkMoments;
        int lastChunkParticles;
        CudaMultipoleMoments *bufRemoteMoments;
        CompactPartData *bufRemoteParts;

        CudaMultipoleMoments *bufLocalMoments;
        CompactPartData *bufLocalParts;
        VariablePartData *bufLocalVars;

#ifdef HAPI_INSTRUMENT_WRS
        int activeRung;
        int treePiecesDoneInitInstrumentation;
#endif
#endif
    Tree::GenericTreeNode * root;
    CkVec<GenericTreeNode *> nodeTable;

    int oldNumChunks;
    Tree::NodeKey *chunkRoots;
        Tree::NodeLookupType chunkRootTable;

public:

    /* 
     ** Cooling 
     */
    COOL *Cool;
    StarLog *starLog;
    CmiNodeLock lockStarLog;

    DataManager(const CkArrayID& treePieceID);
    DataManager(CkMigrateMessage *);

        void startLocalWalk();
        void resumeRemoteChunk();
#ifdef CUDA
        void donePrefetch(int chunk); // serialize remote chunk wrapper
        void serializeLocalTree();

#ifdef GPU_LOCAL_TREE_WALK
        void transformLocalTreeRecursive(GenericTreeNode *node, CkVec<CudaMultipoleMoments>& localMoments);
#endif //GPU_LOCAL_TREE_WALK
        
        // actual serialization methods
        PendingBuffers *serializeRemoteChunk(GenericTreeNode *);
    void serializeLocal(GenericTreeNode *);
        void freeLocalTreeMemory();
        void freeRemoteChunkMemory(int chunk);
        void transferParticleVarsBack();
        void updateParticles(UpdateParticlesStruct *data);
        void updateParticlesFreeMemory(UpdateParticlesStruct *data);
        void initiateNextChunkTransfer();
#ifdef HAPI_INSTRUMENT_WRS
        int initInstrumentation();
#endif
        DataManager(){} 
#endif
        void clearInstrument(CkCallback const& cb);

private:
        void init();

public:

    ~DataManager() {
        for (unsigned int i = 0; i < nodeTable.length(); i++) {
            delete nodeTable[i];
            }
            nodeTable.clear();

        CoolFinalize(Cool);
        delete starLog;
        CmiDestroyLock(lockStarLog);
        }

    void acceptResponsibleIndex(const int* responsible, const int n,
                    const CkCallback& cb);
    void acceptFinalKeys(const SFC::Key* keys, const int* responsible, uint64_t* bins, const int n, const CkCallback& cb);
    void pup(PUP::er& p);

#ifdef CUDA
        /*
        std::map<NodeKey, int> &getLocalPartsOnGpuTable(){
          return localPartsOnGpu;
        }
        */
        std::map<NodeKey, int> &getCachedPartsOnGpuTable(){
          return cachedPartsOnGpu;
        }
#endif
    // Functions used to create a tree inside the DataManager comprising
    // all the trees in the TreePieces in the local node
private:
    Tree::GenericTreeNode *buildProcessorTree(int n, Tree::GenericTreeNode **gtn);
    int createLookupRoots(Tree::GenericTreeNode *node, Tree::NodeKey *keys);
public:

    void notifyPresence(Tree::GenericTreeNode *root, TreePiece *treePiece);
    void clearRegisteredPieces(const CkCallback& cb);
    void combineLocalTrees(CkReductionMsg *msg);
    void getChunks(int &num, Tree::NodeKey *&roots);
    inline Tree::GenericTreeNode *chunkRootToNode(const Tree::NodeKey k) {
      NodeLookupType::iterator iter = chunkRootTable.find(k);
      if (iter != chunkRootTable.end()) return iter->second;
      else return NULL;
    }
    inline Tree::GenericTreeNode *getRoot() { return root; }
    void initCooling(double dGmPerCcUnit, double dComovingGmPerCcUnit,
             double dErgPerGmUnit, double dSecUnit, double dKpcUnit,
             COOLPARAM inParam, const CkCallback& cb);
    void initStarLog(std::string _fileName, const CkCallback &cb);
    void dmCoolTableRead(double *dTableData, int nData, const CkCallback& cb);
    void CoolingSetTime(double z, // redshift
            double dTime, // Time
            const CkCallback& cb);
    void SetStarCM(double dCenterOfMass[4], const CkCallback& cb);
    void memoryStats(const CkCallback& cb);
    void resetReadOnly(Parameters param, const CkCallback &cb);

  public:
  static Tree::GenericTreeNode *pickNodeFromMergeList(int n, GenericTreeNode **gtn, int &nUnresolved, int &pickedIndex);
};

inline static void setBIconfig()
{
#if CHARM_VERSION > 60401 && CMK_BALANCED_INJECTION_API
    if (CkMyRank()==0) {
#define GNI_BI_DEFAULT    64
      uint16_t cur_bi = ck_get_GNI_BIConfig();
      if (cur_bi > GNI_BI_DEFAULT) {
        ck_set_GNI_BIConfig(GNI_BI_DEFAULT);
      }
    }
    if (CkMyPe() == 0)
      CkPrintf("Balanced injection is set to %d.\n", ck_get_GNI_BIConfig());
#endif
}

class ProjectionsControl : public CBase_ProjectionsControl { 
  public: 
  ProjectionsControl() {
    setBIconfig();
    LBTurnCommOff();
#ifndef LB_MANAGER_VERSION
    // Older Charm++ requires this to avoid excessive delays between successive LBs even
    // when using AtSync mode
    LBSetPeriod(0.0);
#endif
  } 
  ProjectionsControl(CkMigrateMessage *m) : CBase_ProjectionsControl(m) {
    setBIconfig();
    LBTurnCommOff();
#ifndef LB_MANAGER_VERSION
    // Older Charm++ requires this to avoid excessive delays between successive LBs even
    // when using AtSync mode
    LBSetPeriod(0.0);
#endif
  } 
 
  void on(CkCallback cb) { 
    if(CkMyPe() == 0){ 
      CkPrintf("\n\n**** PROJECTIONS ON *****\n\n"); 
    } 
    traceBegin();  
    contribute(cb); 
  } 
 
  void off(CkCallback cb) { 
    if(CkMyPe() == 0){ 
      CkPrintf("\n\n**** PROJECTIONS OFF *****\n\n"); 
    } 
    traceEnd();  
    contribute(cb); 
  } 

  void pup(PUP::er &p){
    CBase_ProjectionsControl::pup(p);
  }
}; 

#endif //DATAMANAGER_H