4 #ifndef PARALLELGRAVITY_H
5 #define PARALLELGRAVITY_H
18 #include "TipsyFile.h"
23 #include "parameters.h"
25 #include "dumpframe.h"
28 #include "TaggedVector3D.h"
34 #include "EwaldCUDA.h"
53 #define MERGE_REMOTE_REQUESTS_VERBOSE(X)
58 if (!cond) CkAbort(
"%s", err);
78 #ifdef SELECTIVE_TRACING
106 inline void operator|(PUP::er &p,
DomainsDec &d) {
108 if (p.isUnpacking()) {
125 int64_t nMaxOrderGas;
126 int64_t nMaxOrderDark;
128 void pup(PUP::er& p) {
137 #include "ParallelGravity.decl.h"
153 #ifdef REDUCTION_HELPER
154 extern CProxy_ReductionHelper reductionHelperProxy;
160 extern CProxy_DataManager
dMProxy;
170 extern CProxy_CkCacheManager<KeyType>
cacheNode;
177 extern int networkProgressUE;
178 extern int nodeForceUE;
179 extern int partForceUE;
181 extern int tbFlushRequestsUE;
182 extern int prefetchDoneUE;
191 extern int remoteNodesPerReq;
192 extern int remoteResumeNodesPerReq;
193 extern int localPartsPerReq;
194 extern int remotePartsPerReq;
195 extern int remoteResumePartsPerReq;
197 extern double largePhaseThreshold;
199 extern cosmoType
theta;
211 class TreePieceStatistics {
212 u_int64_t nodesOpenedLocal;
213 u_int64_t nodesOpenedRemote;
214 u_int64_t nodeInterLocal;
215 u_int64_t nodeInterRemote;
216 u_int64_t particleInterLocal;
217 u_int64_t particleInterRemote;
218 u_int64_t openCriterionCalls;
221 TreePieceStatistics() : nodesOpenedLocal(0), nodesOpenedRemote(0),
222 nodeInterLocal(0), nodeInterRemote(0), particleInterLocal(0),
223 particleInterRemote(0), openCriterionCalls(0), nActive(0) { }
226 TreePieceStatistics(u_int64_t nol, u_int64_t nor, u_int64_t occ, u_int64_t nil, u_int64_t nir,
227 u_int64_t pil, u_int64_t pir,
int na) :
228 nodesOpenedLocal(nol), nodesOpenedRemote(nor), nodeInterLocal(nil),
229 nodeInterRemote(nir), particleInterLocal(pil), particleInterRemote(pir),
230 openCriterionCalls(occ), nActive(na) { }
232 void printTo(CkOStream &os) {
233 os <<
" TreePiece: " << nActive <<
" particles active." << endl;
234 os <<
" TreePiece: " << nodesOpenedLocal <<
" local nodes opened, ";
235 os << nodesOpenedRemote <<
" remote" << endl;
236 os <<
" TreePiece: " << openCriterionCalls <<
" num of open criterion calls" << endl;
237 os <<
" TreePiece: " << nodeInterLocal <<
" local particle-node interactions, ";
238 os << nodeInterRemote <<
" remote" << endl;
239 os <<
" TreePiece: " << particleInterLocal <<
" local particle-particle interactions, ";
240 os << particleInterRemote <<
" remote" << endl;
242 << (particleInterLocal + particleInterRemote)/(
double) nActive
244 << (nodeInterLocal + nodeInterRemote)/(
double) nActive
245 << " nodes per particle" << endl;
248 static CkReduction::reducerType sum;
250 static CkReductionMsg *sumFn(
int nMsg, CkReductionMsg **msgs) {
251 TreePieceStatistics ret;
252 for (
int i=0; i<nMsg; ++i) {
253 CkAssert(msgs[i]->getSize() ==
sizeof(TreePieceStatistics));
254 TreePieceStatistics *data = (TreePieceStatistics *)msgs[i]->getData();
255 ret.nodesOpenedLocal += data->nodesOpenedLocal;
256 ret.nodesOpenedRemote += data->nodesOpenedRemote;
257 ret.openCriterionCalls += data->openCriterionCalls;
258 ret.nodeInterLocal += data->nodeInterLocal;
259 ret.nodeInterRemote += data->nodeInterRemote;
260 ret.particleInterLocal += data->particleInterLocal;
261 ret.particleInterRemote += data->particleInterRemote;
262 ret.nActive += data->nActive;
264 return CkReductionMsg::buildNew(
sizeof(TreePieceStatistics), &ret);
295 ORBSplittersMsg(
int len, CkCallback callback): length (len), cb(callback) {}
307 unsigned int *parts_per_phase;
312 nloads(nload), n(npart), nSPH(nsph), nStar(nstar) {}
316 #include "ckmulticast.h"
318 struct BucketMsg :
public CkMcastBaseMsg,
public CMessage_BucketMsg {
339 void pup(PUP::er& p) {
365 const int MAXRUNG = 30;
366 const int MAXSUBSTEPS = 1 << MAXRUNG;
367 const double MAXSUBSTEPS_INV = 1 / (double)MAXSUBSTEPS;
371 CkAssert(iRung <= MAXRUNG);
372 return 1 << (MAXRUNG - iRung);
377 inline int DtToRung(
double dDelta,
double dTideal) {
378 int iSteps = (int) ceil(dDelta/dTideal);
411 double l2 = p->soft*p->soft;
415 double e2 = p->soft*p->soft;
416 if (l2 < e2) l2 = e2;
419 return l2*dResolveJeans*p->fDensity;
424 const double GAMMA_NONCOOL = 5.0/3.0;
436 class Main :
public CBase_Main {
438 std::string basefilename;
442 CkVec<std::string> *NCgasNames;
443 CkVec<std::string> *NCdarkNames;
444 CkVec<std::string> *NCstarNames;
448 Ck::IO::File fIOFile;
449 CProxy_Sorter sorter;
450 int64_t nTotalParticles;
456 int64_t nMaxOrderGas;
457 int64_t nMaxOrderDark;
468 CkVec<double> vdOutTime;
479 double dSimStartTime;
486 double localNodesPerReqDouble;
487 double remoteNodesPerReqDouble;
488 double remoteResumeNodesPerReqDouble;
489 double localPartsPerReqDouble;
490 double remotePartsPerReqDouble;
491 double remoteResumePartsPerReqDouble;
494 #ifdef CHECK_TIME_WITHIN_BIGSTEP
495 double wallTimeStart;
500 class timing_fields {
517 tGrav = tuDot = tDD = tLoadB = tTBuild = tAdjust
518 = tEmergAdjust = tKick = tDrift = tCache = 0.0;
522 CkVec<timing_fields> timings;
523 void writeTimings(
int iStep);
525 #ifdef SELECTIVE_TRACING
528 int numTraceIterations;
529 int numSkipIterations;
535 void turnProjectionsOn(
int activeRung);
536 void turnProjectionsOff();
543 Main(CkMigrateMessage *m);
547 void initialForces();
549 void restart(CkCheckpointStatusMsg *msg);
550 void waitForGravity(
const CkCallback &cb,
double startTime,
552 void advanceBigStep(
int);
553 void domainDecomp(
int iPhase);
554 void loadBalance(
int iPhase);
555 void buildTree(
int iPhase);
556 void startGravity(
const CkCallback& cbGravity,
int iActiveRung,
558 void externalGravity(
int iActiveRung);
559 void updateuDot(
int iActiveRung,
const double duKick[],
560 const double dStartTime[],
int bUpdateState,
int bAll);
561 void kick(
bool bClosing,
int iActiveRung,
int nextMaxRung,
562 const CkCallback &cbGravity,
double gravStartTime);
563 int adjust(
int iKickRung);
565 void countActive(
int activeRung);
566 void emergencyAdjust(
int iRung);
567 void starCenterOfMass();
568 void calcEnergy(
double,
double,
const char *);
572 void writeOutput(
int iStep) ;
574 const CkCallback& cb);
575 void cbOpen(Ck::IO::FileReadyMsg *msg);
576 void cbIOReady(Ck::IO::SessionReadyMsg *msg);
577 void cbIOComplete(CkMessage *msg);
578 void cbIOClosed(CkMessage *msg);
580 void writeNCXML(std::string filename);
581 void NCXMLattrib(ofstream *desc, CkVec<std::string> *names, std::string family);
583 void growMass(
double dTime,
double dDelta);
587 int ReadASCII(
char *extension,
int nDataPerLine,
double *dDataOut);
589 void doSph(
int activeRung,
int bNeedDensity = 1);
590 void AGORAfeedbackPreCheck(
double dTime,
double dDelta,
double dTimeToSF);
591 void FormStars(
double dTime,
double dDelta);
592 void StellarFeedback(
double dTime,
double dDelta);
593 void outputBlackHoles(
double dTime);
595 void FormSinks(
double dTime,
double dDelta,
int iKickRung);
596 void doSinks(
double dTime,
double dDelta,
int iKickRung);
597 int DumpFrameInit(
double dTime,
double dStep,
int bRestart);
598 void DumpFrame(
double dTime,
double dStep);
599 int nextMaxRungIncDF(
int nextMaxRung);
600 void addDelParticles();
602 void memoryStatsCache();
603 void pup(PUP::er& p);
604 void liveVizImagePrep(liveVizRequestMsg *msg);
605 void doSIDM(
double dTime,
double dDelta,
int activeRung);
627 #if INTERLIST_VER > 0
640 Vector3D<cosmoType> offset;
641 #if defined CHANGA_REFACTOR_PRINT_INTERACTIONS || defined CHANGA_REFACTOR_WALKCHECK_INTERLIST || defined CUDA
653 Vector3D<cosmoType> offset;
654 #if defined CHANGA_REFACTOR_PRINT_INTERACTIONS || defined CHANGA_REFACTOR_WALKCHECK_INTERLIST || defined CUDA
657 #if COSMO_DEBUG > 1 || defined CUDA
670 CkVec<int> bucketids;
682 struct BucketActiveInfo{
691 #if INTERLIST_VER > 0 && defined CUDA
692 template<
typename T>
class GenericList;
701 clientTreePiece(NULL),
714 CkVec<NonLocalMomentsClient> clients;
725 clients.push_back(cli);
743 #if INTERLIST_VER > 0 && defined CUDA
745 template<
typename T>
friend class GenericList;
754 #if INTERLIST_VER > 0
757 State *sInterListStateRemoteResume;
762 Opt *sLocal, *sRemote, *sPref;
765 State *sPrefetchState;
767 State *sLocalGravityState, *sRemoteGravityState, *sSmoothState;
768 typedef std::map<KeyType, CkVec<int>* > SmPartRequestType;
770 SmPartRequestType smPartRequests;
772 CkVec<ActiveWalk> activeWalks;
773 int completedActiveWalks;
780 unsigned int nPrevActiveParts;
781 std::vector<double> savedPhaseLoad;
782 std::vector<unsigned int> savedPhaseParticle;
784 std::vector<double> savedPhaseLoadTmp;
786 std::vector<unsigned int> savedPhaseParticleTmp;
788 int memWithCache, memPostCache;
789 int nNodeCacheEntries, nPartCacheEntries;
793 bool createdSpanningTree;
794 CProxySection_TreePiece allTreePieceSection;
795 CkVec<GravityParticle> foreignParticles;
796 CkVec<double> foreignParticleAccelerations;
798 map<int,CkSectionInfo> cookieJar;
800 BucketMsg *createBucketMsg();
801 void unpackBuckets(BucketMsg *,
GenericTreeNode *&foreignBuckets,
int &numForeignBuckets);
802 void calculateForces(
GenericTreeNode *foreignBuckets,
int numForeignBuckets);
809 void startPushGravity(
int am,
double myTheta);
810 void recvPushBuckets(BucketMsg *);
811 void recvPushAccelerations(CkReductionMsg *);
814 #if COSMO_PRINT_BK > 1
815 State *getSRemoteGravityState(){
return sRemoteGravityState; }
816 State *getSLocalGravityState(){
return sLocalGravityState; }
818 void memCacheStats(
const CkCallback &cb);
826 void markSmoothWalkDone();
828 void finishSmoothWalk();
836 nodeInterRemote[chunk] += howmany;
841 particleInterRemote[chunk] += howmany;
846 nodeInterLocal += howmany;
851 particleInterLocal += howmany;
856 void initiatePrefetch(
int chunk);
858 void startRemoteChunk();
862 return myNumParticles;
879 int numActiveBuckets;
880 int myNumActiveParticles;
882 int FirstGPUParticleIndex;
883 int LastGPUParticleIndex;
884 int NumberOfGPUParticles;
885 BucketActiveInfo *bucketActiveInfo;
891 void callFreeRemoteChunkMemory(
int chunk);
893 int getActiveRung(){
return activeRung; }
894 #ifdef HAPI_INSTRUMENT_WRS
895 int getInstrumentId(){
return instrumentId; }
900 int getDMNumParticles(){
902 return myNumParticles;
905 return myNumActiveParticles;
909 int getNumActiveParticles(){
910 return myNumActiveParticles;
913 void calculateNumActiveParticles(){
914 myNumActiveParticles = 0;
915 for(
int i = 1; i <= myNumParticles; i++){
916 if(myParticles[i].rung >= activeRung){
917 myNumActiveParticles++;
923 return (1.0*myNumActiveParticles/myNumParticles) >= largePhaseThreshold;
927 NumberOfGPUParticles = 0;
928 FirstGPUParticleIndex = fillIndex;
930 for(
int b = 0; b < numBuckets; b++){
935 bucket->bucketArrayIndex = fillIndex;
936 for(
int i = buckstart; i <= buckend; i++){
937 fillArray[fillIndex] = buckparts[i-buckstart];
943 for(
int b = 0; b < numBuckets; b++){
945 if(bucket->
rungs < activeRung){
948 BucketActiveInfo *binfo = &(bucketActiveInfo[b]);
954 binfo->start = fillIndex;
955 for(
int i = buckstart; i <= buckend; i++){
956 if(buckparts[i-buckstart].rung >= activeRung){
957 fillArray[fillIndex] = buckparts[i-buckstart];
961 binfo->size = fillIndex-binfo->start;
965 if(FirstGPUParticleIndex == fillIndex){
967 FirstGPUParticleIndex = -1;
968 LastGPUParticleIndex = -1;
969 NumberOfGPUParticles = 0;
972 LastGPUParticleIndex = fillIndex - 1;
973 NumberOfGPUParticles = LastGPUParticleIndex - FirstGPUParticleIndex + 1;
977 bool isActive(
int partNum){
978 return myParticles[partNum].rung >= activeRung;
981 void clearMarkedBuckets(CkVec<GenericTreeNode *> &markedBuckets);
982 void clearMarkedBucketsAll();
985 long long localNodeInteractions;
986 long long localPartInteractions;
987 long long remoteNodeInteractions;
988 long long remotePartInteractions;
989 long long remoteResumeNodeInteractions;
990 long long remoteResumePartInteractions;
993 #ifdef HAPI_INSTRUMENT_WRS
996 double localNodeListConstructionTime;
997 double remoteNodeListConstructionTime;
998 double remoteResumeNodeListConstructionTime;
999 double localPartListConstructionTime;
1000 double remotePartListConstructionTime;
1001 double remoteResumePartListConstructionTime;
1004 int nRemoteNodeReqs;
1005 int nRemoteResumeNodeReqs;
1007 int nRemotePartReqs;
1008 int nRemoteResumePartReqs;
1014 void continueStartRemoteChunk(
int chunk);
1016 void updateParticles(intptr_t data,
int partIndex);
1018 void continueWrapUp();
1020 #if INTERLIST_VER > 0
1021 void getBucketsBeneathBounds(
GenericTreeNode *&node,
int &start,
int &end);
1022 void updateBucketState(
int start,
int end,
int n,
int chunk,
State *state);
1023 void updateUnfinishedBucketState(
int start,
int end,
int n,
int chunk,
State *state);
1026 #if defined CHANGA_REFACTOR_WALKCHECK || defined CHANGA_REFACTOR_WALKCHECK_INTERLIST
1027 void addToBucketChecklist(
int bucketIndex, NodeKey k){
1028 bucketcheckList[bucketIndex].insert(k);
1029 if(bucketIndex == TEST_BUCKET && thisIndex == TEST_TP)
1030 CkPrintf(
"[%d] add %ld\n", thisIndex, k);
1034 #if INTERLIST_VER > 0
1039 NodeLookupType::iterator iter = nodeLookupTable.find(k);
1040 if (iter != nodeLookupTable.end())
return iter->second;
1045 return bucketList[i];
1053 liveVizRequestMsg * savedLiveVizMsg;
1057 Vector3D<float> savedCentroid;
1065 CkCallback callback;
1067 CkCallback cbGravity;
1069 CkCallback cbSmooth;
1070 CkCallback after_dd_callback;
1072 unsigned int myNumParticles;
1073 unsigned int numActiveParticles;
1076 int nbor_msgs_count_;
1080 bool bBucketsInited;
1085 std::vector<GravityParticle> myTmpShuffleParticle;
1086 std::vector<extraSPHData> myTmpShuffleSphParticle;
1087 std::vector<extraStarData> myTmpShuffleStarParticle;
1091 int myTreeParticles;
1103 unsigned int myNumSPH;
1109 unsigned int myNumStar;
1122 std::vector<GenericTreeNode *> bucketList;
1125 std::vector<GravityParticle> mySortedParticles;
1127 std::vector<extraSPHData> mySortedParticlesSPH;
1129 std::vector<extraStarData> mySortedParticlesStar;
1131 std::vector<ParticleShuffleMsg*> incomingParticlesMsg;
1133 int incomingParticlesArrived;
1138 bool incomingParticlesSelf;
1155 CProxy_TreePiece pieces;
1163 #ifndef REDUCTION_HELPER
1164 CkVec<int64_t> myBinCounts;
1166 std::vector<int> myBinCountsORB;
1171 SFC::Key leftSplitter, rightSplitter;
1173 std::string basefilename;
1175 OrientedBox<float> boundingBox;
1176 unsigned iterationNo;
1182 typedef std::map<NodeKey, CkVec<int>* > MomentRequestType;
1186 MomentRequestType momentRequests;
1200 Vector3D<cosmoType> fPeriod;
1215 #ifndef COOLING_NONE
1218 std::vector<int> iSeTab;
1221 int nSetupWriteStage;
1222 int64_t nStartWrite;
1226 NodeLookupType nodeLookupTable;
1231 Tree::NodeKey *prefetchRoots;
1233 OrientedBox<cosmoType> prefetchReq;
1239 u_int64_t myNumMACChecks;
1240 u_int64_t nodesOpenedLocal;
1241 u_int64_t nodesOpenedRemote;
1242 u_int64_t numOpenCriterionCalls;
1244 u_int64_t nodeInterLocal;
1247 u_int64_t *nodeInterRemote;
1249 u_int64_t particleInterLocal;
1251 u_int64_t *particleInterRemote;
1256 unsigned int numBuckets;
1257 #if INTERLIST_VER > 0
1258 int prevRemoteBucket;
1263 unsigned int ewaldCurrentBucket;
1280 #if COSMO_DEBUG > 1 || defined CHANGA_REFACTOR_WALKCHECK || defined CHANGA_REFACTOR_WALKCHECK_INTERLIST
1281 typedef std::vector< std::multiset<Tree::NodeKey> > DebugList;
1283 DebugList bucketcheckList;
1292 std::vector<int> tempBinCounts;
1295 std::list<GravityParticle *> orbBoundaries;
1298 int myExpectedCount;
1301 int myExpectedCountSPH;
1304 int myExpectedCountStar;
1307 unsigned int chunkRootLevel;
1311 OrientedBox<float>* boxes;
1320 #if INTERLIST_VER > 0
1326 void finishedChunk(
int chunk);
1336 CkCallback *cbEwaldGPU;
1339 void EwaldGPUComplete();
1341 #if COSMO_DEBUG > 1 || defined CHANGA_REFACTOR_WALKCHECK || defined CHANGA_REFACTOR_WALKCHECK_INTERLIST
1342 void checkWalkCorrectness();
1344 void combineKeys(Tree::NodeKey key,
int bucket);
1351 NodeLookupType &getNodeLookupTable() {
1352 return nodeLookupTable;
1356 return nodeLookupTable.size();
1361 if(pTreeNodes != NULL) {
1365 nodeLookupTable.clear();
1369 root->fullyDelete();
1372 nodeLookupTable.clear();
1390 bool nodeOwnership(
const Tree::NodeKey nkey,
int &firstOwner,
int &lastOwner);
1396 template <
class Tsmooth>
1397 void initBucketsSmooth(Tsmooth tSmooth);
1398 void smoothNextBucket();
1399 void reSmoothNextBucket();
1400 void markSmoothNextBucket();
1401 void smoothBucketComputation();
1405 void startNextBucket();
1409 void doAllBuckets();
1414 TreePiece() : pieces(thisArrayID), root(0),
1415 iPrevRungLB (-1), sTopDown(0), sGravity(0),
1416 sPrefetch(0), sLocal(0), sRemote(0), sPref(0), sSmooth(0),
1417 nPrevActiveParts(0) {
1427 nNodeCacheEntries = 0;
1428 nPartCacheEntries = 0;
1429 completedActiveWalks = 0;
1431 nSetupWriteStage = -1;
1436 nodesOpenedLocal = 0;
1437 nodesOpenedRemote = 0;
1439 particleInterLocal = 0;
1441 numOpenCriterionCalls=0;
1448 particleInterRemote = NULL;
1449 nodeInterRemote = NULL;
1451 #if INTERLIST_VER > 0
1452 sInterListWalk = NULL;
1455 numActiveBuckets = -1;
1457 localNodeInteractions = 0;
1458 localPartInteractions = 0;
1459 remoteNodeInteractions = 0;
1460 remotePartInteractions = 0;
1461 remoteResumeNodeInteractions = 0;
1462 remoteResumePartInteractions = 0;
1470 prefetchRoots = NULL;
1473 bEwaldInited =
false;
1475 incomingParticlesMsg.clear();
1476 incomingParticlesArrived = 0;
1477 incomingParticlesSelf =
false;
1480 mySPHParticles = NULL;
1481 myStarParticles = NULL;
1482 myNumParticles = myNumSPH = myNumStar = 0;
1483 nStore = nStoreSPH = nStoreStar = 0;
1484 bBucketsInited =
false;
1485 myTreeParticles = -1;
1486 orbBoundaries.clear();
1492 createdSpanningTree =
false;
1495 localTreeBuildComplete =
false;
1498 TreePiece(CkMigrateMessage* m): pieces(thisArrayID) {
1508 nNodeCacheEntries = 0;
1509 nPartCacheEntries = 0;
1510 completedActiveWalks = 0;
1511 prefetchRoots = NULL;
1514 bEwaldInited =
false;
1522 #if INTERLIST_VER > 0
1523 sInterListWalk = NULL;
1526 incomingParticlesMsg.clear();
1527 incomingParticlesArrived = 0;
1528 incomingParticlesSelf =
false;
1531 nodeInterRemote = NULL;
1532 particleInterRemote = NULL;
1534 orbBoundaries.clear();
1537 bBucketsInited =
false;
1538 myTreeParticles = -1;
1541 localTreeBuildComplete =
false;
1545 void freeWalkObjects();
1546 void allocateStars() {
1547 nStoreStar = (int) (myNumStar*(1.0 +
dExtraStore));
1556 if (
verbosity>1) ckout <<
"Deallocating treepiece "<<thisIndex<<endl;
1557 if(nStore > 0)
delete[] myParticles;
1558 if(nStoreSPH > 0)
delete[] mySPHParticles;
1559 if(nStoreStar > 0)
delete[] myStarParticles;
1560 delete[] nodeInterRemote;
1561 delete[] particleInterRemote;
1562 delete[] bucketReqs;
1567 if(boxes!= NULL )
delete[] boxes;
1568 if(splitDims != NULL)
delete[] splitDims;
1570 #ifndef COOLING_NONE
1572 CoolDerivsFinalize(CoolData);
1574 if (
verbosity>1) ckout <<
"Finished deallocation of treepiece "<<thisIndex<<endl;
1577 void setPeriodic(
int nReplicas, Vector3D<cosmoType> fPeriod,
int bEwald,
1578 double fEwCut,
double fEwhCut,
int bPeriod,
1579 int bComove,
double dRhoFac);
1583 void calculateEwaldUsingCkLoop(
dummyMsg *msg,
int yield_num);
1584 void callBucketEwald(
int id);
1585 void doParallelNextBucketWork(
int id,
LoopParData* lpdata);
1586 void initCoolingData(
const CkCallback& cb);
1589 void velScale(
double dScale,
const CkCallback& cb);
1594 void loadNChilada(
const std::string& filename,
const double dTuFac,
1595 const CkCallback& cb);
1596 void readFloatBinary(
OutputParams& params,
int bParaRead,
1597 const CkCallback& cb);
1604 void loadTipsy(
const std::string& filename,
const double dTuFac,
1605 const bool bDoublePos,
1606 const bool bDoubleVel,
1607 const CkCallback& cb);
1609 void readTipsyArray(
OutputParams& params,
const CkCallback& cb);
1610 void resetMetals(
const CkCallback& cb);
1611 void getMaxIOrds(
const CkCallback& cb);
1612 void RestartEnergy(
double dTuFac,
const CkCallback& cb);
1613 void findTotalMass(
const CkCallback &cb);
1614 void recvTotalMass(CkReductionMsg *msg);
1617 void writeTipsy(Tipsy::TipsyWriter& w,
1619 const double duTFac,
1620 const bool bDoublePos,
1621 const bool bDoubleVel,
1624 void setupWrite(
int iStage, u_int64_t iPrevOffset,
1625 const std::string& filename,
const double dTime,
1626 const double dvFac,
const double duTFac,
1627 const bool bDoublePos,
1628 const bool bDoubleVel,
1629 const int bCool,
const CkCallback& cb);
1631 void parallelWrite(
int iPass,
const CkCallback& cb,
1632 const std::string& filename,
const double dTime,
1634 const double duTFac,
1635 const bool bDoublePos,
1636 const bool bDoubleVel,
1639 void serialWrite(u_int64_t iPrevOffset,
const std::string& filename,
1641 const double dvFac,
const double duTFac,
1642 const bool bDoublePos,
1643 const bool bDoubleVel,
1644 const int bCool,
const CkCallback& cb);
1646 void oneNodeWrite(
int iIndex,
1656 const u_int64_t iPrevOffset,
1657 const std::string& filename,
1660 const double duTFac,
1662 const bool bDoublePos,
1663 const bool bDoubleVel,
1665 const CkCallback &cb);
1667 void reOrder(int64_t nMaxOrder,
const CkCallback& cb);
1669 void ioShuffle(CkReductionMsg *msg);
1673 void resetObjectLoad(
const CkCallback& cb);
1675 void assignKeys(CkReductionMsg* m);
1676 void evaluateBoundaries(SFC::Key* keys,
const int n,
int isRefine,
const CkCallback& cb);
1677 void unshuffleParticles(CkReductionMsg* m);
1679 void shuffleAfterQD();
1680 void unshuffleParticlesWoDD(
const CkCallback& cb);
1682 void setNumExpectedNeighborMsgs();
1685 void initORBPieces(
const CkCallback& cb);
1686 void initBeforeORBSend(
unsigned int myCount,
unsigned int myCountGas,
1687 unsigned int myCountStar,
1688 const CkCallback& cb,
const CkCallback& cback);
1689 void sendORBParticles();
1690 void acceptORBParticles(
const GravityParticle* particles,
const int n);
1691 void acceptORBParticles(
const GravityParticle* particles,
const int n,
1698 void kick(
int iKickRung,
double dDelta[MAXRUNG+1],
int bClosing,
1699 int bNeedVPred,
int bGasIsothermal,
double dMaxEnergy,
double duDelta[MAXRUNG+1],
1700 double gammam1,
double dThermalCondSatCoeff,
1701 double dMultiPhaseMaxTime,
double dMultiPhaseMinTemp,
double dEvapCoeff,
const CkCallback& cb);
1702 void drift(
double dDelta,
int bNeedVPred,
int bGasIsothermal,
double dvDelta,
1703 double duDelta,
int nGrowMass,
bool buildTree,
double dMaxEnergy,
1704 const CkCallback& cb);
1705 void initAccel(
int iKickRung,
const CkCallback& cb);
1706 #ifdef COOLING_MOLECULARH
1707 void distribLymanWerner(
const CkCallback& cb);
1710 void applyFrameAcc(
int iKickRung, Vector3D<double> frameAcc,
const CkCallback& cb);
1718 const CkCallback& cb);
1739 void adjust(
int iKickRung,
int bEpsAccStep,
int bGravStep,
1740 int bSphStep,
int bViscosityLimitdt,
1741 double dEta,
double dEtaCourant,
double dEtauDot,
1742 double dDiffCoeff,
double dEtaDiffusion,
1743 double dDelta,
double dAccFac,
1744 double dCosmoFac,
double dhMinOverSoft,
1745 double dResolveJeans,
1747 const CkCallback& cb);
1753 void truncateRung(
int iCurrMaxRung,
const CkCallback& cb);
1754 void rungStats(
const CkCallback& cb);
1755 void countActive(
int activeRung,
const CkCallback& cb);
1757 void countType(
int iType,
const CkCallback& cb);
1758 void outputBlackHoles(
const std::string& pszFileName,
double dvFac,
1759 long lFPos,
const CkCallback &cb);
1761 void SetSink(
double dSinkMassMin,
const CkCallback &cb);
1763 void SinkStep(
int iCurrSinkRung,
int iKickRung,
const CkCallback &cb);
1764 void formSinks(
int bJeans,
double dJConst2,
int bDensity,
1765 double dDensityCut,
double dTime,
int iKickRung,
int bSimple,
1766 const CkCallback &cb);
1767 void emergencyAdjust(
int iRung,
double dDelta,
double dDeltaThresh,
1768 const CkCallback &cb);
1769 void assignDomain(
const CkCallback& cb);
1770 void starCenterOfMass(
const CkCallback& cb);
1771 void calcEnergy(
const CkCallback& cb);
1776 void colNParts(
const CkCallback &cb);
1778 void newOrder(
const NewMaxOrder *nStarts,
const int n,
const CkCallback &cb) ;
1781 void setNParts(int64_t _nTotalSPH, int64_t _nTotalDark,
1782 int64_t _nTotalStar,
const CkCallback &cb);
1784 void setSoft(
const double dSoft,
const CkCallback &cb);
1785 void physicalSoft(
const double dSoftMax,
const double dFac,
1786 const int bSoftMaxMul,
const CkCallback& cb);
1787 void growMass(
int nGrowMass,
double dDeltaM,
const CkCallback& cb);
1788 void InitEnergy(
double dTuFac,
double z,
double dTime,
double gammam1,
1789 const CkCallback& cb);
1790 void updateuDot(
int activeRung,
double duDelta[MAXRUNG+1],
1791 double dStartTime[MAXRUNG+1],
int bCool,
int bAll,
1792 int bUpdateState,
double gammam1,
double dResolveJeans,
const CkCallback& cb);
1793 void ballMax(
int activeRung,
double dFac,
const CkCallback& cb);
1794 void sphViscosityLimiter(
int bOn,
int activeRung,
const CkCallback& cb);
1795 void getAdiabaticGasPressure(
double gamma,
double gammam1,
double dTuFac,
double dThermalCondCoeff,
1796 double dThermalCond2Coeff,
double dThermalCondSatCoeff,
double dThermalCond2SatCoeff,
1797 double dEvapMinTemp,
double dDtCourantFac,
double dResolveJeans,
const CkCallback &cb);
1798 void getCoolingGasPressure(
double gamma,
double gammam1,
double dThermalCondCoeff,
1799 double dThermalCond2Coeff,
double dThermalCondSatCoeff,
double dThermalCond2SatCoeff,
1800 double dEvapMinTemp,
double dDtCourantFac,
double dResolveJeans,
const CkCallback &cb);
1802 void SplitGas(
double dInitGasMass,
const CkCallback& cb);
1804 inline COOL* Cool() {
return dm->Cool;}
1806 void initRand(
int iRand,
const CkCallback &cb);
1807 void FormStars(
Stfm param,
double dTime,
double dDelta,
double dCosmoFac,
1808 const CkCallback& cb);
1809 void flushStarLog(
const CkCallback& cb);
1810 void Feedback(
const Fdbk &fb,
double dTime,
double dDelta,
1811 const CkCallback& cb);
1812 void massMetalsEnergyCheck(
int bPreDist,
const CkCallback& cb);
1813 void SetTypeFromFileSweep(
int iSetMask,
char *file,
1814 struct SortStruct *ss,
int nss,
int *pniOrder,
int *pnSet);
1815 void setTypeFromFile(
int iSetMask,
char *file,
const CkCallback& cb);
1816 void getCOM(
const CkCallback& cb,
int bLiveViz);
1817 void getCOMByType(
int iType,
const CkCallback& cb,
int bLiveViz);
1818 void DumpFrame(
InDumpFrame in,
const CkCallback& cb,
int liveVizDump) ;
1819 void liveVizDumpFrameInit(liveVizRequestMsg *msg);
1820 void setProjections(
int bOn);
1824 void buildTree(
int bucketSize,
const CkCallback& cb,
bool merge);
1826 void buildTree(
int bucketSize,
const CkCallback& cb);
1830 void startOctTreeBuild(CkReductionMsg* m);
1831 void recvBoundary(SFC::Key key,
NborDir dir);
1832 void recvdBoundaries(CkReductionMsg* m);
1836 void startORBTreeBuild(CkReductionMsg* m);
1837 OrientedBox<float> constructBoundingBox(
GenericTreeNode *node,
int level,
int numChild);
1842 bool sendFillReqNodeWhenNull(CkCacheRequestMsg<KeyType> *msg);
1844 void requestRemoteMoments(
const Tree::NodeKey key,
int sender);
1845 void receiveRemoteMoments(
const Tree::NodeKey key, Tree::NodeType type,
1846 int firstParticle,
int numParticles,
int remIdx,
1848 const OrientedBox<double>& boxBall,
1849 const unsigned int iParticleTypes,
const int64_t nSPH);
1855 void calculateGravityLocal();
1858 void commenceCalculateGravityLocal();
1865 void executeCkLoopParallelization(
LoopParData *lpdata,
int startbucket,
1866 int yield_num,
int chunkNum,
State* gravityState);
1867 int doBookKeepingForTargetActive(
int curbucket,
int end,
int chunkNum,
bool
1868 updatestate,
State* gravityState);
1869 int doBookKeepingForTargetInactive(
int chunkNum,
bool updatestate,
1870 State* gravityState);
1873 void calculateSmoothLocal();
1874 void calculateReSmoothLocal();
1875 void calculateMarkSmoothLocal();
1876 void nextBucketSmooth(
dummyMsg *msg);
1877 void nextBucketReSmooth(
dummyMsg *msg);
1878 void nextBucketMarkSmooth(
dummyMsg *msg);
1879 #if INTERLIST_VER > 0
1881 void calculateForceRemoteBucket(
int bucketIndex,
int chunk);
1882 void calculateForceLocalBucket(
int bucketIndex);
1890 void startGravity(
int am,
double myTheta,
const CkCallback& cb);
1900 void startSmooth(
SmoothParams *p,
int iLowhFix,
int nSmooth,
1901 double dfBall2OverSoft2,
const CkCallback &cb);
1902 void startReSmooth(
SmoothParams *p,
const CkCallback& cb);
1903 void startMarkSmooth(
SmoothParams *p,
const CkCallback& cb);
1905 void finishNodeCache(
const CkCallback& cb);
1908 GenericTreeNode* requestNode(
int remoteIndex, Tree::NodeKey lookupKey,
1909 int chunk,
int reqID,
int awi,
void *source);
1912 void fillRequestNode(CkCacheRequestMsg<KeyType> *msg);
1918 void receiveNode(
GenericTreeNode &node,
int chunk,
unsigned int reqID);
1927 void finishBucket(
int iBucket);
1935 void cachedWalkBucketTree(
GenericTreeNode* node,
int chunk,
int reqID);
1937 #if INTERLIST_VER > 0
1939 int level,
int chunk);
1941 int level,
int chunk);
1945 int remoteIndex,
int begin,
1946 int end,
int reqID,
int awi,
1949 int remoteIndex,
int begin,
int end,
1950 int reqID,
int awi,
void *source);
1951 void fillRequestParticles(CkCacheRequestMsg<KeyType> *msg);
1952 void fillRequestSmoothParticles(CkCacheRequestMsg<KeyType> *msg);
1953 void flushSmoothParticles(CkCacheFillMsg<KeyType> *msg);
1954 void processReqSmoothParticles();
1956 void getParticleInfoForLB(int64_t active_part, int64_t total_part);
1957 void startlb(
const CkCallback &cb,
int activeRung);
1958 void setTreePieceLoad(
int activeRung);
1959 void populateSavedPhaseData(
int phase,
double tpload,
unsigned int activeparts);
1960 bool havePhaseData(
int phase);
1961 void savePhaseData(std::vector<double> &loads, std::vector<unsigned int>
1962 &parts_per_phase,
double* shuffleloads,
unsigned int *shuffleparts,
1964 void ResumeFromSync();
1967 const CkCallback& cb);
1968 void oneNodeOutVec(
OutputParams& params, Vector3D<double>* avOut,
1969 int nPart,
int iIndex,
int bDone,
1970 const CkCallback& cb) ;
1971 void oneNodeOutArr(
OutputParams& params,
double* adOut,
1972 int nPart,
int iIndex,
int bDone,
1973 const CkCallback& cb) ;
1974 void oneNodeOutIntArr(
OutputParams& params,
int *aiOut,
1975 int nPart,
int iIndex,
const CkCallback& cb);
1976 void outputBinaryStart(
OutputParams& params, int64_t nStart,
1977 const CkCallback& cb);
1978 void outputBinary(Ck::IO::Session,
OutputParams& params);
1979 void minmaxNCOut(
OutputParams& params,
const CkCallback& cb);
1981 void outputStatistics(
const CkCallback& cb);
1983 void collectStatistics(
const CkCallback &cb);
1993 void nextBucketUsingCkLoop(
dummyMsg *m);
1998 void pup(PUP::er& p);
2004 inline Vector3D<cosmoType> decodeOffset(
int reqID) {
2005 int offsetcode = reqID >> 22;
2006 int x = (offsetcode & 0x7) - 3;
2007 int y = ((offsetcode >> 3) & 0x7) - 3;
2008 int z = ((offsetcode >> 6) & 0x7) - 3;
2010 Vector3D<cosmoType> offset(x*fPeriod.x, y*fPeriod.y, z*fPeriod.z);
2015 void receiveNodeCallback(
GenericTreeNode *node,
int chunk,
int reqID,
int awi,
void *source);
2016 void receiveParticlesCallback(
ExternalGravityParticle *egp,
int num,
int chunk,
int reqID, Tree::NodeKey &remoteBucket,
int awi,
void *source);
2017 void receiveParticlesFullCallback(
GravityParticle *egp,
int num,
int chunk,
int reqID, Tree::NodeKey &remoteBucket,
int awi,
void *source);
2020 void balanceBeforeInitialForces(
const CkCallback &cb);
2026 void mergeNonLocalRequestsDone();
2028 std::map<NodeKey,NonLocalMomentsClientList>::iterator createTreeBuildMomentsEntry(
GenericTreeNode *pickedNode);
2033 std::map<NodeKey,NonLocalMomentsClientList> nonLocalMomentsClients;
2034 bool localTreeBuildComplete;
2035 int getResponsibleIndex(
int first,
int last);
2042 void deliverMomentsToClients(
const std::map<NodeKey,NonLocalMomentsClientList>::iterator &it);
2043 void treeBuildComplete();
2044 void processRemoteRequestsForMoments();
2045 void sendParticlesDuringDD(
bool withqd);
2046 void mergeAllParticlesAndSaveCentroid();
2047 bool otherIdlePesAvail();
2056 LvArray(CkMigrateMessage* m) {}
2065 #ifdef REDUCTION_HELPER
2067 class TreePieceCounter :
public CkLocIterator {
2069 TreePieceCounter() { reset(); }
2070 void addLocation(CkLocation &loc);
2074 CkVec<TreePiece *> presentTreePieces;
2079 class ReductionHelper :
public CBase_ReductionHelper {
2082 ReductionHelper(CkMigrateMessage *);
2083 void pup(PUP::er &p);
2085 void countTreePieces(
const CkCallback &cb);
2086 void reduceBinCounts(
int nBins, int64_t *binCounts,
const CkCallback &cb);
2087 void evaluateBoundaries(SFC::Key *keys,
const int n,
int isRefine,
const CkCallback& cb);
2088 void evaluateBoundaries(
const CkBitVector &binsToSplit,
const CkCallback& cb);
2091 void senseLocalTreePieces();
2095 CkVec<int64_t> myBinCounts;
2096 int numTreePiecesCheckedIn;
2098 TreePieceCounter localTreePieces;
2099 std::vector<SFC::Key> splitters;
2107 void addLocation(CkLocation &loc);
2114 #endif //PARALLELGRAVITY_H
int decodeReqID(int)
Given a requestID, return the bucket number.
Definition: TreePiece.cpp:3584
LBStrategy
Load balancers that need the spatial information.
Definition: ParallelGravity.h:66
void addToNodeInterRemote(int chunk, int howmany)
accumulate node interaction count for statistics
Definition: ParallelGravity.h:835
Cache interface to the particles for smooth calculations. This cache is a writeback cache...
Definition: CacheInterface.h:73
struct OffsetNodeStruct OffsetNode
Remote Cell interaction lists for all tree levels.
CkVec< CkVec< OffsetNode > > clists
Cell interactions.
Definition: ParallelGravity.h:673
int nIOProcessor
Number of pieces to be doing I/O at once.
Definition: ParallelGravity.cpp:119
double RungToDt(double dDelta, int iRung)
Given the size of the big step, and a rung, return the corresponding timestep size.
Definition: ParallelGravity.h:391
CProxy_IntraNodeLBManager nodeLBMgrProxy
Proxy for Managing IntraNode load balancing with ckloop.
Definition: ParallelGravity.cpp:81
A representation of a multipole expansion.
Definition: MultipoleMoments.h:163
NborDir
Directions for sending boundaries.
Definition: ParallelGravity.h:97
Class to build the remote part of the tree. Fills in Boundary and NonLocal nodes. ...
Definition: Compute.h:286
List of clients needing a particular moment.
Definition: ParallelGravity.h:712
Base clase for all tree based computations.
Definition: Compute.h:26
int verbosity
verbosity level. Higher is more verbose.
Definition: ParallelGravity.cpp:63
int64_t nTotalParticles
Total Particles in the simulation.
Definition: ParallelGravity.h:1094
CkQ< OffsetNode > CheckList
Queue of nodes to check for interactions.
Definition: ParallelGravity.h:629
int _numChunks
Definition: ParallelGravity.cpp:121
Message to efficiently start entry methods with no arguments.
Definition: ParallelGravity.h:206
Parameters and methods to implement star formation.
Definition: starform.h:10
double dFracLoadBalance
Definition: ParallelGravity.cpp:104
Coefficients for the Fourier space part of the Ewald sum.
Definition: ParallelGravity.h:611
int localNodesPerReq
GPU related settings.
Definition: ParallelGravity.cpp:131
Hold parameters of the run.
Definition: parameters.h:14
const int PHASE_FEEDBACK
slot in MultistepLB to hold feedback phase load information
Definition: ParallelGravity.h:396
GravityParticle * particlePointer
Pointer to the first particle in this node.
Definition: GenericTreeNode.h:117
CkVec< CkVec< RemotePartInfo > > rpilists
Remote particle interactions.
Definition: ParallelGravity.h:674
Class for new maxOrder broadcast.
Definition: ParallelGravity.h:122
double dGlassDamper
for doing load balancing.
Definition: ParallelGravity.cpp:106
Message to start a remote gravity walk.
Definition: ParallelGravity.h:270
Class to compute gravity using a "bucket walk".
Definition: Compute.h:105
Used to count non-empty treepieces on the local processor.
Definition: ParallelGravity.h:2104
int _nocache
Disable the cache (always off)
Definition: ParallelGravity.cpp:93
unsigned int _yieldPeriod
The number of buckets to process in the local gravity walk before yielding the processor.
Definition: ParallelGravity.cpp:99
Compute for the remote node prefetch walk.
Definition: Compute.h:226
bool bIsReplica(int reqID)
is this a periodic replica?
Definition: TreePiece.cpp:3573
Computation over "inverse" nearest neighbors.
Definition: smooth.h:224
External gravity parameters and routines.
Definition: externalGravity.h:7
Cache interface to particles for the gravity calculation. This is a read-only cache of particles...
Definition: CacheInterface.h:37
CProxy_CkCacheManager< KeyType > cacheSmoothPart
Proxy for the smooth particle cache group.
Definition: ParallelGravity.cpp:75
GenericTrees useTree
type of tree to use.
Definition: ParallelGravity.cpp:110
Computations for Stadel-style interaction list walk.
Definition: Compute.h:148
CProxy_PETreeMerger peTreeMergerProxy
Proxy for the PETreeMerger group.
Definition: ParallelGravity.cpp:86
CkCallback cb
Callback for reduction of particle counts.
Definition: ParallelGravity.h:293
CProxy_LvArray lvProxy
Proxy for the liveViz array.
Definition: ParallelGravity.cpp:69
int64_t nTotalStar
Total Star Particles.
Definition: ParallelGravity.h:1100
unsigned int numTreePieces
Number of pieces into which to divide the tree.
Definition: ParallelGravity.cpp:115
double dExtraStore
fraction of extra particle storage
Definition: ParallelGravity.cpp:102
Base class for output parameters.
Definition: InOutput.h:16
int RungToSubsteps(int iRung)
Given a rung, return the number of substeps in one big step.
Definition: ParallelGravity.h:370
const double ddTolerance
tolerance for unequal pieces in SFC based decompositions.
Definition: ParallelGravity.h:104
Definition: DataManager.h:60
Base class for walking trees.
Definition: TreeWalk.h:11
int _cacheLineDepth
Size of a Node Cache line, specified by how deep in the tree it goes.
Definition: ParallelGravity.cpp:96
const double GAMMA_JEANS
Adiabatic index to use with the Jeans pressure floor.
Definition: ParallelGravity.h:423
int DtToRung(double dDelta, double dTideal)
Given the size of the big step, and a desired timestep, return the rung of the largest timestep less ...
Definition: ParallelGravity.h:377
Object containing the parameter information.
Definition: param.h:38
struct particlesInfoR RemotePartInfo
Remote particles in an interaction list.
void addToNodeInterLocal(int howmany)
accumulate node interaction count for statistics
Definition: ParallelGravity.h:845
CkVec< int > chunkids
remote walk chunk number
Definition: ParallelGravity.h:672
client that has requested a moment.
Definition: ParallelGravity.h:696
int weightBalanceUE
Weight balancing during Oct decomposition user event (for Projections tracing).
Definition: ParallelGravity.cpp:149
Class to build the local part of the tree. Builds Internal nodes.
Definition: Compute.h:305
double * pos
Positions of splits.
Definition: ParallelGravity.h:289
cosmoType theta
BH-like opening criterion.
Definition: ParallelGravity.cpp:142
struct LoopParDataStruct LoopParData
Data needed for the CkLoop intranode parallelization.
CProxy_TreePiece treeProxy
Proxy for the TreePiece chare array.
Definition: ParallelGravity.cpp:65
int _prefetch
Prefetch nodes for the remote walk.
Definition: ParallelGravity.cpp:120
Object to bookkeep a Bucket MarkSmooth Walk.
Definition: smooth.h:261
char * dim
Dimension of splits.
Definition: ParallelGravity.h:291
CProxy_CkCacheManager< KeyType > cacheNode
Proxy for the tree node cache group.
Definition: ParallelGravity.cpp:77
Base class for tree nodes.
Definition: GenericTreeNode.h:59
Class for shadow arrays to avoid reduction conflicts in overlapping liveViz, SPH and gravity...
Definition: ParallelGravity.h:2053
CProxy_CkCacheManager< KeyType > cacheGravPart
Proxy for the gravity particle cache group.
Definition: ParallelGravity.cpp:73
CProxy_DumpFrameData dfDataProxy
Proxy for the dumpframe image data (DumpFrameData).
Definition: ParallelGravity.cpp:84
Base class for maintaining the state of a tree walk.
Definition: State.h:6
Information needed to calculate gravity.
Definition: GravityParticle.h:33
CkVec< OffsetNode > UndecidedList
Vector of nodes that are undecided at this level.
Definition: ParallelGravity.h:631
int numInitDecompBins
Definition: ParallelGravity.cpp:182
Data needed for a TreePiece to render its part of the image.
Definition: dumpframe.h:116
Message for evaluating splits for the ORB domain decomposition.
Definition: ParallelGravity.h:284
Object to bookkeep a Bucket Walk.
Definition: GravityParticle.h:18
moment tensor components for complete multipoles.
Definition: moments.h:40
int lastParticle
An index to the last particle contained by this node, myNumParticles+1 means outside the node...
Definition: GenericTreeNode.h:108
Data needed for the CkLoop intranode parallelization.
Definition: ParallelGravity.h:668
int boundaryEvaluationUE
only cells.
Definition: ParallelGravity.cpp:147
Definition: TreeWalk.h:82
int encodeOffset(int reqID, int x, int y, int z)
Given a bucket number and a periodic offset, encode these into one number.
Definition: TreePiece.cpp:3593
int rungs
Definition: GenericTreeNode.h:122
Particle data needed on the GPU to calculate gravity.
Definition: cuda_typedef.h:231
Fundamental structure that holds particle and tree data.
Definition: ParallelGravity.h:730
CkVec< CkVec< LocalPartInfo > > lpilists
Local particle interactions.
Definition: ParallelGravity.h:675
Overall flow control of the simulation.
Definition: ParallelGravity.h:436
void CkMustAssert(bool cond, const char *err)
CkAssert() replacement works even in production mode.
Definition: ParallelGravity.h:56
Heating/Cooling context: parameters and tables.
Definition: cooling_boley.h:83
Stellar/Supernova feedback parameters and routines.
Definition: feedback.h:45
Object to bookkeep a Bucket Smooth Walk.
Definition: smooth.h:27
Global simulation parameters for dumpframe.
Definition: dumpframe.h:209
int firstParticle
An index for the first particle contained by this node, 0 means outside the node. ...
Definition: GenericTreeNode.h:106
int64_t nTotalDark
Total Dark Particles.
Definition: ParallelGravity.h:1098
A base class from which parameters for all smooth operations can be derived.
Definition: smoothparams.h:8
CProxy_TreePiece streamingProxy
A potentially optimized proxy for the tree pieces. Its use is deprecated.
Definition: ParallelGravity.cpp:113
unsigned int particlesPerChare
Number of particles per TreePiece. Used to determine the number of TreePieces.
Definition: ParallelGravity.cpp:118
Class for computation over a set smoothing length.
Definition: smooth.h:186
int getNumParticles()
Return the number of particles on this TreePiece.
Definition: ParallelGravity.h:861
unsigned int bucketSize
Definition: ParallelGravity.cpp:125
int bUseCkLoopPar
Use Ckloop for node parallelization.
Definition: ParallelGravity.cpp:127
CkVec< UndecidedList > UndecidedLists
Vector of undecided lists, one for each level.
Definition: ParallelGravity.h:633
DomainsDec
Possible domain decomposition methods.
Definition: ParallelGravity.h:86
void addToParticleInterLocal(int howmany)
accumulate particle interaction count for statistics
Definition: ParallelGravity.h:850
CkVec< GenericTreeNode * > lowNodes
Definition: ParallelGravity.h:669
structure to hold information specific to GPU Ewald
Definition: EwaldCUDA.h:65
double dStartTime
Time read in from input file.
Definition: ParallelGravity.h:1049
Class to count added and deleted particles.
Definition: ParallelGravity.h:328
void addToParticleInterRemote(int chunk, int howmany)
accumulate particle interaction count for statistics
Definition: ParallelGravity.h:840
Cache interface to the Tree Nodes.
Definition: CacheInterface.h:96
Class for computation over k nearest neighbors.
Definition: smooth.h:121
TreePiece * tp
Treepiece that owns this data.
Definition: ParallelGravity.h:676
Base class for optimizing walk actions.
Definition: Opt.h:12
cosmoType thetaMono
Definition: ParallelGravity.cpp:143
Fundamental type for a particle.
Definition: GravityParticle.h:316
double PoverRhoFloorJeans(double dResolveJeans, GravityParticle *p)
Pressure floor to force Jeans length to be larger than the spatial resolution.
Definition: ParallelGravity.h:401
CProxy_Main mainChare
Proxy for Charm Main Chare.
Definition: ParallelGravity.cpp:61
Utility to pool allocations of tree nodes.
Definition: GenericTreeNode.h:324
CProxy_LvArray gravityProxy
Definition: ParallelGravity.cpp:71
void deleteTree()
delete treenodes if allocated
Definition: ParallelGravity.h:1360
Local particles in an interaction list.
Definition: ParallelGravity.h:650
void printGenericTree(GenericTreeNode *node, std::ostream &os)
Print a text version of a tree.
Definition: TreePiece.cpp:6020
int octRefineLevel
Definition: ParallelGravity.cpp:186
context for calculating cooling derivatives
Definition: cooling_boley.h:108
CProxy_LvArray smoothProxy
Proxy for smooth reductions.
Definition: ParallelGravity.cpp:70
Super class for Smooth and Resmooth computation.
Definition: smooth.h:78
struct particlesInfoL LocalPartInfo
Local particles in an interaction list.
bool _cache
Use the cache (always on)
Definition: ParallelGravity.cpp:91
Remote Cell interaction lists for all tree levels.
Definition: ParallelGravity.h:621
int length
Number of splits.
Definition: ParallelGravity.h:287
struct ewaldTable EWT
Coefficients for the Fourier space part of the Ewald sum.
double fBall
Neighbor search radius for smoothing.
Definition: GravityParticle.h:326
GravityParticle * getParticles()
Return the pointer to the particles on this TreePiece.
Definition: ParallelGravity.h:866
Message for shuffling particles during domain decomposition.
Definition: ParallelGravity.h:300
int _randChunks
Definition: ParallelGravity.cpp:123
structure for efficiently reading iOrders.
Definition: TreePiece.cpp:2430
DomainsDec domainDecomposition
The type of domain decomposition to use.
Definition: ParallelGravity.cpp:101
CProxy_DataManager dMProxy
Proxy for the DataManager.
Definition: ParallelGravity.cpp:79
Remote particles in an interaction list.
Definition: ParallelGravity.h:637
int64_t nTotalSPH
Total Gas Particles.
Definition: ParallelGravity.h:1096
double dMaxBalance
Max piece imbalance for load balancing.
Definition: ParallelGravity.cpp:103
CkGroupID dataManagerID
The group ID of your DataManager. You must set this!
Definition: ParallelGravity.cpp:157