1#include "../include/miluphpc.h"
2#include "../include/integrator/explicit_euler.h"
3#include "../include/integrator/predictor_corrector_euler.h"
4#include "../include/integrator/leapfrog.h"
5#include "../include/utils/config_parser.h"
6#include <boost/filesystem.hpp>
15#define ENV_LOCAL_RANK "OMPI_COMM_WORLD_LOCAL_RANK"
17bool checkFile(
const std::string file,
bool terminate=
false,
const std::string message=
"");
18bool checkFile(
const std::string file,
bool terminate,
const std::string message) {
19 std::ifstream fileStream(file.c_str());
20 if (fileStream.good()) {
26 Logger(
ERROR) <<
"Provided file: " << file <<
" not available!";
36 char * localRankStr = NULL;
42 rank = atoi(localRankStr);
52int main(
int argc,
char** argv)
61 boost::mpi::environment env(argc, argv);
62 boost::mpi::communicator comm;
64 int rank = comm.rank();
65 int numProcesses = comm.size();
70 cudaGetDevice(&device);
72 cudaGetDeviceCount(&numDevices);
73 cudaDeviceSynchronize();
75 cudaDeviceProp deviceProp;
76 cudaGetDeviceProperties(&deviceProp, device);
77 int numberOfMultiprocessors = deviceProp.multiProcessorCount;
78 printf(
"numberOfMultiprocessors: %i\n", numberOfMultiprocessors);
86 cxxopts::Options options(
"HPC NBody",
"Multi-GPU CUDA Barnes-Hut NBody/SPH code");
91 (
"n,number-output-files",
"number of output files", cxxopts::value<int>()->default_value(
"100"))
92 (
"t,max-time-step",
"time step", cxxopts::value<real>()->default_value(
"-1."))
93 (
"l,load-balancing",
"load balancing", cxxopts::value<bool>(
loadBalancing))
94 (
"L,load-balancing-interval",
"load balancing interval", cxxopts::value<int>()->default_value(
"-1"))
95 (
"C,config",
"config file", cxxopts::value<std::string>()->default_value(
"config/config.info"))
96 (
"m,material-config",
"material config file", cxxopts::value<std::string>()->default_value(
"config/material.cfg"))
97 (
"c,curve-type",
"curve type (Lebesgue: 0/Hilbert: 1)", cxxopts::value<int>()->default_value(
"-1"))
98 (
"f,input-file",
"File name", cxxopts::value<std::string>()->default_value(
"-"))
99 (
"v,verbosity",
"Verbosity level", cxxopts::value<int>()->default_value(
"0"))
100 (
"h,help",
"Show this help");
102 cxxopts::ParseResult result;
104 result = options.parse(argc, argv);
106 catch (cxxopts::OptionException &exception) {
107 std::cerr << exception.what();
112 std::cerr <<
"Error parsing ...";
117 if (result.count(
"help")) {
119 std::cout << options.help() << std::endl;
126 std::string configFile = result[
"config"].as<std::string>();
127 checkFile(configFile,
true, std::string{
"Provided config file not available!"});
132 parameters.
verbosity = result[
"verbosity"].as<
int>();
135 if (parameters.verbosity == 0) {
138 else if (parameters.verbosity == 1) {
141 else if (parameters.verbosity >= 2) {
157 Logger(
DEBUG) <<
"rank: " << rank <<
" | number of processes: " << numProcesses;
158 Logger(
DEBUG) <<
"device: " << device <<
" | num devices: " << numDevices;
161 Logger(
INFO) <<
"sizeof(intmax_t) = " << (int)
sizeof(intmax_t);
162 Logger(
INFO) <<
"sizeof(uintmax_t) = " << (int)
sizeof(uintmax_t);
163 Logger(
INFO) <<
"sizeof(__int128) = " << (int)
sizeof(__int128);
169 parameters.directory = confP.getVal<std::string>(
"directory");
170 if (boost::filesystem::create_directories(parameters.directory)) {
171 Logger(
DEBUG) <<
"Created directory: " << parameters.directory;
173 parameters.logDirectory = parameters.directory + std::string{
"log/"};
174 if (boost::filesystem::create_directories(parameters.logDirectory)) {
175 Logger(
DEBUG) <<
"Created directory: " << parameters.logDirectory;
178 std::stringstream logFileName;
179 logFileName << parameters.logDirectory <<
"miluphpc.log";
181 Logger(
TRACE) <<
"log file to: " << logFileName.str();
185 parameters.timeStep = confP.getVal<
real>(
"timeStep");
186 parameters.maxTimeStep = result[
"max-time-step"].as<
real>();
187 if (parameters.maxTimeStep < 0.) {
188 parameters.maxTimeStep = confP.getVal<
real>(
"maxTimeStep");
190 parameters.timeEnd = confP.getVal<
real>(
"timeEnd");
191 parameters.outputRank = confP.getVal<
int>(
"outputRank");
192 if (parameters.outputRank < 0 || parameters.outputRank >= numProcesses) {
193 parameters.outputRank = -1;
196 parameters.performanceLog = confP.getVal<
bool>(
"performanceLog");
197 parameters.particlesSent2H5 = confP.getVal<
bool>(
"particlesSent2H5");
198 parameters.sfcSelection = confP.getVal<
int>(
"sfc");
199 if (result[
"curve-type"].as<int>() != -1) {
200 parameters.sfcSelection = result[
"curve-type"].as<
int>();
202 parameters.integratorSelection = confP.getVal<
int>(
"integrator");
204 parameters.theta = confP.getVal<
real>(
"theta");
205 parameters.smoothing = confP.getVal<
real>(
"smoothing");
206 parameters.gravityForceVersion = confP.getVal<
int>(
"gravityForceVersion");
209 parameters.smoothingKernelSelection = confP.getVal<
int>(
"smoothingKernel");
210 parameters.sphFixedRadiusNNVersion = confP.getVal<
int>(
"sphFixedRadiusNNVersion");
212 parameters.removeParticles = confP.getVal<
bool>(
"removeParticles");
213 parameters.removeParticlesCriterion = confP.getVal<
int>(
"removeParticlesCriterion");
214 parameters.removeParticlesDimension = confP.getVal<
real>(
"removeParticlesDimension");
215 parameters.numOutputFiles = result[
"number-output-files"].as<
int>();
216 parameters.timeKernels =
true;
217 parameters.loadBalancing = confP.getVal<
bool>(
"loadBalancing");
219 parameters.loadBalancing =
true;
221 parameters.loadBalancingInterval = confP.getVal<
int>(
"loadBalancingInterval");
222 if (result[
"load-balancing-interval"].as<int>() > 0) {
223 parameters.loadBalancingInterval = result[
"load-balancing-interval"].as<
int>();
225 parameters.loadBalancingBins = confP.getVal<
int>(
"loadBalancingBins");
226 parameters.verbosity = result[
"verbosity"].as<
int>();
227 parameters.materialConfigFile = result[
"material-config"].as<std::string>();
228 parameters.inputFile = result[
"input-file"].as<std::string>();
229 parameters.particleMemoryContingent = confP.getVal<
real>(
"particleMemoryContingent");
230 if (parameters.particleMemoryContingent > 1.0 || parameters.particleMemoryContingent < 0.0) {
231 parameters.particleMemoryContingent = 1.0;
232 Logger(
WARN) <<
"Setting particle memory contingent to: " << parameters.particleMemoryContingent;
234 parameters.calculateCenterOfMass = confP.getVal<
bool>(
"calculateCenterOfMass");
236 parameters.calculateAngularMomentum = confP.getVal<
bool>(
"calculateAngularMomentum");
237 parameters.calculateEnergy = confP.getVal<
bool>(
"calculateEnergy");
241 Logger(
DEBUG) <<
"domainListSize: " << parameters.domainListSize;
243 if (!
checkFile(parameters.materialConfigFile,
false)) {
244 parameters.materialConfigFile = std::string{
"config/material.cfg"};
245 checkFile(parameters.materialConfigFile,
true,
246 std::string{
"Provided material config file and default (config/material.cfg) not available!"});
248 checkFile(parameters.inputFile,
true, std::string{
"Provided input file not available!"});
253 std::stringstream profilerFile;
254 profilerFile << parameters.logDirectory <<
"performance.h5";
258 if (!parameters.performanceLog) {
321 Logger(
INFO) <<
"Gravity is decoupled! Therefore switching to explicit Euler integrator!";
330 switch (parameters.integratorSelection) {
338 miluphpc =
new Leapfrog(parameters);
348 Logger(
TRACE) <<
"---------------STARTING---------------";
356 for (
int i_step=0; i_step<parameters.numOutputFiles; i_step++) {
360 Logger(
TRACE) <<
"-----------------------------------------------------------------";
362 Logger(
TRACE) <<
"-----------------------------------------------------------------";
365 Logger(
DEBUG) <<
"subEndTime += " << (parameters.timeEnd/(
real)parameters.numOutputFiles);
375 Logger(
TIME) <<
"particles2file: " << timeElapsed <<
" ms";
379 t += parameters.timeStep;
389 Logger(
TRACE) <<
"---------------FINISHED---------------";
390 Logger(
TRACE) <<
"Input file: " << parameters.inputFile;
391 Logger(
TRACE) <<
"Config file: " << parameters.materialConfigFile;
392 Logger(
TRACE) <<
"Material config: " << parameters.materialConfigFile;
396 if (parameters.performanceLog) {
397 Logger(
TRACE) <<
"Performance log saved to " << profilerFile.str();
399 if (parameters.particlesSent2H5) {
400 Logger(
TRACE) <<
"(Most recent) particles sent saved to: " << parameters.logDirectory;
402 Logger(
TRACE) <<
"Generated " << parameters.numOutputFiles <<
" files!";
403 Logger(
TRACE) <<
"Data saved to " << parameters.directory;
404 Logger(
TRACE) <<
"---------------FINISHED---------------";
Config parser class for reading input parameter/settings.
Singleton class for HDF5 profiler.
static H5Profiler & getInstance(const std::string &outfile="")
Constructor/Instance getter for HDF5 profiler.
void const setNumProcs(const int &_numProcs)
Set number of MPI processes.
void createVectorDataSet(const std::string &path, int steps, std::size_t size, std::size_t maxSteps=HighFive::DataSpace::UNLIMITED)
Track vector values (per MPI rank).
void createValueDataSet(const std::string &path, int steps, std::size_t maxSteps=HighFive::DataSpace::UNLIMITED)
Track single value (per MPI rank).
void const setRank(const int &_myRank)
Set MPI rank.
void const disableWrite()
Disable write to output file.
void value2file(const std::string &path, T value)
Write value to single value data set.
void afterIntegrationStep()
virtual void integrate(int step=0)=0
SimulationTimeHandler * simulationTimeHandler
Instance to handle the SimulationTime instances on device and host.
real particles2file(int step)
void copy(To::Target target)
double elapsed() const
Get elapsed time since instantiation/latest reset.
void reset()
Reset timer instance.
#define gpuErrorcheck(ans)
check CUDA call
int main(int argc, char **argv)
void SetDeviceBeforeInit()
bool checkFile(const std::string file, bool terminate=false, const std::string message="")
const char *const gravityParticles
const char *const gravityPseudoParticles
const char *const gravityPseudoParticles
const char *const gravityParticles
const char *const compTheta
const char *const repairTree
const char *const symbolicForce
const char *const insertReceivedParticles
const char *const insertReceivedPseudoParticles
const char *const sending
const char *const insertReceivedParticles
const char *const internalForces
const char *const soundSpeed
const char *const symbolicForce
const char *const pressure
const char *const sending
const char *const fixedRadiusNN
const char *const determineSearchRadii
const char *const repairTree
const char *const density
const char *const compTheta
const char *const buildDomain
const char *const createDomain
const char *const pseudoParticle
const char *const gravity
const char *const removeParticles
const char *const loadBalancing
const char *const assignParticles
const char *const boundingBox
const char *const integrate
const char *const rhsElapsed
const char *const numParticlesLocal
const char *const numParticles
@ predictor_corrector_euler
int verbosity
verbosity level
typeLog level
Minimum logging level to be shown.
bool omitTime
omit time output/logging
std::string logFileName
log file to be written
bool write2LogFile
write additionally to log file
int outputRank
MPI rank to be displayed (default: -1 -> display all)
int rank
whether to use MPI