escript  Revision_Unversioneddirectory
Esys_MPI.h
Go to the documentation of this file.
1 
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2016 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Apache License, version 2.0
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16 
17 
18 #ifndef INC_ESYS_MPI
19 #define INC_ESYS_MPI
20 
21 #include "system_dep.h"
22 #include "types.h"
23 
24 #include <sstream>
25 #include <boost/shared_ptr.hpp>
26 
27 #ifdef ESYS_MPI
28 
29 #include <mpi.h>
30 
31 #ifdef ESYS_INDEXTYPE_LONG
32 #define MPI_DIM_T MPI_LONG
33 #else
34 #define MPI_DIM_T MPI_INT
35 #endif
36 
37 #else
38  typedef int MPI_Comm;
39  typedef int MPI_Request;
40  typedef int MPI_Op;
41  typedef int MPI_Status;
42  #define MPI_INT 6
43  #define MPI_DOUBLE 11
44  #define MPI_COMM_WORLD 91
45  #define MPI_COMM_NULL 0
46 
47 // MPI_Op replacements for non-MPI - these values are arbitrary
48 
49  #define MPI_SUM 100
50  #define MPI_MIN 101
51  #define MPI_MAX 102
52 
53  #define MPI_OP_NULL 17
54 // end MPI_op
55 
56 
57 #endif
58 
59 typedef int Esys_MPI_rank;
60 
61 #define ESYS_MPI_TODO { fprintf( stdout, "\nTODO : %s:%d\n", __FILE__, __LINE__); MPI_Finalize(); exit(1); }
62 
63 // Modding by 7 digit prime to avoid overflow
64 #define ESYS_MPI_INC_COUNTER(V,I) {(V).msg_tag_counter=((V).msg_tag_counter+(I))%1010201;}
65 #define ESYS_MPI_SET_COUNTER(V,I) {(V).msg_tag_counter=(I)%1010201;}
66 
67 namespace esysUtils {
68 
74 inline int getSubWorldTag()
75 {
76  return (('S'<< 24) + ('u' << 16) + ('b' << 8) + 'W')%1010201;
77 }
78 
79 class JMPI_;
80 
81 typedef boost::shared_ptr<JMPI_> JMPI;
82 
83 class JMPI_
84 {
85 public:
86  ~JMPI_();
87  int size;
91  bool ownscomm; // if true, destroy comm on destruct
92 
93  dim_t setDistribution(index_t min_id,index_t max_id,index_t* distribution);
94  void split(dim_t N, dim_t* local_N,index_t* offset);
95 
96  void incCounter(int i)
97  {
98  msg_tag_counter+=i;
99  msg_tag_counter%=1010201; // there is no particular significance here other than being 7 digits
100  } // and prime (because why not). It just needs to be big.
101 
102  void setCounter(int i)
103  {
104  msg_tag_counter%=1010201;
105  }
106 
107  bool isValid()
108  {
109  return comm!=MPI_COMM_NULL;
110  }
111 private:
112  JMPI_(MPI_Comm comm, bool ocomm);
113  friend JMPI makeInfo(MPI_Comm comm, bool owncom);
114 };
115 
116 JMPI makeInfo(MPI_Comm comm, bool owncom=false);
117 
119 bool Esys_MPIInfo_noError( const JMPI& mpi_info);
120 
123 
124 
127 inline std::string appendRankToFileName(const std::string &fileName,
128  int mpiSize, int mpiRank)
129 {
130  std::stringstream ss;
131  ss << fileName;
132  if (mpiSize > 1) {
133  ss << '.';
134  ss.fill('0');
135  ss.width(4);
136  ss << mpiRank;
137  }
138  std::string result(ss.str());
139  return result;
140 }
141 
142 // ensure that the any ranks with an empty src argument end up with the string from
143 // one of the other ranks
144 // with no-mpi, it makes dest point at a copy of src
146 bool shipString(const char* src, char** dest, MPI_Comm& comm);
147 
148 
149 // Everyone puts in their error code and everyone gets the largest one
151 bool checkResult(int input, int& output, const JMPI& comm);
152 
153 
154 // Does not cope with nested calls
156 {
157 public:
158  NoCOMM_WORLD();
159  ~NoCOMM_WORLD();
160  static bool active();
161 };
162 
163 } // namespace esysUtils
164 
165 #endif /* INC_ESYS_MPI */
166 
int msg_tag_counter
Definition: Esys_MPI.h:90
int getSubWorldTag()
tag reserved for use by SubWorld code this value should be higher than the modulus used in JMPI_::set...
Definition: Esys_MPI.h:74
#define MPI_COMM_NULL
Definition: Esys_MPI.h:45
void split(dim_t N, dim_t *local_N, index_t *offset)
Definition: Esys_MPI.cpp:108
bool ownscomm
Definition: Esys_MPI.h:91
bool isValid()
Definition: Esys_MPI.h:107
bool shipString(const char *src, char **dest, MPI_Comm &comm)
Definition: Esys_MPI.cpp:273
index_t mod_rank(index_t n, index_t k)
Definition: Esys_MPI.cpp:156
void setCounter(int i)
Definition: Esys_MPI.h:102
int MPI_Comm
Definition: Esys_MPI.h:38
friend JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: Esys_MPI.cpp:35
#define ESYSUTILS_DLL_API
Definition: esysUtils/src/system_dep.h:31
dim_t setDistribution(index_t min_id, index_t max_id, index_t *distribution)
Definition: Esys_MPI.cpp:79
int MPI_Request
Definition: Esys_MPI.h:39
Definition: Esys_MPI.h:155
void incCounter(int i)
Definition: Esys_MPI.h:96
MPI_Comm comm
Definition: Esys_MPI.h:89
~JMPI_()
Definition: Esys_MPI.cpp:69
bool checkResult(int input, int &output, const JMPI &comm)
Definition: Esys_MPI.cpp:197
static dim_t N
Definition: SparseMatrix_saveHB.cpp:37
Esys_MPI_rank rank
Definition: Esys_MPI.h:88
Definition: Esys_MPI.cpp:32
int index_t
Definition: types.h:24
int Esys_MPI_rank
Definition: Esys_MPI.h:59
int MPI_Op
Definition: Esys_MPI.h:40
Definition: Esys_MPI.h:83
index_t dim_t
Definition: types.h:27
int MPI_Status
Definition: Esys_MPI.h:41
JMPI_(MPI_Comm comm, bool ocomm)
Definition: Esys_MPI.cpp:46
bool Esys_MPIInfo_noError(const JMPI &mpi_info)
Definition: Esys_MPI.cpp:173
boost::shared_ptr< JMPI_ > JMPI
Definition: Esys_MPI.h:79
std::string appendRankToFileName(const std::string &fileName, int mpiSize, int mpiRank)
Appends MPI rank to a file name if MPI size > 1.
Definition: Esys_MPI.h:127
int size
Definition: Esys_MPI.h:87