Zoltan2
Loading...
Searching...
No Matches
Zoltan2_AlltoAll.hpp
Go to the documentation of this file.
1// @HEADER
2//
3// ***********************************************************************
4//
5// Zoltan2: A package of combinatorial algorithms for scientific computing
6// Copyright 2012 Sandia Corporation
7//
8// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9// the U.S. Government retains certain rights in this software.
10//
11// Redistribution and use in source and binary forms, with or without
12// modification, are permitted provided that the following conditions are
13// met:
14//
15// 1. Redistributions of source code must retain the above copyright
16// notice, this list of conditions and the following disclaimer.
17//
18// 2. Redistributions in binary form must reproduce the above copyright
19// notice, this list of conditions and the following disclaimer in the
20// documentation and/or other materials provided with the distribution.
21//
22// 3. Neither the name of the Corporation nor the names of the
23// contributors may be used to endorse or promote products derived from
24// this software without specific prior written permission.
25//
26// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37//
38// Questions? Contact Karen Devine (kddevin@sandia.gov)
39// Erik Boman (egboman@sandia.gov)
40// Siva Rajamanickam (srajama@sandia.gov)
41//
42// ***********************************************************************
43//
44// @HEADER
45
50#ifndef _ZOLTAN2_ALLTOALL_HPP_
51#define _ZOLTAN2_ALLTOALL_HPP_
52
53#include <Zoltan2_Standards.hpp>
55
56#include <vector>
57#include <climits>
58
59namespace Zoltan2
60{
61
62extern void AlltoAllCount(const Comm<int> &comm, const Environment &env,
63 const ArrayView<const int> &sendCount, const ArrayView<int> &recvCount);
64
92template <typename T>
93void AlltoAllv(const Comm<int> &comm,
94 const Environment &env,
95 const ArrayView<const T> &sendBuf,
96 const ArrayView<const int> &sendCount,
97 ArrayRCP<T> &recvBuf, // output, allocated here
98 const ArrayView<int> &recvCount // output
99)
100{
101 int nprocs = comm.getSize();
102 int rank = comm.getRank();
103
104 try{
105 Zoltan2::AlltoAllCount(comm, env, sendCount, recvCount);
106 }
108
109 // Allocate the receive buffer.
110 size_t totalrecv = 0;
111 int maxMsg = 0;
112 int nrecvranks = 0;
113 for(int i = 0; i < nprocs; i++) {
114 if (recvCount[i] > 0) {
115 totalrecv += recvCount[i];
116 nrecvranks++;
117 if (recvCount[i] > maxMsg) maxMsg = recvCount[i];
118 }
119 }
120
121
122 T *rbuf = NULL;
123 if (totalrecv) rbuf = new T[totalrecv];
124
125 if (nprocs > 1) {
126
127 RCP<CommRequest<int> > *requests = new RCP<CommRequest<int> > [nrecvranks];
128
129 // Error checking for memory and message size.
130 int OK[2] = {1,1};
131 // OK[0] -- true/false indicating whether each message size
132 // fits in an int (for MPI).
133 // OK[1] -- true/false indicating whether memory allocs are OK
134 int gOK[2]; // For global reduce of OK.
135
136 if (size_t(maxMsg) * sizeof(T) > INT_MAX && nprocs > 1) OK[0] = false;
137 if (totalrecv && !rbuf) OK[1] = 0;
138 if (!requests) OK[1] = 0;
139
140 // Post receives
141
142 size_t offset = 0;
143 size_t myrecvoffset = 0;
144 size_t mysendoffset = 0;
145
146 if (OK[0] && OK[1]) {
147 int rcnt = 0;
148 for (int i = 0; i < nprocs; i++) {
149 if (i != rank && recvCount[i]) {
150 try {
151 requests[rcnt++] = Teuchos::ireceive<int,T>(comm,
152 Teuchos::arcp(&rbuf[offset],0,recvCount[i],false),
153 i);
154 }
156 }
157 else if (i == rank) {
158 myrecvoffset = offset;
159 }
160 offset += recvCount[i];
161 }
162 }
163
164 // Use barrier for error checking
165 Teuchos::reduceAll<int>(comm, Teuchos::REDUCE_MIN, 2, OK, gOK);
166 if (!gOK[0] || !gOK[1]) {
167 delete [] rbuf;
168 delete [] requests;
169 if (!gOK[0])
170 throw std::runtime_error("Max single message length exceeded");
171 else
172 throw std::bad_alloc();
173 }
174
175 // Send data; can use readySend since receives are posted.
176 offset = 0;
177 for (int i = 0; i < nprocs; i++) {
178 if (i != rank && sendCount[i]) {
179 try {
180 Teuchos::readySend<int,T>(comm,
181 Teuchos::arrayView(&sendBuf[offset],sendCount[i]),
182 i);
183 }
185 }
186 else if (i == rank) {
187 mysendoffset = offset;
188 }
189 offset += sendCount[i];
190 }
191
192 // Copy local data
193 for (int j = 0; j < sendCount[rank]; j++)
194 rbuf[myrecvoffset++] = sendBuf[mysendoffset++];
195
196 // Wait for messages to return.
197 try {
198 Teuchos::waitAll<int>(comm, Teuchos::arrayView(requests, nrecvranks));
199 }
201
202 delete [] requests;
203 }
204 else { // nprocs == 1; no communication needed
205
206 if (totalrecv && !rbuf)
207 throw std::bad_alloc();
208
209 for (int j = 0; j < sendCount[0]; j++)
210 rbuf[j] = sendBuf[j];
211 }
212
213 if (totalrecv)
214 recvBuf = ArrayRCP<T>(rbuf, 0, totalrecv, true);
215 else
216 recvBuf = Teuchos::null;
217}
218
219/* \brief Specialization for std::string.
220
221 For string of char. Number of chars in a string limited to SCHAR_MAX.
222 Send as chars: 1 char for length of string, then chars in string,
223 1 char for length of next string, and so on.
224 \todo error checking
225 */
226template <>
227void AlltoAllv(const Comm<int> &comm,
228 const Environment &env,
229 const ArrayView<const std::string> &sendBuf,
230 const ArrayView<const int> &sendCount,
231 ArrayRCP<std::string> &recvBuf,
232 const ArrayView<int> &recvCount);
233
234/* \brief Specialization for unsigned long long
235 */
236#ifdef HAVE_TPETRA_INT_LONG_LONG
237template <>
238void AlltoAllv(const Comm<int> &comm,
239 const Environment &env,
240 const ArrayView<const unsigned long long> &sendBuf,
241 const ArrayView<const int> &sendCount,
242 ArrayRCP<unsigned long long> &recvBuf,
243 const ArrayView<int> &recvCount);
244#endif
245
246/* \brief Specialization for unsigned short
247 */
248template <>
249void AlltoAllv(const Comm<int> &comm,
250 const Environment &env,
251 const ArrayView<const unsigned short> &sendBuf,
252 const ArrayView<const int> &sendCount,
253 ArrayRCP<unsigned short> &recvBuf,
254 const ArrayView<int> &recvCount);
255
256/* \brief For data type unsigned char (no Teuchos::DirectSerializationTraits)
257 */
258template <>
259void AlltoAllv(const Comm<int> &comm,
260 const Environment &env,
261 const ArrayView<const unsigned char> &sendBuf,
262 const ArrayView<const int> &sendCount,
263 ArrayRCP<unsigned char> &recvBuf,
264 const ArrayView<int> &recvCount);
265
266} // namespace Z2
267#endif
Defines the Environment class.
#define Z2_FORWARD_EXCEPTIONS
Forward an exception back through call stack.
#define Z2_THROW_OUTSIDE_ERROR(env)
Throw an error returned from outside the Zoltan2 library.
Gathering definitions used in software development.
The user parameters, debug, timing and memory profiling output objects, and error checking methods.
Created by mbenlioglu on Aug 31, 2020.
void AlltoAllCount(const Comm< int > &comm, const Environment &env, const ArrayView< const int > &sendCount, const ArrayView< int > &recvCount)
Each process sends a value to every process, an all-to-all.
void AlltoAllv(const Comm< int > &comm, const Environment &env, const ArrayView< const std::string > &sendBuf, const ArrayView< const int > &sendCount, ArrayRCP< std::string > &recvBuf, const ArrayView< int > &recvCount)