/[escript]/trunk/paso/src/Coupler.cpp
ViewVC logotype

Contents of /trunk/paso/src/Coupler.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 5148 - (show annotations)
Mon Sep 15 01:25:23 2014 UTC (5 years ago) by caltinay
File size: 5414 byte(s)
Merging ripley diagonal storage + CUDA support into trunk.
Options file version has been incremented due to new options
'cuda' and 'nvccflags'.

1
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2014 by University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Open Software License version 3.0
9 * http://www.opensource.org/licenses/osl-3.0.php
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16
17
18 #include "Coupler.h"
19
20 #include <cstring> // memcpy
21
22 namespace paso {
23
24 /****************************************************************************
25 *
26 * allocates a Coupler
27 *
28 ****************************************************************************/
29
30 Coupler::Coupler(const_Connector_ptr conn, dim_t blockSize) :
31 connector(conn),
32 block_size(blockSize),
33 in_use(false),
34 data(NULL),
35 send_buffer(NULL),
36 recv_buffer(NULL),
37 mpi_requests(NULL),
38 mpi_stati(NULL)
39 {
40 Esys_resetError();
41 mpi_info = conn->mpi_info;
42 #ifdef ESYS_MPI
43 mpi_requests = new MPI_Request[conn->send->numNeighbors +
44 conn->recv->numNeighbors];
45 mpi_stati = new MPI_Status[conn->send->numNeighbors +
46 conn->recv->numNeighbors];
47 #endif
48 if (mpi_info->size > 1) {
49 send_buffer=new double[conn->send->numSharedComponents * block_size];
50 recv_buffer=new double[conn->recv->numSharedComponents * block_size];
51 }
52 }
53
54 Coupler::~Coupler()
55 {
56 delete[] send_buffer;
57 delete[] recv_buffer;
58 #ifdef ESYS_MPI
59 delete[] mpi_requests;
60 delete[] mpi_stati;
61 #endif
62 }
63
64 void Coupler::startCollect(const double* in)
65 {
66 data = const_cast<double*>(in);
67 if (mpi_info->size > 1) {
68 if (in_use) {
69 Esys_setError(SYSTEM_ERROR,"Coupler::startCollect: Coupler in use.");
70 }
71 // start receiving input
72 for (dim_t i=0; i < connector->recv->numNeighbors; ++i) {
73 #ifdef ESYS_MPI
74 MPI_Irecv(&recv_buffer[connector->recv->offsetInShared[i]*block_size],
75 (connector->recv->offsetInShared[i+1]-connector->recv->offsetInShared[i])*block_size,
76 MPI_DOUBLE, connector->recv->neighbor[i],
77 mpi_info->msg_tag_counter+connector->recv->neighbor[i],
78 mpi_info->comm, &mpi_requests[i]);
79 #endif
80 }
81 // collect values into buffer
82 const int numSharedSend = connector->send->numSharedComponents;
83 if (block_size > 1) {
84 const size_t block_size_size=block_size*sizeof(double);
85 #pragma omp parallel for
86 for (dim_t i=0; i < numSharedSend; ++i) {
87 memcpy(&(send_buffer[(block_size)*i]),
88 &(in[block_size*connector->send->shared[i]]),
89 block_size_size);
90 }
91 } else {
92 #pragma omp parallel for
93 for (dim_t i=0; i < numSharedSend; ++i) {
94 send_buffer[i]=in[connector->send->shared[i]];
95 }
96 }
97 // send buffer out
98 for (dim_t i=0; i < connector->send->numNeighbors; ++i) {
99 #ifdef ESYS_MPI
100 MPI_Issend(&send_buffer[connector->send->offsetInShared[i]*block_size],
101 (connector->send->offsetInShared[i+1] - connector->send->offsetInShared[i])*block_size,
102 MPI_DOUBLE, connector->send->neighbor[i],
103 mpi_info->msg_tag_counter+mpi_info->rank, mpi_info->comm,
104 &mpi_requests[i+connector->recv->numNeighbors]);
105 #endif
106 }
107 ESYS_MPI_INC_COUNTER(*mpi_info, mpi_info->size)
108 in_use = true;
109 }
110 }
111
112 double* Coupler::finishCollect()
113 {
114 if (mpi_info->size > 1) {
115 if (!in_use) {
116 Esys_setError(SYSTEM_ERROR, "Coupler::finishCollect: Communication has not been initiated.");
117 return NULL;
118 }
119 // wait for receive
120 #ifdef ESYS_MPI
121 MPI_Waitall(connector->recv->numNeighbors+connector->send->numNeighbors,
122 mpi_requests, mpi_stati);
123 #endif
124 in_use = false;
125 }
126 return recv_buffer;
127 }
128
129 void Coupler::copyAll(Coupler_ptr target) const
130 {
131 const dim_t overlap = getNumOverlapValues();
132 const dim_t localSize = getLocalLength()*block_size;
133 #pragma omp parallel
134 {
135 #pragma omp for
136 for (dim_t i=0; i < overlap; ++i) {
137 target->recv_buffer[i] = recv_buffer[i];
138 }
139 #pragma omp for
140 for (dim_t i=0; i < localSize; ++i) {
141 target->data[i] = data[i];
142 }
143 }
144 }
145
146 void Coupler::fillOverlap(dim_t n, double* x)
147 {
148 const dim_t overlap_n = getNumOverlapValues();
149 const dim_t my_n= n - overlap_n;
150 const dim_t offset = block_size * my_n;
151
152 startCollect(x);
153 double* remote_values = finishCollect();
154
155 #pragma omp parallel for
156 for (dim_t i=0; i < overlap_n * block_size; ++i) {
157 x[offset+i] = remote_values[i];
158 }
159 }
160
161 /* adjusts max values across shared values x */
162 void Coupler::max(dim_t n, double* x)
163 {
164 const dim_t overlap_n = getNumOverlapValues();
165 const dim_t my_n = n - overlap_n;
166
167 startCollect(x);
168 double* remote_values = finishCollect();
169
170 #pragma omp parallel for
171 for (dim_t i=0; i < overlap_n; ++i)
172 x[my_n+i] = std::max(x[my_n+i], remote_values[i]);
173 }
174
175 } // namespace paso
176

Properties

Name Value
svn:mergeinfo /branches/amg_from_3530/paso/src/Coupler.cpp:3531-3826 /branches/diaplayground/paso/src/Coupler.cpp:4940-5147 /branches/lapack2681/paso/src/Coupler.cpp:2682-2741 /branches/pasowrap/paso/src/Coupler.cpp:3661-3674 /branches/py3_attempt2/paso/src/Coupler.cpp:3871-3891 /branches/restext/paso/src/Coupler.cpp:2610-2624 /branches/ripleygmg_from_3668/paso/src/Coupler.cpp:3669-3791 /branches/stage3.0/paso/src/Coupler.cpp:2569-2590 /branches/symbolic_from_3470/paso/src/Coupler.cpp:3471-3974 /branches/symbolic_from_3470/ripley/test/python/paso/src/Coupler.cpp:3517-3974 /release/3.0/paso/src/Coupler.cpp:2591-2601 /trunk/paso/src/Coupler.cpp:4257-4344 /trunk/ripley/test/python/paso/src/Coupler.cpp:3480-3515

  ViewVC Help
Powered by ViewVC 1.1.26