/[escript]/trunk/dudley/src/NodeFile_gather.c
ViewVC logotype

Contents of /trunk/dudley/src/NodeFile_gather.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3603 - (show annotations)
Mon Sep 19 03:42:53 2011 UTC (7 years, 10 months ago) by caltinay
File MIME type: text/plain
File size: 7561 byte(s)
Fixed some unused var errors in non-mpi build.

1
2 /*******************************************************
3 *
4 * Copyright (c) 2003-2010 by University of Queensland
5 * Earth Systems Science Computational Center (ESSCC)
6 * http://www.uq.edu.au/esscc
7 *
8 * Primary Business: Queensland, Australia
9 * Licensed under the Open Software License version 3.0
10 * http://www.opensource.org/licenses/osl-3.0.php
11 *
12 *******************************************************/
13
14 /**************************************************************
15 *
16 * Dudley: Mesh: NodeFile
17 * gathers the NodeFile out from the NodeFile in using the entries
18 * in index[0:out->numNodes-1] which are between min_index and max_index (exclusive)
19 * the node index[i]
20 *
21 **************************************************************/
22
23 #include "NodeFile.h"
24
25 /**************************************************************/
26
27 void Dudley_NodeFile_gatherEntries(dim_t n, index_t * index, index_t min_index, index_t max_index,
28 index_t * Id_out, index_t * Id_in,
29 index_t * Tag_out, index_t * Tag_in,
30 index_t * globalDegreesOfFreedom_out, index_t * globalDegreesOfFreedom_in,
31 dim_t numDim, double *Coordinates_out, double *Coordinates_in)
32 {
33 dim_t i;
34 register index_t k;
35 const index_t range = max_index - min_index;
36 const size_t numDim_size = (size_t) numDim * sizeof(double);
37 #pragma omp parallel for private(i,k) schedule(static)
38 for (i = 0; i < n; i++)
39 {
40 k = index[i] - min_index;
41 if ((k >= 0) && (k < range))
42 {
43 Id_out[i] = Id_in[k];
44 Tag_out[i] = Tag_in[k];
45 globalDegreesOfFreedom_out[i] = globalDegreesOfFreedom_in[k];
46 memcpy(&(Coordinates_out[INDEX2(0, i, numDim)]), &(Coordinates_in[INDEX2(0, k, numDim)]), numDim_size);
47 }
48 }
49 }
50
51 void Dudley_NodeFile_gather(index_t * index, Dudley_NodeFile * in, Dudley_NodeFile * out)
52 {
53 index_t min_id, max_id;
54 Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);
55 Dudley_NodeFile_gatherEntries(out->numNodes, index, min_id, max_id,
56 out->Id, in->Id,
57 out->Tag, in->Tag,
58 out->globalDegreesOfFreedom, in->globalDegreesOfFreedom,
59 out->numDim, out->Coordinates, in->Coordinates);
60 }
61
62 void Dudley_NodeFile_gather_global(index_t * index, Dudley_NodeFile * in, Dudley_NodeFile * out)
63 {
64 index_t min_id, max_id, undefined_node;
65 Esys_MPI_rank buffer_rank, *distribution = NULL;
66 index_t *Id_buffer = NULL, *Tag_buffer = NULL, *globalDegreesOfFreedom_buffer = NULL;
67 double *Coordinates_buffer = NULL;
68 dim_t p, buffer_len, n;
69 char error_msg[100];
70 #ifdef ESYS_MPI
71 Esys_MPI_rank dest, source;
72 MPI_Status status;
73 #endif
74
75 /* get the global range of node ids */
76 Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);
77 undefined_node = min_id - 1;
78
79 distribution = TMPMEMALLOC(in->MPIInfo->size + 1, index_t);
80
81 if (!Dudley_checkPtr(distribution))
82 {
83 /* distribute the range of node ids */
84 buffer_len = Esys_MPIInfo_setDistribution(in->MPIInfo, min_id, max_id, distribution);
85 /* allocate buffers */
86 Id_buffer = TMPMEMALLOC(buffer_len, index_t);
87 Tag_buffer = TMPMEMALLOC(buffer_len, index_t);
88 globalDegreesOfFreedom_buffer = TMPMEMALLOC(buffer_len, index_t);
89 Coordinates_buffer = TMPMEMALLOC(buffer_len * out->numDim, double);
90 if (!(Dudley_checkPtr(Id_buffer) || Dudley_checkPtr(Tag_buffer) ||
91 Dudley_checkPtr(globalDegreesOfFreedom_buffer) || Dudley_checkPtr(Coordinates_buffer)))
92 {
93 /* fill Id_buffer by the undefined_node marker to check if nodes are defined */
94 #pragma omp parallel for private(n) schedule(static)
95 for (n = 0; n < buffer_len; n++)
96 Id_buffer[n] = undefined_node;
97
98 /* fill the buffer by sending portions around in a circle */
99 #ifdef ESYS_MPI
100 dest = Esys_MPIInfo_mod(in->MPIInfo->size, in->MPIInfo->rank + 1);
101 source = Esys_MPIInfo_mod(in->MPIInfo->size, in->MPIInfo->rank - 1);
102 #endif
103 buffer_rank = in->MPIInfo->rank;
104 for (p = 0; p < in->MPIInfo->size; ++p)
105 {
106 if (p > 0)
107 { /* the initial send can be skipped */
108 #ifdef ESYS_MPI
109 MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,
110 dest, in->MPIInfo->msg_tag_counter, source, in->MPIInfo->msg_tag_counter,
111 in->MPIInfo->comm, &status);
112 MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,
113 dest, in->MPIInfo->msg_tag_counter + 1, source,
114 in->MPIInfo->msg_tag_counter + 1, in->MPIInfo->comm, &status);
115 MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,
116 in->MPIInfo->msg_tag_counter + 2, source, in->MPIInfo->msg_tag_counter + 2,
117 in->MPIInfo->comm, &status);
118 MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,
119 in->MPIInfo->msg_tag_counter + 3, source, in->MPIInfo->msg_tag_counter + 3,
120 in->MPIInfo->comm, &status);
121 #endif
122 in->MPIInfo->msg_tag_counter += 4;
123 }
124 buffer_rank = Esys_MPIInfo_mod(in->MPIInfo->size, buffer_rank - 1);
125 Dudley_NodeFile_scatterEntries(in->numNodes, in->Id,
126 distribution[buffer_rank], distribution[buffer_rank + 1],
127 Id_buffer, in->Id,
128 Tag_buffer, in->Tag,
129 globalDegreesOfFreedom_buffer, in->globalDegreesOfFreedom,
130 out->numDim, Coordinates_buffer, in->Coordinates);
131 }
132 /* now entries are collected from the buffer again by sending the entries around in a circle */
133 #ifdef ESYS_MPI
134 dest = Esys_MPIInfo_mod(in->MPIInfo->size, in->MPIInfo->rank + 1);
135 source = Esys_MPIInfo_mod(in->MPIInfo->size, in->MPIInfo->rank - 1);
136 #endif
137 buffer_rank = in->MPIInfo->rank;
138 for (p = 0; p < in->MPIInfo->size; ++p)
139 {
140 Dudley_NodeFile_gatherEntries(out->numNodes, index,
141 distribution[buffer_rank], distribution[buffer_rank + 1],
142 out->Id, Id_buffer,
143 out->Tag, Tag_buffer,
144 out->globalDegreesOfFreedom, globalDegreesOfFreedom_buffer,
145 out->numDim, out->Coordinates, Coordinates_buffer);
146 if (p < in->MPIInfo->size - 1)
147 { /* the last send can be skipped */
148 #ifdef ESYS_MPI
149 MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,
150 dest, in->MPIInfo->msg_tag_counter, source, in->MPIInfo->msg_tag_counter,
151 in->MPIInfo->comm, &status);
152 MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,
153 dest, in->MPIInfo->msg_tag_counter + 1, source,
154 in->MPIInfo->msg_tag_counter + 1, in->MPIInfo->comm, &status);
155 MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,
156 in->MPIInfo->msg_tag_counter + 2, source, in->MPIInfo->msg_tag_counter + 2,
157 in->MPIInfo->comm, &status);
158 MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,
159 in->MPIInfo->msg_tag_counter + 3, source, in->MPIInfo->msg_tag_counter + 3,
160 in->MPIInfo->comm, &status);
161 #endif
162 in->MPIInfo->msg_tag_counter += 4;
163 }
164 buffer_rank = Esys_MPIInfo_mod(in->MPIInfo->size, buffer_rank - 1);
165 }
166 /* check if all nodes are set: */
167 #pragma omp parallel for private(n) schedule(static)
168 for (n = 0; n < out->numNodes; ++n)
169 {
170 if (out->Id[n] == undefined_node)
171 {
172 sprintf(error_msg,
173 "Dudley_NodeFile_gather_global: Node id %d at position %d is referenced but is not defined.",
174 out->Id[n], n);
175 Dudley_setError(VALUE_ERROR, error_msg);
176 }
177 }
178
179 }
180 TMPMEMFREE(Id_buffer);
181 TMPMEMFREE(Tag_buffer);
182 TMPMEMFREE(globalDegreesOfFreedom_buffer);
183 TMPMEMFREE(Coordinates_buffer);
184 }
185 TMPMEMFREE(distribution);
186 /* make sure that the error is global */
187 Esys_MPIInfo_noError(in->MPIInfo);
188 }

Properties

Name Value
svn:eol-style native
svn:keywords Author Date Id Revision

  ViewVC Help
Powered by ViewVC 1.1.26