/[escript]/trunk/dudley/src/NodeFile_gather.c
ViewVC logotype

Contents of /trunk/dudley/src/NodeFile_gather.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3259 - (show annotations)
Mon Oct 11 01:48:14 2010 UTC (9 years ago) by jfenwick
File MIME type: text/plain
File size: 7497 byte(s)
Merging dudley and scons updates from branches

1
2 /*******************************************************
3 *
4 * Copyright (c) 2003-2010 by University of Queensland
5 * Earth Systems Science Computational Center (ESSCC)
6 * http://www.uq.edu.au/esscc
7 *
8 * Primary Business: Queensland, Australia
9 * Licensed under the Open Software License version 3.0
10 * http://www.opensource.org/licenses/osl-3.0.php
11 *
12 *******************************************************/
13
14 /**************************************************************
15 *
16 * Dudley: Mesh: NodeFile
17 * gathers the NodeFile out from the NodeFile in using the entries
18 * in index[0:out->numNodes-1] which are between min_index and max_index (exclusive)
19 * the node index[i]
20 *
21 **************************************************************/
22
23 #include "NodeFile.h"
24
25 /**************************************************************/
26
27 void Dudley_NodeFile_gatherEntries(dim_t n, index_t * index, index_t min_index, index_t max_index,
28 index_t * Id_out, index_t * Id_in,
29 index_t * Tag_out, index_t * Tag_in,
30 index_t * globalDegreesOfFreedom_out, index_t * globalDegreesOfFreedom_in,
31 dim_t numDim, double *Coordinates_out, double *Coordinates_in)
32 {
33 dim_t i;
34 register index_t k;
35 const index_t range = max_index - min_index;
36 const size_t numDim_size = (size_t) numDim * sizeof(double);
37 #pragma omp parallel for private(i,k) schedule(static)
38 for (i = 0; i < n; i++)
39 {
40 k = index[i] - min_index;
41 if ((k >= 0) && (k < range))
42 {
43 Id_out[i] = Id_in[k];
44 Tag_out[i] = Tag_in[k];
45 globalDegreesOfFreedom_out[i] = globalDegreesOfFreedom_in[k];
46 memcpy(&(Coordinates_out[INDEX2(0, i, numDim)]), &(Coordinates_in[INDEX2(0, k, numDim)]), numDim_size);
47 }
48 }
49 }
50
51 void Dudley_NodeFile_gather(index_t * index, Dudley_NodeFile * in, Dudley_NodeFile * out)
52 {
53 index_t min_id, max_id;
54 Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);
55 Dudley_NodeFile_gatherEntries(out->numNodes, index, min_id, max_id,
56 out->Id, in->Id,
57 out->Tag, in->Tag,
58 out->globalDegreesOfFreedom, in->globalDegreesOfFreedom,
59 out->numDim, out->Coordinates, in->Coordinates);
60 }
61
62 void Dudley_NodeFile_gather_global(index_t * index, Dudley_NodeFile * in, Dudley_NodeFile * out)
63 {
64 index_t min_id, max_id, undefined_node;
65 Esys_MPI_rank buffer_rank, dest, source, *distribution = NULL;
66 index_t *Id_buffer = NULL, *Tag_buffer = NULL, *globalDegreesOfFreedom_buffer = NULL;
67 double *Coordinates_buffer = NULL;
68 dim_t p, buffer_len, n;
69 char error_msg[100];
70 #ifdef ESYS_MPI
71 MPI_Status status;
72 #endif
73
74 /* get the global range of node ids */
75 Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);
76 undefined_node = min_id - 1;
77
78 distribution = TMPMEMALLOC(in->MPIInfo->size + 1, index_t);
79
80 if (!Dudley_checkPtr(distribution))
81 {
82 /* distribute the range of node ids */
83 buffer_len = Esys_MPIInfo_setDistribution(in->MPIInfo, min_id, max_id, distribution);
84 /* allocate buffers */
85 Id_buffer = TMPMEMALLOC(buffer_len, index_t);
86 Tag_buffer = TMPMEMALLOC(buffer_len, index_t);
87 globalDegreesOfFreedom_buffer = TMPMEMALLOC(buffer_len, index_t);
88 Coordinates_buffer = TMPMEMALLOC(buffer_len * out->numDim, double);
89 if (!(Dudley_checkPtr(Id_buffer) || Dudley_checkPtr(Tag_buffer) ||
90 Dudley_checkPtr(globalDegreesOfFreedom_buffer) || Dudley_checkPtr(Coordinates_buffer)))
91 {
92 /* fill Id_buffer by the undefined_node marker to check if nodes are defined */
93 #pragma omp parallel for private(n) schedule(static)
94 for (n = 0; n < buffer_len; n++)
95 Id_buffer[n] = undefined_node;
96
97 /* fill the buffer by sending portions around in a circle */
98 dest = Esys_MPIInfo_mod(in->MPIInfo->size, in->MPIInfo->rank + 1);
99 source = Esys_MPIInfo_mod(in->MPIInfo->size, in->MPIInfo->rank - 1);
100 buffer_rank = in->MPIInfo->rank;
101 for (p = 0; p < in->MPIInfo->size; ++p)
102 {
103 if (p > 0)
104 { /* the initial send can be skipped */
105 #ifdef ESYS_MPI
106 MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,
107 dest, in->MPIInfo->msg_tag_counter, source, in->MPIInfo->msg_tag_counter,
108 in->MPIInfo->comm, &status);
109 MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,
110 dest, in->MPIInfo->msg_tag_counter + 1, source,
111 in->MPIInfo->msg_tag_counter + 1, in->MPIInfo->comm, &status);
112 MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,
113 in->MPIInfo->msg_tag_counter + 2, source, in->MPIInfo->msg_tag_counter + 2,
114 in->MPIInfo->comm, &status);
115 MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,
116 in->MPIInfo->msg_tag_counter + 3, source, in->MPIInfo->msg_tag_counter + 3,
117 in->MPIInfo->comm, &status);
118 #endif
119 in->MPIInfo->msg_tag_counter += 4;
120 }
121 buffer_rank = Esys_MPIInfo_mod(in->MPIInfo->size, buffer_rank - 1);
122 Dudley_NodeFile_scatterEntries(in->numNodes, in->Id,
123 distribution[buffer_rank], distribution[buffer_rank + 1],
124 Id_buffer, in->Id,
125 Tag_buffer, in->Tag,
126 globalDegreesOfFreedom_buffer, in->globalDegreesOfFreedom,
127 out->numDim, Coordinates_buffer, in->Coordinates);
128 }
129 /* now entries are collected from the buffer again by sending the entries around in a circle */
130 dest = Esys_MPIInfo_mod(in->MPIInfo->size, in->MPIInfo->rank + 1);
131 source = Esys_MPIInfo_mod(in->MPIInfo->size, in->MPIInfo->rank - 1);
132 buffer_rank = in->MPIInfo->rank;
133 for (p = 0; p < in->MPIInfo->size; ++p)
134 {
135 Dudley_NodeFile_gatherEntries(out->numNodes, index,
136 distribution[buffer_rank], distribution[buffer_rank + 1],
137 out->Id, Id_buffer,
138 out->Tag, Tag_buffer,
139 out->globalDegreesOfFreedom, globalDegreesOfFreedom_buffer,
140 out->numDim, out->Coordinates, Coordinates_buffer);
141 if (p < in->MPIInfo->size - 1)
142 { /* the last send can be skipped */
143 #ifdef ESYS_MPI
144 MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,
145 dest, in->MPIInfo->msg_tag_counter, source, in->MPIInfo->msg_tag_counter,
146 in->MPIInfo->comm, &status);
147 MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,
148 dest, in->MPIInfo->msg_tag_counter + 1, source,
149 in->MPIInfo->msg_tag_counter + 1, in->MPIInfo->comm, &status);
150 MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,
151 in->MPIInfo->msg_tag_counter + 2, source, in->MPIInfo->msg_tag_counter + 2,
152 in->MPIInfo->comm, &status);
153 MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,
154 in->MPIInfo->msg_tag_counter + 3, source, in->MPIInfo->msg_tag_counter + 3,
155 in->MPIInfo->comm, &status);
156 #endif
157 in->MPIInfo->msg_tag_counter += 4;
158 }
159 buffer_rank = Esys_MPIInfo_mod(in->MPIInfo->size, buffer_rank - 1);
160 }
161 /* check if all nodes are set: */
162 #pragma omp parallel for private(n) schedule(static)
163 for (n = 0; n < out->numNodes; ++n)
164 {
165 if (out->Id[n] == undefined_node)
166 {
167 sprintf(error_msg,
168 "Dudley_NodeFile_gather_global: Node id %d at position %d is referenced but is not defined.",
169 out->Id[n], n);
170 Dudley_setError(VALUE_ERROR, error_msg);
171 }
172 }
173
174 }
175 TMPMEMFREE(Id_buffer);
176 TMPMEMFREE(Tag_buffer);
177 TMPMEMFREE(globalDegreesOfFreedom_buffer);
178 TMPMEMFREE(Coordinates_buffer);
179 }
180 TMPMEMFREE(distribution);
181 /* make sure that the error is global */
182 Esys_MPIInfo_noError(in->MPIInfo);
183 }

Properties

Name Value
svn:eol-style native
svn:keywords Author Date Id Revision

  ViewVC Help
Powered by ViewVC 1.1.26