/[escript]/trunk/ripley/src/Rectangle.cpp
ViewVC logotype

Contents of /trunk/ripley/src/Rectangle.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3701 - (show annotations)
Fri Dec 2 02:04:58 2011 UTC (7 years, 10 months ago) by caltinay
Original Path: branches/ripleygmg_from_3668/ripley/src/Rectangle.cpp
File size: 28971 byte(s)
Prepared interpolation methods.

1
2 /*******************************************************
3 *
4 * Copyright (c) 2003-2011 by University of Queensland
5 * Earth Systems Science Computational Center (ESSCC)
6 * http://www.uq.edu.au/esscc
7 *
8 * Primary Business: Queensland, Australia
9 * Licensed under the Open Software License version 3.0
10 * http://www.opensource.org/licenses/osl-3.0.php
11 *
12 *******************************************************/
13
14 #include <ripley/Rectangle.h>
15 extern "C" {
16 #include "paso/SystemMatrixPattern.h"
17 }
18
19 #if USE_SILO
20 #include <silo.h>
21 #ifdef ESYS_MPI
22 #include <pmpio.h>
23 #endif
24 #endif
25
26 #include <iomanip>
27
28 using namespace std;
29
30 namespace ripley {
31
32 Rectangle::Rectangle(int n0, int n1, double l0, double l1, int d0, int d1) :
33 RipleyDomain(2),
34 m_gNE0(n0),
35 m_gNE1(n1),
36 m_l0(l0),
37 m_l1(l1),
38 m_NX(d0),
39 m_NY(d1)
40 {
41 // ensure number of subdivisions is valid and nodes can be distributed
42 // among number of ranks
43 if (m_NX*m_NY != m_mpiInfo->size)
44 throw RipleyException("Invalid number of spatial subdivisions");
45
46 if (n0%m_NX > 0 || n1%m_NY > 0)
47 throw RipleyException("Number of elements must be separable into number of ranks in each dimension");
48
49 // local number of elements
50 m_NE0 = n0/m_NX;
51 m_NE1 = n1/m_NY;
52 // local number of nodes (not necessarily owned)
53 m_N0 = m_NE0+1;
54 m_N1 = m_NE1+1;
55 // bottom-left node is at (offset0,offset1) in global mesh
56 m_offset0 = m_NE0*(m_mpiInfo->rank%m_NX);
57 m_offset1 = m_NE1*(m_mpiInfo->rank/m_NX);
58 populateSampleIds();
59 }
60
61 Rectangle::~Rectangle()
62 {
63 }
64
65 string Rectangle::getDescription() const
66 {
67 return "ripley::Rectangle";
68 }
69
70 bool Rectangle::operator==(const AbstractDomain& other) const
71 {
72 if (dynamic_cast<const Rectangle*>(&other))
73 return this==&other;
74
75 return false;
76 }
77
78 void Rectangle::dump(const string& fileName) const
79 {
80 #if USE_SILO
81 string fn(fileName);
82 if (fileName.length() < 6 || fileName.compare(fileName.length()-5, 5, ".silo") != 0) {
83 fn+=".silo";
84 }
85
86 const int NUM_SILO_FILES = 1;
87 const char* blockDirFmt = "/block%04d";
88 int driver=DB_HDF5;
89 string siloPath;
90 DBfile* dbfile = NULL;
91
92 #ifdef ESYS_MPI
93 PMPIO_baton_t* baton = NULL;
94 #endif
95
96 if (m_mpiInfo->size > 1) {
97 #ifdef ESYS_MPI
98 baton = PMPIO_Init(NUM_SILO_FILES, PMPIO_WRITE, m_mpiInfo->comm,
99 0x1337, PMPIO_DefaultCreate, PMPIO_DefaultOpen,
100 PMPIO_DefaultClose, (void*)&driver);
101 // try the fallback driver in case of error
102 if (!baton && driver != DB_PDB) {
103 driver = DB_PDB;
104 baton = PMPIO_Init(NUM_SILO_FILES, PMPIO_WRITE, m_mpiInfo->comm,
105 0x1338, PMPIO_DefaultCreate, PMPIO_DefaultOpen,
106 PMPIO_DefaultClose, (void*)&driver);
107 }
108 if (baton) {
109 char str[64];
110 snprintf(str, 64, blockDirFmt, PMPIO_RankInGroup(baton, m_mpiInfo->rank));
111 siloPath = str;
112 dbfile = (DBfile*) PMPIO_WaitForBaton(baton, fn.c_str(), siloPath.c_str());
113 }
114 #endif
115 } else {
116 dbfile = DBCreate(fn.c_str(), DB_CLOBBER, DB_LOCAL,
117 getDescription().c_str(), driver);
118 // try the fallback driver in case of error
119 if (!dbfile && driver != DB_PDB) {
120 driver = DB_PDB;
121 dbfile = DBCreate(fn.c_str(), DB_CLOBBER, DB_LOCAL,
122 getDescription().c_str(), driver);
123 }
124 }
125
126 if (!dbfile)
127 throw RipleyException("dump: Could not create Silo file");
128
129 /*
130 if (driver==DB_HDF5) {
131 // gzip level 1 already provides good compression with minimal
132 // performance penalty. Some tests showed that gzip levels >3 performed
133 // rather badly on escript data both in terms of time and space
134 DBSetCompression("ERRMODE=FALLBACK METHOD=GZIP LEVEL=1");
135 }
136 */
137
138 boost::scoped_ptr<double> x(new double[m_N0]);
139 boost::scoped_ptr<double> y(new double[m_N1]);
140 double* coords[2] = { x.get(), y.get() };
141 pair<double,double> xdx = getFirstCoordAndSpacing(0);
142 pair<double,double> ydy = getFirstCoordAndSpacing(1);
143 #pragma omp parallel
144 {
145 #pragma omp for
146 for (dim_t i0 = 0; i0 < m_N0; i0++) {
147 coords[0][i0]=xdx.first+i0*xdx.second;
148 }
149 #pragma omp for
150 for (dim_t i1 = 0; i1 < m_N1; i1++) {
151 coords[1][i1]=ydy.first+i1*ydy.second;
152 }
153 }
154 IndexVector dims = getNumNodesPerDim();
155
156 // write mesh
157 DBPutQuadmesh(dbfile, "mesh", NULL, coords, &dims[0], 2, DB_DOUBLE,
158 DB_COLLINEAR, NULL);
159
160 // write node ids
161 DBPutQuadvar1(dbfile, "nodeId", "mesh", (void*)&m_nodeId[0], &dims[0], 2,
162 NULL, 0, DB_INT, DB_NODECENT, NULL);
163
164 // write element ids
165 dims = getNumElementsPerDim();
166 DBPutQuadvar1(dbfile, "elementId", "mesh", (void*)&m_elementId[0],
167 &dims[0], 2, NULL, 0, DB_INT, DB_ZONECENT, NULL);
168
169 // rank 0 writes multimesh and multivar
170 if (m_mpiInfo->rank == 0) {
171 vector<string> tempstrings;
172 vector<char*> names;
173 for (dim_t i=0; i<m_mpiInfo->size; i++) {
174 stringstream path;
175 path << "/block" << setw(4) << setfill('0') << right << i << "/mesh";
176 tempstrings.push_back(path.str());
177 names.push_back((char*)tempstrings.back().c_str());
178 }
179 vector<int> types(m_mpiInfo->size, DB_QUAD_RECT);
180 DBSetDir(dbfile, "/");
181 DBPutMultimesh(dbfile, "multimesh", m_mpiInfo->size, &names[0],
182 &types[0], NULL);
183 tempstrings.clear();
184 names.clear();
185 for (dim_t i=0; i<m_mpiInfo->size; i++) {
186 stringstream path;
187 path << "/block" << setw(4) << setfill('0') << right << i << "/nodeId";
188 tempstrings.push_back(path.str());
189 names.push_back((char*)tempstrings.back().c_str());
190 }
191 types.assign(m_mpiInfo->size, DB_QUADVAR);
192 DBPutMultivar(dbfile, "nodeId", m_mpiInfo->size, &names[0],
193 &types[0], NULL);
194 tempstrings.clear();
195 names.clear();
196 for (dim_t i=0; i<m_mpiInfo->size; i++) {
197 stringstream path;
198 path << "/block" << setw(4) << setfill('0') << right << i << "/elementId";
199 tempstrings.push_back(path.str());
200 names.push_back((char*)tempstrings.back().c_str());
201 }
202 DBPutMultivar(dbfile, "elementId", m_mpiInfo->size, &names[0],
203 &types[0], NULL);
204 }
205
206 if (m_mpiInfo->size > 1) {
207 #ifdef ESYS_MPI
208 PMPIO_HandOffBaton(baton, dbfile);
209 PMPIO_Finish(baton);
210 #endif
211 } else {
212 DBClose(dbfile);
213 }
214
215 #else // USE_SILO
216 throw RipleyException("dump(): no Silo support");
217 #endif
218 }
219
220 const int* Rectangle::borrowSampleReferenceIDs(int fsType) const
221 {
222 switch (fsType) {
223 case Nodes:
224 return &m_nodeId[0];
225 case Elements:
226 return &m_elementId[0];
227 case FaceElements:
228 return &m_faceId[0];
229 default:
230 break;
231 }
232
233 stringstream msg;
234 msg << "borrowSampleReferenceIDs() not implemented for function space type "
235 << functionSpaceTypeAsString(fsType);
236 throw RipleyException(msg.str());
237 }
238
239 bool Rectangle::ownSample(int fsCode, index_t id) const
240 {
241 #ifdef ESYS_MPI
242 if (fsCode == Nodes) {
243 const index_t myFirst=m_nodeDistribution[m_mpiInfo->rank];
244 const index_t myLast=m_nodeDistribution[m_mpiInfo->rank+1]-1;
245 return (m_nodeId[id]>=myFirst && m_nodeId[id]<=myLast);
246 } else
247 throw RipleyException("ownSample() only implemented for Nodes");
248 #else
249 return true;
250 #endif
251 }
252
253 void Rectangle::interpolateNodesOnElements(escript::Data& out, escript::Data& in) const
254 {
255 const dim_t numComp = in.getDataPointSize();
256 /* GENERATOR SNIP_INTERPOLATE_ELEMENTS TOP */
257 const double tmp0_2 = 0.62200846792814621559;
258 const double tmp0_1 = 0.044658198738520451079;
259 const double tmp0_0 = 0.16666666666666666667;
260 #pragma omp parallel for
261 for (index_t k1=0; k1 < m_NE1; ++k1) {
262 for (index_t k0=0; k0 < m_NE0; ++k0) {
263 const register double* f_10 = in.getSampleDataRO(INDEX2(k0+1,k1, m_N0));
264 const register double* f_11 = in.getSampleDataRO(INDEX2(k0+1,k1+1, m_N0));
265 const register double* f_01 = in.getSampleDataRO(INDEX2(k0,k1+1, m_N0));
266 const register double* f_00 = in.getSampleDataRO(INDEX2(k0,k1, m_N0));
267 double* o = out.getSampleDataRW(INDEX2(k0,k1,m_NE0));
268 for (index_t i=0; i < numComp; ++i) {
269 o[INDEX2(i,numComp,0)] = f_00[i]*tmp0_2 + f_11[i]*tmp0_1 + tmp0_0*(f_01[i] + f_10[i]);
270 o[INDEX2(i,numComp,1)] = f_01[i]*tmp0_1 + f_10[i]*tmp0_2 + tmp0_0*(f_00[i] + f_11[i]);
271 o[INDEX2(i,numComp,2)] = f_01[i]*tmp0_2 + f_10[i]*tmp0_1 + tmp0_0*(f_00[i] + f_11[i]);
272 o[INDEX2(i,numComp,3)] = f_00[i]*tmp0_1 + f_11[i]*tmp0_2 + tmp0_0*(f_01[i] + f_10[i]);
273 } /* end of component loop i */
274 } /* end of k0 loop */
275 } /* end of k1 loop */
276 /* GENERATOR SNIP_INTERPOLATE_ELEMENTS BOTTOM */
277 }
278
279 void Rectangle::interpolateNodesOnFaces(escript::Data& out, escript::Data& in) const
280 {
281 throw RipleyException("interpolateNodesOnFaces() not implemented");
282 }
283
284 void Rectangle::addPDEToSystem(escript::AbstractSystemMatrix& mat,
285 escript::Data& rhs, const escript::Data& A, const escript::Data& B,
286 const escript::Data& C, const escript::Data& D,
287 const escript::Data& X, const escript::Data& Y,
288 const escript::Data& d, const escript::Data& y,
289 const escript::Data& d_contact, const escript::Data& y_contact,
290 const escript::Data& d_dirac, const escript::Data& y_dirac) const
291 {
292 throw RipleyException("addPDEToSystem() not implemented");
293 }
294
295 Paso_SystemMatrixPattern* Rectangle::getPattern(bool reducedRowOrder,
296 bool reducedColOrder) const
297 {
298 if (reducedRowOrder || reducedColOrder)
299 throw RipleyException("getPattern() not implemented for reduced order");
300
301 // connector
302 RankVector neighbour;
303 IndexVector offsetInShared(1,0);
304 IndexVector sendShared, recvShared;
305 const IndexVector faces=getNumFacesPerBoundary();
306 const index_t left = (m_offset0==0 ? 0 : 1);
307 const index_t bottom = (m_offset1==0 ? 0 : 1);
308 // corner node from bottom-left
309 if (faces[0] == 0 && faces[2] == 0) {
310 neighbour.push_back(m_mpiInfo->rank-m_NX-1);
311 offsetInShared.push_back(offsetInShared.back()+1);
312 sendShared.push_back(m_nodeId[m_N0+left]);
313 recvShared.push_back(m_nodeId[0]);
314 }
315 // bottom edge
316 if (faces[2] == 0) {
317 neighbour.push_back(m_mpiInfo->rank-m_NX);
318 offsetInShared.push_back(offsetInShared.back()+m_N0-left);
319 for (dim_t i=left; i<m_N0; i++) {
320 // easy case, we know the neighbour id's
321 sendShared.push_back(m_nodeId[m_N0+i]);
322 recvShared.push_back(m_nodeId[i]);
323 }
324 }
325 // corner node from bottom-right
326 if (faces[1] == 0 && faces[2] == 0) {
327 neighbour.push_back(m_mpiInfo->rank-m_NX+1);
328 const index_t N0=(neighbour.back()%m_NX == 0 ? m_N0 : m_N0-1);
329 const index_t N1=(neighbour.back()/m_NX == 0 ? m_N1 : m_N1-1);
330 const index_t first=m_nodeDistribution[neighbour.back()];
331 offsetInShared.push_back(offsetInShared.back()+1);
332 sendShared.push_back(m_nodeId[(bottom+1)*m_N0-1]);
333 recvShared.push_back(first+N0*(N1-1));
334 }
335 // left edge
336 if (faces[0] == 0) {
337 neighbour.push_back(m_mpiInfo->rank-1);
338 offsetInShared.push_back(offsetInShared.back()+m_N1-bottom);
339 for (dim_t i=bottom; i<m_N1; i++) {
340 // easy case, we know the neighbour id's
341 sendShared.push_back(m_nodeId[i*m_N0+1]);
342 recvShared.push_back(m_nodeId[i*m_N0]);
343 }
344 }
345 // right edge
346 if (faces[1] == 0) {
347 neighbour.push_back(m_mpiInfo->rank+1);
348 const index_t rightN0=(neighbour.back()%m_NX == 0 ? m_N0 : m_N0-1);
349 const index_t first=m_nodeDistribution[neighbour.back()];
350 offsetInShared.push_back(offsetInShared.back()+m_N1-bottom);
351 for (dim_t i=bottom; i<m_N1; i++) {
352 sendShared.push_back(m_nodeId[(i+1)*m_N0-1]);
353 recvShared.push_back(first+rightN0*(i-bottom));
354 }
355 }
356 // corner node from top-left
357 if (faces[0] == 0 && faces[3] == 0) {
358 neighbour.push_back(m_mpiInfo->rank+m_NX-1);
359 const index_t N0=(neighbour.back()%m_NX == 0 ? m_N0 : m_N0-1);
360 const index_t first=m_nodeDistribution[neighbour.back()];
361 offsetInShared.push_back(offsetInShared.back()+1);
362 sendShared.push_back(m_nodeId[m_N0*(m_N1-1)+left]);
363 recvShared.push_back(first+N0-1);
364 }
365 // top edge
366 if (faces[3] == 0) {
367 neighbour.push_back(m_mpiInfo->rank+m_NX);
368 const index_t first=m_nodeDistribution[neighbour.back()];
369 offsetInShared.push_back(offsetInShared.back()+m_N0-left);
370 for (dim_t i=left; i<m_N0; i++) {
371 sendShared.push_back(m_nodeId[m_N0*(m_N1-1)+i]);
372 recvShared.push_back(first+i-left);
373 }
374 }
375 // corner node from top-right
376 if (faces[1] == 0 && faces[3] == 0) {
377 neighbour.push_back(m_mpiInfo->rank+m_NX+1);
378 const index_t first=m_nodeDistribution[neighbour.back()];
379 offsetInShared.push_back(offsetInShared.back()+1);
380 sendShared.push_back(m_nodeId[m_N0*m_N1-1]);
381 recvShared.push_back(first);
382 }
383 const int numDOF=m_nodeDistribution[m_mpiInfo->rank+1]-m_nodeDistribution[m_mpiInfo->rank];
384 cout << "--- rcv_shcomp ---" << endl;
385 cout << "numDOF=" << numDOF << ", numNeighbors=" << neighbour.size() << endl;
386 for (size_t i=0; i<neighbour.size(); i++) {
387 cout << "neighbor[" << i << "]=" << neighbour[i]
388 << " offsetInShared[" << i+1 << "]=" << offsetInShared[i+1] << endl;
389 }
390 for (size_t i=0; i<recvShared.size(); i++) {
391 cout << "shared[" << i << "]=" << recvShared[i] << endl;
392 }
393 cout << "--- snd_shcomp ---" << endl;
394 for (size_t i=0; i<sendShared.size(); i++) {
395 cout << "shared[" << i << "]=" << sendShared[i] << endl;
396 }
397
398 Paso_SharedComponents *snd_shcomp = Paso_SharedComponents_alloc(
399 numDOF, neighbour.size(), &neighbour[0], &sendShared[0],
400 &offsetInShared[0], 1, 0, m_mpiInfo);
401 Paso_SharedComponents *rcv_shcomp = Paso_SharedComponents_alloc(
402 numDOF, neighbour.size(), &neighbour[0], &recvShared[0],
403 &offsetInShared[0], 1, 0, m_mpiInfo);
404 Paso_Connector* connector = Paso_Connector_alloc(snd_shcomp, rcv_shcomp);
405 Paso_SharedComponents_free(snd_shcomp);
406 Paso_SharedComponents_free(rcv_shcomp);
407
408 // create patterns
409 dim_t M, N;
410 IndexVector ptr(1,0);
411 IndexVector index;
412
413 // main pattern
414 for (index_t i=0; i<numDOF; i++) {
415 // always add the node itself
416 index.push_back(i);
417 int num=insertNeighbours(index, i);
418 ptr.push_back(ptr.back()+num+1);
419 }
420 M=N=ptr.size()-1;
421 // paso will manage the memory
422 index_t* indexC = MEMALLOC(index.size(),index_t);
423 index_t* ptrC = MEMALLOC(ptr.size(), index_t);
424 copy(index.begin(), index.end(), indexC);
425 copy(ptr.begin(), ptr.end(), ptrC);
426 Paso_Pattern *mainPattern = Paso_Pattern_alloc(MATRIX_FORMAT_DEFAULT,
427 M, N, ptrC, indexC);
428
429 cout << "--- main_pattern ---" << endl;
430 cout << "M=" << M << ", N=" << N << endl;
431 for (size_t i=0; i<ptr.size(); i++) {
432 cout << "ptr[" << i << "]=" << ptr[i] << endl;
433 }
434 for (size_t i=0; i<index.size(); i++) {
435 cout << "index[" << i << "]=" << index[i] << endl;
436 }
437
438 ptr.clear();
439 index.clear();
440
441 // column & row couple patterns
442 Paso_Pattern *colCouplePattern, *rowCouplePattern;
443 generateCouplePatterns(&colCouplePattern, &rowCouplePattern);
444
445 // allocate paso distribution
446 Paso_Distribution* distribution = Paso_Distribution_alloc(m_mpiInfo,
447 const_cast<index_t*>(&m_nodeDistribution[0]), 1, 0);
448
449 Paso_SystemMatrixPattern* pattern = Paso_SystemMatrixPattern_alloc(
450 MATRIX_FORMAT_DEFAULT, distribution, distribution,
451 mainPattern, colCouplePattern, rowCouplePattern,
452 connector, connector);
453 Paso_Pattern_free(mainPattern);
454 Paso_Pattern_free(colCouplePattern);
455 Paso_Pattern_free(rowCouplePattern);
456 Paso_Distribution_free(distribution);
457 return pattern;
458 }
459
460 void Rectangle::Print_Mesh_Info(const bool full) const
461 {
462 RipleyDomain::Print_Mesh_Info(full);
463 if (full) {
464 cout << " Id Coordinates" << endl;
465 cout.precision(15);
466 cout.setf(ios::scientific, ios::floatfield);
467 pair<double,double> xdx = getFirstCoordAndSpacing(0);
468 pair<double,double> ydy = getFirstCoordAndSpacing(1);
469 for (index_t i=0; i < getNumNodes(); i++) {
470 cout << " " << setw(5) << m_nodeId[i]
471 << " " << xdx.first+(i%m_N0)*xdx.second
472 << " " << ydy.first+(i/m_N0)*ydy.second << endl;
473 }
474 }
475 }
476
477 IndexVector Rectangle::getNumNodesPerDim() const
478 {
479 IndexVector ret;
480 ret.push_back(m_N0);
481 ret.push_back(m_N1);
482 return ret;
483 }
484
485 IndexVector Rectangle::getNumElementsPerDim() const
486 {
487 IndexVector ret;
488 ret.push_back(m_NE0);
489 ret.push_back(m_NE1);
490 return ret;
491 }
492
493 IndexVector Rectangle::getNumFacesPerBoundary() const
494 {
495 IndexVector ret(4, 0);
496 //left
497 if (m_offset0==0)
498 ret[0]=m_NE1;
499 //right
500 if (m_mpiInfo->rank%m_NX==m_NX-1)
501 ret[1]=m_NE1;
502 //bottom
503 if (m_offset1==0)
504 ret[2]=m_NE0;
505 //top
506 if (m_mpiInfo->rank/m_NX==m_NY-1)
507 ret[3]=m_NE0;
508 return ret;
509 }
510
511 pair<double,double> Rectangle::getFirstCoordAndSpacing(dim_t dim) const
512 {
513 if (dim==0) {
514 return pair<double,double>((m_l0*m_offset0)/m_gNE0, m_l0/m_gNE0);
515 } else if (dim==1) {
516 return pair<double,double>((m_l1*m_offset1)/m_gNE1, m_l1/m_gNE1);
517 }
518 throw RipleyException("getFirstCoordAndSpacing(): invalid argument");
519 }
520
521 //protected
522 dim_t Rectangle::getNumFaceElements() const
523 {
524 const IndexVector faces = getNumFacesPerBoundary();
525 dim_t n=0;
526 for (size_t i=0; i<faces.size(); i++)
527 n+=faces[i];
528 return n;
529 }
530
531 //protected
532 void Rectangle::assembleCoordinates(escript::Data& arg) const
533 {
534 escriptDataC x = arg.getDataC();
535 int numDim = m_numDim;
536 if (!isDataPointShapeEqual(&x, 1, &numDim))
537 throw RipleyException("setToX: Invalid Data object shape");
538 if (!numSamplesEqual(&x, 1, getNumNodes()))
539 throw RipleyException("setToX: Illegal number of samples in Data object");
540
541 pair<double,double> xdx = getFirstCoordAndSpacing(0);
542 pair<double,double> ydy = getFirstCoordAndSpacing(1);
543 arg.requireWrite();
544 #pragma omp parallel for
545 for (dim_t i1 = 0; i1 < m_N1; i1++) {
546 for (dim_t i0 = 0; i0 < m_N0; i0++) {
547 double* point = arg.getSampleDataRW(i0+m_N0*i1);
548 point[0] = xdx.first+i0*xdx.second;
549 point[1] = ydy.first+i1*ydy.second;
550 }
551 }
552 }
553
554 //private
555 void Rectangle::populateSampleIds()
556 {
557 // identifiers are ordered from left to right, bottom to top on each rank,
558 // except for the shared nodes which are owned by the rank below / to the
559 // left of the current rank
560
561 // build node distribution vector first.
562 // m_nodeDistribution[i] is the first node id on rank i, that is
563 // rank i owns m_nodeDistribution[i+1]-nodeDistribution[i] nodes
564 m_nodeDistribution.assign(m_mpiInfo->size+1, 0);
565 m_nodeDistribution[1]=getNumNodes();
566 for (dim_t k=1; k<m_mpiInfo->size-1; k++) {
567 const index_t x=k%m_NX;
568 const index_t y=k/m_NX;
569 index_t numNodes=getNumNodes();
570 if (x>0)
571 numNodes-=m_N1;
572 if (y>0)
573 numNodes-=m_N0;
574 if (x>0 && y>0)
575 numNodes++; // subtracted corner twice -> fix that
576 m_nodeDistribution[k+1]=m_nodeDistribution[k]+numNodes;
577 }
578 m_nodeDistribution[m_mpiInfo->size]=getNumDataPointsGlobal();
579
580 m_nodeId.resize(getNumNodes());
581
582 // the bottom row and left column are not owned by this rank so the
583 // identifiers need to be computed accordingly
584 const index_t left = (m_offset0==0 ? 0 : 1);
585 const index_t bottom = (m_offset1==0 ? 0 : 1);
586 if (left>0) {
587 const int neighbour=m_mpiInfo->rank-1;
588 const index_t leftN0=(neighbour%m_NX == 0 ? m_N0 : m_N0-1);
589 #pragma omp parallel for
590 for (dim_t i1=bottom; i1<m_N1; i1++) {
591 m_nodeId[i1*m_N0]=m_nodeDistribution[neighbour]
592 + (i1-bottom+1)*leftN0-1;
593 }
594 }
595 if (bottom>0) {
596 const int neighbour=m_mpiInfo->rank-m_NX;
597 const index_t bottomN0=(neighbour%m_NX == 0 ? m_N0 : m_N0-1);
598 const index_t bottomN1=(neighbour/m_NX == 0 ? m_N1 : m_N1-1);
599 #pragma omp parallel for
600 for (dim_t i0=left; i0<m_N0; i0++) {
601 m_nodeId[i0]=m_nodeDistribution[neighbour]
602 + (bottomN1-1)*bottomN0 + i0 - left;
603 }
604 }
605 if (left>0 && bottom>0) {
606 const int neighbour=m_mpiInfo->rank-m_NX-1;
607 const index_t N0=(neighbour%m_NX == 0 ? m_N0 : m_N0-1);
608 const index_t N1=(neighbour/m_NX == 0 ? m_N1 : m_N1-1);
609 m_nodeId[0]=m_nodeDistribution[neighbour]+N0*N1-1;
610 }
611
612 // the rest of the id's are contiguous
613 const index_t firstId=m_nodeDistribution[m_mpiInfo->rank];
614 #pragma omp parallel for
615 for (dim_t i1=bottom; i1<m_N1; i1++) {
616 for (dim_t i0=left; i0<m_N0; i0++) {
617 m_nodeId[i0+i1*m_N0] = firstId+i0-left+(i1-bottom)*(m_N0-left);
618 }
619 }
620
621 // elements
622 m_elementId.resize(getNumElements());
623 #pragma omp parallel for
624 for (dim_t k=0; k<getNumElements(); k++) {
625 m_elementId[k]=k;
626 }
627
628 // face elements
629 m_faceId.resize(getNumFaceElements());
630 #pragma omp parallel for
631 for (dim_t k=0; k<getNumFaceElements(); k++) {
632 m_faceId[k]=k;
633 }
634 }
635
636 //private
637 int Rectangle::insertNeighbours(IndexVector& index, index_t node) const
638 {
639 const dim_t myN0 = (m_offset0==0 ? m_N0 : m_N0-1);
640 const dim_t myN1 = (m_offset1==0 ? m_N1 : m_N1-1);
641 const int x=node%myN0;
642 const int y=node/myN0;
643 int num=0;
644 if (y>0) {
645 if (x>0) {
646 // bottom-left
647 index.push_back(node-myN0-1);
648 num++;
649 }
650 // bottom
651 index.push_back(node-myN0);
652 num++;
653 if (x<myN0-1) {
654 // bottom-right
655 index.push_back(node-myN0+1);
656 num++;
657 }
658 }
659 if (x<myN0-1) {
660 // right
661 index.push_back(node+1);
662 num++;
663 if (y<myN1-1) {
664 // top-right
665 index.push_back(node+myN0+1);
666 num++;
667 }
668 }
669 if (y<myN1-1) {
670 // top
671 index.push_back(node+myN0);
672 num++;
673 if (x>0) {
674 // top-left
675 index.push_back(node+myN0-1);
676 num++;
677 }
678 }
679 if (x>0) {
680 // left
681 index.push_back(node-1);
682 num++;
683 }
684
685 return num;
686 }
687
688 //private
689 void Rectangle::generateCouplePatterns(Paso_Pattern** colPattern, Paso_Pattern** rowPattern) const
690 {
691 IndexVector ptr(1,0);
692 IndexVector index;
693 const dim_t myN0 = (m_offset0==0 ? m_N0 : m_N0-1);
694 const dim_t myN1 = (m_offset1==0 ? m_N1 : m_N1-1);
695 const IndexVector faces=getNumFacesPerBoundary();
696
697 // bottom edge
698 dim_t n=0;
699 if (faces[0] == 0) {
700 index.push_back(2*(myN0+myN1+1));
701 index.push_back(2*(myN0+myN1+1)+1);
702 n+=2;
703 if (faces[2] == 0) {
704 index.push_back(0);
705 index.push_back(1);
706 index.push_back(2);
707 n+=3;
708 }
709 } else if (faces[2] == 0) {
710 index.push_back(1);
711 index.push_back(2);
712 n+=2;
713 }
714 // n=neighbours of bottom-left corner node
715 ptr.push_back(ptr.back()+n);
716 n=0;
717 if (faces[2] == 0) {
718 for (dim_t i=1; i<myN0-1; i++) {
719 index.push_back(i);
720 index.push_back(i+1);
721 index.push_back(i+2);
722 ptr.push_back(ptr.back()+3);
723 }
724 index.push_back(myN0-1);
725 index.push_back(myN0);
726 n+=2;
727 if (faces[1] == 0) {
728 index.push_back(myN0+1);
729 index.push_back(myN0+2);
730 index.push_back(myN0+3);
731 n+=3;
732 }
733 } else {
734 for (dim_t i=1; i<myN0-1; i++) {
735 ptr.push_back(ptr.back());
736 }
737 if (faces[1] == 0) {
738 index.push_back(myN0+2);
739 index.push_back(myN0+3);
740 n+=2;
741 }
742 }
743 // n=neighbours of bottom-right corner node
744 ptr.push_back(ptr.back()+n);
745
746 // 2nd row to 2nd last row
747 for (dim_t i=1; i<myN1-1; i++) {
748 // left edge
749 if (faces[0] == 0) {
750 index.push_back(2*(myN0+myN1+2)-i);
751 index.push_back(2*(myN0+myN1+2)-i-1);
752 index.push_back(2*(myN0+myN1+2)-i-2);
753 ptr.push_back(ptr.back()+3);
754 } else {
755 ptr.push_back(ptr.back());
756 }
757 for (dim_t j=1; j<myN0-1; j++) {
758 ptr.push_back(ptr.back());
759 }
760 // right edge
761 if (faces[1] == 0) {
762 index.push_back(myN0+i+1);
763 index.push_back(myN0+i+2);
764 index.push_back(myN0+i+3);
765 ptr.push_back(ptr.back()+3);
766 } else {
767 ptr.push_back(ptr.back());
768 }
769 }
770
771 // top edge
772 n=0;
773 if (faces[0] == 0) {
774 index.push_back(2*myN0+myN1+5);
775 index.push_back(2*myN0+myN1+4);
776 n+=2;
777 if (faces[3] == 0) {
778 index.push_back(2*myN0+myN1+3);
779 index.push_back(2*myN0+myN1+2);
780 index.push_back(2*myN0+myN1+1);
781 n+=3;
782 }
783 } else if (faces[3] == 0) {
784 index.push_back(2*myN0+myN1+2);
785 index.push_back(2*myN0+myN1+1);
786 n+=2;
787 }
788 // n=neighbours of top-left corner node
789 ptr.push_back(ptr.back()+n);
790 n=0;
791 if (faces[3] == 0) {
792 for (dim_t i=1; i<myN0-1; i++) {
793 index.push_back(2*myN0+myN1+i+1);
794 index.push_back(2*myN0+myN1+i);
795 index.push_back(2*myN0+myN1+i-1);
796 ptr.push_back(ptr.back()+3);
797 }
798 index.push_back(myN0+myN1+4);
799 index.push_back(myN0+myN1+3);
800 n+=2;
801 if (faces[1] == 0) {
802 index.push_back(myN0+myN1+2);
803 index.push_back(myN0+myN1+1);
804 index.push_back(myN0+myN1);
805 n+=3;
806 }
807 } else {
808 for (dim_t i=1; i<myN0-1; i++) {
809 ptr.push_back(ptr.back());
810 }
811 if (faces[1] == 0) {
812 index.push_back(myN0+myN1+1);
813 index.push_back(myN0+myN1);
814 n+=2;
815 }
816 }
817 // n=neighbours of top-right corner node
818 ptr.push_back(ptr.back()+n);
819
820 dim_t M=ptr.size()-1;
821 map<index_t,index_t> idMap;
822 dim_t N=0;
823 for (IndexVector::iterator it=index.begin(); it!=index.end(); it++) {
824 if (idMap.find(*it)==idMap.end()) {
825 idMap[*it]=N;
826 *it=N++;
827 } else {
828 *it=idMap[*it];
829 }
830 }
831
832 cout << "--- colCouple_pattern ---" << endl;
833 cout << "M=" << M << ", N=" << N << endl;
834 for (size_t i=0; i<ptr.size(); i++) {
835 cout << "ptr[" << i << "]=" << ptr[i] << endl;
836 }
837 for (size_t i=0; i<index.size(); i++) {
838 cout << "index[" << i << "]=" << index[i] << endl;
839 }
840
841 // now build the row couple pattern
842 IndexVector ptr2(1,0);
843 IndexVector index2;
844 for (dim_t id=0; id<N; id++) {
845 n=0;
846 for (dim_t i=0; i<M; i++) {
847 for (dim_t j=ptr[i]; j<ptr[i+1]; j++) {
848 if (index[j]==id) {
849 index2.push_back(i);
850 n++;
851 break;
852 }
853 }
854 }
855 ptr2.push_back(ptr2.back()+n);
856 }
857
858 cout << "--- rowCouple_pattern ---" << endl;
859 cout << "M=" << N << ", N=" << M << endl;
860 for (size_t i=0; i<ptr2.size(); i++) {
861 cout << "ptr[" << i << "]=" << ptr2[i] << endl;
862 }
863 for (size_t i=0; i<index2.size(); i++) {
864 cout << "index[" << i << "]=" << index2[i] << endl;
865 }
866
867 // paso will manage the memory
868 index_t* indexC = MEMALLOC(index.size(), index_t);
869 index_t* ptrC = MEMALLOC(ptr.size(), index_t);
870 copy(index.begin(), index.end(), indexC);
871 copy(ptr.begin(), ptr.end(), ptrC);
872 *colPattern=Paso_Pattern_alloc(MATRIX_FORMAT_DEFAULT, M, N, ptrC, indexC);
873
874 // paso will manage the memory
875 indexC = MEMALLOC(index2.size(), index_t);
876 ptrC = MEMALLOC(ptr2.size(), index_t);
877 copy(index2.begin(), index2.end(), indexC);
878 copy(ptr2.begin(), ptr2.end(), ptrC);
879 *rowPattern=Paso_Pattern_alloc(MATRIX_FORMAT_DEFAULT, N, M, ptrC, indexC);
880 }
881
882 } // end of namespace ripley
883

  ViewVC Help
Powered by ViewVC 1.1.26