/[escript]/trunk/ripley/test/python/run_readWriteOnMultiRes.py
ViewVC logotype

Contents of /trunk/ripley/test/python/run_readWriteOnMultiRes.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 6141 - (show annotations)
Wed Apr 6 03:51:30 2016 UTC (2 years, 8 months ago) by caltinay
File MIME type: text/x-python
File size: 16481 byte(s)
more namespacing of defines.

1
2 ##############################################################################
3 #
4 # Copyright (c) 2003-2016 by The University of Queensland
5 # http://www.uq.edu.au
6 #
7 # Primary Business: Queensland, Australia
8 # Licensed under the Apache License, version 2.0
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 # Development 2012-2013 by School of Earth Sciences
13 # Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 #
15 ##############################################################################
16
17 from __future__ import print_function, division
18
19 __copyright__="""Copyright (c) 2003-2016 by The University of Queensland
20 http://www.uq.edu.au
21 Primary Business: Queensland, Australia"""
22 __license__="""Licensed under the Apache License, version 2.0
23 http://www.apache.org/licenses/LICENSE-2.0"""
24 __url__="https://launchpad.net/escript-finley"
25
26 import os
27 import numpy as np
28 import esys.escriptcore.utestselect as unittest
29 from esys.escriptcore.testing import *
30 from esys.escript import *
31 from esys.ripley import *
32
33 try:
34 RIPLEY_WORKDIR=os.environ['RIPLEY_WORKDIR']
35 except KeyError:
36 RIPLEY_WORKDIR='/tmp'
37
38 HAVE_UNZIP = getEscriptParamInt('HAVE_UNZIP')
39
40 mpiSize = getMPISizeWorld()
41 mpiRank = getMPIRankWorld()
42
43 def adjust(NE, ftype):
44 if ftype in (ContinuousFunction, Solution):
45 return [i+1 for i in NE]
46 return NE
47
48 def Rectangle(**kwargs):
49 m = MultiResolutionDomain(2, **kwargs)
50 return m.getLevel(1)
51
52 def Brick(**kwargs):
53 m = MultiResolutionDomain(3, **kwargs)
54 return m.getLevel(1)
55
56 @unittest.skipIf(mpiSize > 1, "Multiresolution domains don't support multiprocess yet")
57 class WriteBinaryGridTestBase(unittest.TestCase): #subclassing required
58 NX = 10*mpiSize-1
59 NZ = 10
60
61 def generateUniqueData(self, ftype):
62 dim = self.domain.getDim()
63 FSx=ftype(self.domain).getX()
64 NE = adjust(self.NE, ftype)
65 # normalise and scale range of values
66 x = [FSx[i]-inf(FSx[i]) for i in range(dim)]
67 x = [(NE[i]-1)*(x[i]/sup(x[i])) for i in range(dim)]
68 xMax = [int(sup(x[i]))+1 for i in range(dim)]
69 nvals=NE[0]*NE[1]
70 data = x[0] + xMax[0]*x[1]
71 if self.datatype == DATATYPE_INT32:
72 data += 0.05
73 if dim > 2:
74 data = data + xMax[0]*xMax[1]*x[2]
75 nvals*=NE[2]
76
77 grid = np.array(range(nvals), dtype=self.dtype).reshape(tuple(reversed(NE)))
78 return data, grid
79
80 def writeThenRead(self, data, ftype, fcode):
81 filename = os.path.join(RIPLEY_WORKDIR, "_wgrid%dd%s"%(self.domain.getDim(),fcode))
82 filename = filename + self.dtype.replace('<','L').replace('>','B')
83 self.domain.writeBinaryGrid(data, filename, self.byteorder, self.datatype)
84 MPIBarrierWorld()
85 result = np.fromfile(filename, dtype=self.dtype).reshape(
86 tuple(reversed(adjust(self.NE,ftype))))
87 return result
88
89 def test_writeGrid2D(self):
90 self.NE = [self.NX, self.NZ]
91 self.domain = Rectangle(n0=self.NE[0], n1=self.NE[1], d1=1)
92 for ftype,fcode in [(ReducedFunction,'RF'), (ContinuousFunction,'CF'), (Solution, 'Sol')]:
93 data, ref = self.generateUniqueData(ftype)
94 with self.assertRaises(RuntimeError):
95 result = self.writeThenRead(data, ftype, fcode)
96 self.assertAlmostEquals(Lsup(ref-result), 0, delta=1e-9,
97 msg="Data doesn't match for "+str(ftype(self.domain)))
98
99 def test_writeGrid3D(self):
100 self.NE = [self.NX, self.NX, self.NZ]
101 self.domain = Brick(n0=self.NE[0], n1=self.NE[1], n2=self.NE[2], d2=1)
102 for ftype,fcode in [(ReducedFunction,'RF'), (ContinuousFunction,'CF'), (Solution, 'Sol')]:
103 data, ref = self.generateUniqueData(ftype)
104 with self.assertRaises(RuntimeError):
105 result = self.writeThenRead(data, ftype, fcode)
106 self.assertAlmostEquals(Lsup(ref-result), 0, delta=1e-9,
107 msg="Data doesn't match for "+str(ftype(self.domain)))
108
109 class Test_writeBinaryGridRipley_LITTLE_FLOAT32(WriteBinaryGridTestBase):
110 def setUp(self):
111 self.byteorder = BYTEORDER_LITTLE_ENDIAN
112 self.datatype = DATATYPE_FLOAT32
113 self.dtype = "<f4"
114
115 class Test_writeBinaryGridRipley_LITTLE_FLOAT64(WriteBinaryGridTestBase):
116 def setUp(self):
117 self.byteorder = BYTEORDER_LITTLE_ENDIAN
118 self.datatype = DATATYPE_FLOAT64
119 self.dtype = "<f8"
120
121 class Test_writeBinaryGridRipley_LITTLE_INT32(WriteBinaryGridTestBase):
122 def setUp(self):
123 self.byteorder = BYTEORDER_LITTLE_ENDIAN
124 self.datatype = DATATYPE_INT32
125 self.dtype = "<i4"
126
127 class Test_writeBinaryGridRipley_BIG_FLOAT32(WriteBinaryGridTestBase):
128 def setUp(self):
129 self.byteorder = BYTEORDER_BIG_ENDIAN
130 self.datatype = DATATYPE_FLOAT32
131 self.dtype = ">f4"
132
133 class Test_writeBinaryGridRipley_BIG_FLOAT64(WriteBinaryGridTestBase):
134 def setUp(self):
135 self.byteorder = BYTEORDER_BIG_ENDIAN
136 self.datatype = DATATYPE_FLOAT64
137 self.dtype = ">f8"
138
139 class Test_writeBinaryGridRipley_BIG_INT32(WriteBinaryGridTestBase):
140 def setUp(self):
141 self.byteorder = BYTEORDER_BIG_ENDIAN
142 self.datatype = DATATYPE_INT32
143 self.dtype = ">i4"
144
145
146 class ReadBinaryGridTestBase(unittest.TestCase): #subclassing required
147 """
148 The reader tests work in several stages:
149 1) create numpy array and write to temporary file (ref)
150 2) call readBinaryGrid with that filename
151 3) write the resulting Data object using writeBinaryGrid (test)
152 4) read the result using numpy and compare (ref) and (test)
153 As such, it is important to note that a working writeBinaryGrid() method
154 is assumed!
155 """
156 # set defaults which may be overridden in subclasses
157 NX = 10
158 NZ = 8
159 fspaces = [(ReducedFunction,'RF'), (ContinuousFunction,'CF')]
160 byteorder = BYTEORDER_NATIVE
161 datatype = DATATYPE_FLOAT64
162 dtype = "f8"
163 shape = ()
164 fill = -42.57
165 first = [0,0,0]
166 multiplier = [1,1,1]
167 reverse = [0,0,0]
168
169 def generateUniqueData(self, ftype):
170 dim = self.domain.getDim()
171 NE = adjust(self.Ndata, ftype)
172 nvals=NE[0]*NE[1]
173 if dim > 2:
174 nvals*=NE[2]
175 grid = np.array(range(nvals), dtype=self.dtype).reshape(tuple(reversed(NE)))
176 return grid
177
178 def write(self, data, filename):
179 self.domain.writeBinaryGrid(data, filename, self.byteorder, self.datatype)
180
181 def read(self, filename, ftype):
182 first = self.first[:self.domain.getDim()]
183 multiplier = self.multiplier[:self.domain.getDim()]
184 reverse = self.reverse[:self.domain.getDim()]
185 numValues=adjust(self.Ndata, ftype)
186 return readBinaryGrid(filename, ftype(self.domain),
187 shape=self.shape, fill=self.fill, byteOrder=self.byteorder,
188 dataType=self.datatype, first=first, numValues=numValues,
189 multiplier=multiplier, reverse=reverse)
190
191 def numpy2Data2Numpy(self, ref, ftype, fcode):
192 filename = os.path.join(RIPLEY_WORKDIR, "_rgrid%dd%s"%(self.domain.getDim(),fcode))
193 filename = filename + self.dtype.replace('<','L').replace('>','B')
194 if mpiRank == 0:
195 ref.tofile(filename)
196 MPIBarrierWorld()
197 # step 2 - read
198 data = self.read(filename, ftype)
199 MPIBarrierWorld()
200 # step 3 - write
201 self.write(data, filename) # overwrite is ok
202 MPIBarrierWorld()
203 result = np.fromfile(filename, dtype=self.dtype).reshape(
204 tuple(reversed(adjust(self.NE,ftype))))
205 return result
206
207 def test_readGrid2D(self):
208 if self.multiplier[0] == 1:
209 self.NE = [self.NX*mpiSize-1, self.NZ*self.multiplier[1]]
210 else:
211 self.NE = [self.NX*mpiSize*self.multiplier[0]-1, self.NZ*self.multiplier[1]]
212 self.domain = Rectangle(n0=self.NE[0], n1=self.NE[1], d0=mpiSize, d1=1)
213 for ftype,fcode in self.fspaces:
214 self.Ndata = [self.NX*mpiSize-1, self.NZ]
215 if ftype==ContinuousFunction:
216 self.Ndata[1] = self.NZ-1
217 # step 1 - generate
218 ref = self.generateUniqueData(ftype)
219 # step 2 & 3
220 with self.assertRaises(RuntimeError):
221 result = self.numpy2Data2Numpy(ref, ftype, fcode)
222 # apply transformations to be able to compare
223 if self.reverse[0]:
224 result = result[...,::-1]
225 if self.reverse[1]:
226 result = result[::-1,:]
227 for i in range(2):
228 ref = np.repeat(ref, self.multiplier[i], axis=1-i)
229
230 # if domain larger than data: add column(s)/row(s) with fill value
231 fill=np.array(self.fill, dtype=ref.dtype)
232 realNE = adjust(self.NE,ftype)
233 for d in range(2):
234 excess = realNE[d]-ref.shape[1-d]
235 if excess > 0:
236 shape = list(ref.shape)
237 shape[1-d] = excess
238 extra = fill * np.ones(shape)
239 if self.reverse[d]:
240 ref = np.append(extra, ref, axis=1-d)
241 else:
242 ref = np.append(ref, extra, axis=1-d)
243
244 # step 4 - compare
245 self.assertAlmostEquals(Lsup(ref-result), 0, delta=1e-9,
246 msg="Data doesn't match for "+str(ftype(self.domain)))
247
248 @unittest.skipIf(mpiSize > 1, "3D Multiresolution domains don't support multiprocess yet")
249 def test_readGrid3D(self):
250 if self.multiplier[0] == 1:
251 self.NE = [self.NX*mpiSize-1, self.NX*self.multiplier[1], self.NZ*self.multiplier[2]]
252 else:
253 self.NE = [self.NX*mpiSize*self.multiplier[0]-1,
254 self.NX*self.multiplier[1], self.NZ*self.multiplier[2]]
255 self.domain = Brick(n0=self.NE[0], n1=self.NE[1], n2=self.NE[2], d0=mpiSize, d1=1, d2=1)
256 for ftype,fcode in self.fspaces:
257 self.Ndata = [self.NX*mpiSize-1, self.NX, self.NZ]
258 if ftype==ContinuousFunction:
259 self.Ndata[1] = self.NX-1
260 self.Ndata[2] = self.NZ-1
261 # step 1 - generate
262 ref = self.generateUniqueData(ftype)
263 # step 2 & 3
264 with self.assertRaises(RuntimeError):
265 result = self.numpy2Data2Numpy(ref, ftype, fcode)
266 # apply transformations to be able to compare
267 if self.reverse[0]:
268 result = result[...,::-1]
269 if self.reverse[1]:
270 result = result[...,::-1,:]
271 if self.reverse[2]:
272 result = result[::-1,:,:]
273 for i in range(3):
274 ref = np.repeat(ref, self.multiplier[i], axis=2-i)
275
276 # if domain larger than data: add column(s)/row(s) with fill value
277 fill=np.array(self.fill, dtype=ref.dtype)
278 realNE = adjust(self.NE,ftype)
279 for d in range(3):
280 excess = realNE[d]-ref.shape[2-d]
281 if excess > 0:
282 shape = list(ref.shape)
283 shape[2-d] = excess
284 extra = fill * np.ones(shape)
285 if self.reverse[d]:
286 ref = np.append(extra, ref, axis=2-d)
287 else:
288 ref = np.append(ref, extra, axis=2-d)
289
290 # step 4 - compare
291 self.assertAlmostEquals(Lsup(ref-result), 0, delta=1e-9,
292 msg="Data doesn't match for "+str(ftype(self.domain)))
293
294
295 # The following block tests the reader for different byte orders and data
296 # types with domain-filling data (i.e. multiplier=1, reverse=0 and N=NE)
297
298 class Test_readBinaryGridRipley_LITTLE_FLOAT32(ReadBinaryGridTestBase):
299 def setUp(self):
300 self.byteorder = BYTEORDER_LITTLE_ENDIAN
301 self.datatype = DATATYPE_FLOAT32
302 self.dtype = "<f4"
303
304 class Test_readBinaryGridRipley_LITTLE_FLOAT64(ReadBinaryGridTestBase):
305 def setUp(self):
306 self.byteorder = BYTEORDER_LITTLE_ENDIAN
307 self.datatype = DATATYPE_FLOAT64
308 self.dtype = "<f8"
309
310 class Test_readBinaryGridRipley_LITTLE_INT32(ReadBinaryGridTestBase):
311 def setUp(self):
312 self.byteorder = BYTEORDER_LITTLE_ENDIAN
313 self.datatype = DATATYPE_INT32
314 self.dtype = "<i4"
315
316 class Test_readBinaryGridRipley_BIG_FLOAT32(ReadBinaryGridTestBase):
317 def setUp(self):
318 self.byteorder = BYTEORDER_BIG_ENDIAN
319 self.datatype = DATATYPE_FLOAT32
320 self.dtype = ">f4"
321
322 class Test_readBinaryGridRipley_BIG_FLOAT64(ReadBinaryGridTestBase):
323 def setUp(self):
324 self.byteorder = BYTEORDER_BIG_ENDIAN
325 self.datatype = DATATYPE_FLOAT64
326 self.dtype = ">f8"
327
328 class Test_readBinaryGridRipley_BIG_INT32(ReadBinaryGridTestBase):
329 def setUp(self):
330 self.byteorder = BYTEORDER_BIG_ENDIAN
331 self.datatype = DATATYPE_INT32
332 self.dtype = ">i4"
333
334 @unittest.skip("reverseX not supported yet")
335 class Test_readBinaryGridRipley_reverseX(ReadBinaryGridTestBase):
336 def setUp(self):
337 self.reverse = [1,0,0]
338
339 @unittest.skip("reverseY not supported yet")
340 class Test_readBinaryGridRipley_reverseY(ReadBinaryGridTestBase):
341 def setUp(self):
342 self.reverse = [0,1,0]
343
344 class Test_readBinaryGridRipley_reverseZ(ReadBinaryGridTestBase):
345 def setUp(self):
346 self.reverse = [0,0,1]
347
348 class Test_readBinaryGridRipley_multiplierX(ReadBinaryGridTestBase):
349 def setUp(self):
350 self.multiplier = [2,1,1]
351
352 class Test_readBinaryGridRipley_multiplierY(ReadBinaryGridTestBase):
353 def setUp(self):
354 self.multiplier = [1,2,1]
355
356 class Test_readBinaryGridRipley_multiplierZ(ReadBinaryGridTestBase):
357 def setUp(self):
358 self.multiplier = [1,1,2]
359
360 class Test_readBinaryGridRipley_multiplierXYZ(ReadBinaryGridTestBase):
361 def setUp(self):
362 self.multiplier = [2,3,4]
363
364
365 @unittest.skipIf(getMPISizeWorld() > 1,
366 "Skipping compressed binary grid tests due to element stretching")
367 class Test_readBinaryGridZippedRipley(unittest.TestCase):
368 # constants
369 byteorder = BYTEORDER_NATIVE
370 datatype = DATATYPE_FLOAT64
371
372 def read(self, filename, FS, expected, zipped = False):
373 first = [0 for i in expected]
374 reverse = [0 for i in expected]
375 scale = [1 for i in expected]
376
377 if not zipped:
378 return readBinaryGrid(filename, FS, (), 50000,
379 self.byteorder, self.datatype, first, expected, scale, reverse)
380
381 if not HAVE_UNZIP:
382 raise unittest.SkipTest("unzip library not available (boost_iostreams)")
383 return ripleycpp._readBinaryGridFromZipped(filename, FS, (), 50000,
384 self.byteorder, self.datatype, first, expected, scale, reverse)
385
386 def test_readCompressed2D(self):
387 NE = [9, 10]
388 domain = Rectangle(n0=NE[0], n1=NE[1], d1=1)
389 for filename, ftype in [("RectRedF%s.grid.gz", ReducedFunction),
390 ("RectConF%s.grid.gz", ContinuousFunction)]:
391 FS = ftype(domain)
392 filename = os.path.join("ref_data", filename%mpiSize)
393 with self.assertRaises(RuntimeError): #non-parent multidomain
394 unzipped = self.read(filename[:-3], FS, adjust(NE, ftype))
395 zipped = self.read(filename, FS, adjust(NE, ftype), True)
396 self.assertEqual(Lsup(zipped - unzipped), 0, "Data objects don't match for "+str(FS))
397
398 def test_readCompressed3D(self):
399 NE = [9, 9, 10]
400 domain = Brick(n0=NE[0], n1=NE[1], n2=NE[2], d1=1, d2=1)
401 for filename, ftype in [("BrickRedF%s.grid.gz", ReducedFunction),
402 ("BrickConF%s.grid.gz", ContinuousFunction)]:
403 FS = ftype(domain)
404 filename = os.path.join("ref_data", filename%mpiSize)
405 with self.assertRaises(RuntimeError): #non-parent multidomain
406 unzipped = self.read(filename[:-3], FS, adjust(NE, ftype))
407 zipped = self.read(filename, FS, adjust(NE, ftype), True)
408 self.assertEqual(Lsup(zipped - unzipped), 0, "Data objects don't match for "+str(FS))
409
410
411 if __name__ == '__main__':
412 run_tests(__name__, exit_on_failure=True)
413

  ViewVC Help
Powered by ViewVC 1.1.26