/[escript]/trunk/ripley/test/python/run_readWriteOnMultiRes.py
ViewVC logotype

Annotation of /trunk/ripley/test/python/run_readWriteOnMultiRes.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 6141 - (hide annotations)
Wed Apr 6 03:51:30 2016 UTC (2 years, 7 months ago) by caltinay
File MIME type: text/x-python
File size: 16481 byte(s)
more namespacing of defines.

1 sshaw 5528
2     ##############################################################################
3     #
4 jfenwick 5863 # Copyright (c) 2003-2016 by The University of Queensland
5 sshaw 5528 # http://www.uq.edu.au
6     #
7     # Primary Business: Queensland, Australia
8 jfenwick 6112 # Licensed under the Apache License, version 2.0
9     # http://www.apache.org/licenses/LICENSE-2.0
10 sshaw 5528 #
11     # Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12     # Development 2012-2013 by School of Earth Sciences
13     # Development from 2014 by Centre for Geoscience Computing (GeoComp)
14     #
15     ##############################################################################
16    
17 sshaw 5706 from __future__ import print_function, division
18    
19 jfenwick 5863 __copyright__="""Copyright (c) 2003-2016 by The University of Queensland
20 sshaw 5528 http://www.uq.edu.au
21     Primary Business: Queensland, Australia"""
22 jfenwick 6112 __license__="""Licensed under the Apache License, version 2.0
23     http://www.apache.org/licenses/LICENSE-2.0"""
24 sshaw 5528 __url__="https://launchpad.net/escript-finley"
25    
26     import os
27     import numpy as np
28     import esys.escriptcore.utestselect as unittest
29     from esys.escriptcore.testing import *
30     from esys.escript import *
31     from esys.ripley import *
32    
33     try:
34     RIPLEY_WORKDIR=os.environ['RIPLEY_WORKDIR']
35     except KeyError:
36     RIPLEY_WORKDIR='/tmp'
37    
38 caltinay 6141 HAVE_UNZIP = getEscriptParamInt('HAVE_UNZIP')
39 sshaw 5528
40     mpiSize = getMPISizeWorld()
41     mpiRank = getMPIRankWorld()
42    
43     def adjust(NE, ftype):
44     if ftype in (ContinuousFunction, Solution):
45     return [i+1 for i in NE]
46     return NE
47    
48     def Rectangle(**kwargs):
49     m = MultiResolutionDomain(2, **kwargs)
50     return m.getLevel(1)
51    
52     def Brick(**kwargs):
53     m = MultiResolutionDomain(3, **kwargs)
54     return m.getLevel(1)
55    
56     @unittest.skipIf(mpiSize > 1, "Multiresolution domains don't support multiprocess yet")
57     class WriteBinaryGridTestBase(unittest.TestCase): #subclassing required
58     NX = 10*mpiSize-1
59     NZ = 10
60    
61     def generateUniqueData(self, ftype):
62     dim = self.domain.getDim()
63     FSx=ftype(self.domain).getX()
64     NE = adjust(self.NE, ftype)
65     # normalise and scale range of values
66     x = [FSx[i]-inf(FSx[i]) for i in range(dim)]
67     x = [(NE[i]-1)*(x[i]/sup(x[i])) for i in range(dim)]
68     xMax = [int(sup(x[i]))+1 for i in range(dim)]
69     nvals=NE[0]*NE[1]
70     data = x[0] + xMax[0]*x[1]
71     if self.datatype == DATATYPE_INT32:
72     data += 0.05
73     if dim > 2:
74     data = data + xMax[0]*xMax[1]*x[2]
75     nvals*=NE[2]
76    
77     grid = np.array(range(nvals), dtype=self.dtype).reshape(tuple(reversed(NE)))
78     return data, grid
79    
80     def writeThenRead(self, data, ftype, fcode):
81     filename = os.path.join(RIPLEY_WORKDIR, "_wgrid%dd%s"%(self.domain.getDim(),fcode))
82     filename = filename + self.dtype.replace('<','L').replace('>','B')
83     self.domain.writeBinaryGrid(data, filename, self.byteorder, self.datatype)
84     MPIBarrierWorld()
85     result = np.fromfile(filename, dtype=self.dtype).reshape(
86     tuple(reversed(adjust(self.NE,ftype))))
87     return result
88    
89     def test_writeGrid2D(self):
90     self.NE = [self.NX, self.NZ]
91     self.domain = Rectangle(n0=self.NE[0], n1=self.NE[1], d1=1)
92     for ftype,fcode in [(ReducedFunction,'RF'), (ContinuousFunction,'CF'), (Solution, 'Sol')]:
93     data, ref = self.generateUniqueData(ftype)
94     with self.assertRaises(RuntimeError):
95     result = self.writeThenRead(data, ftype, fcode)
96     self.assertAlmostEquals(Lsup(ref-result), 0, delta=1e-9,
97     msg="Data doesn't match for "+str(ftype(self.domain)))
98    
99     def test_writeGrid3D(self):
100     self.NE = [self.NX, self.NX, self.NZ]
101     self.domain = Brick(n0=self.NE[0], n1=self.NE[1], n2=self.NE[2], d2=1)
102     for ftype,fcode in [(ReducedFunction,'RF'), (ContinuousFunction,'CF'), (Solution, 'Sol')]:
103     data, ref = self.generateUniqueData(ftype)
104     with self.assertRaises(RuntimeError):
105     result = self.writeThenRead(data, ftype, fcode)
106     self.assertAlmostEquals(Lsup(ref-result), 0, delta=1e-9,
107     msg="Data doesn't match for "+str(ftype(self.domain)))
108    
109     class Test_writeBinaryGridRipley_LITTLE_FLOAT32(WriteBinaryGridTestBase):
110     def setUp(self):
111     self.byteorder = BYTEORDER_LITTLE_ENDIAN
112     self.datatype = DATATYPE_FLOAT32
113     self.dtype = "<f4"
114    
115     class Test_writeBinaryGridRipley_LITTLE_FLOAT64(WriteBinaryGridTestBase):
116     def setUp(self):
117     self.byteorder = BYTEORDER_LITTLE_ENDIAN
118     self.datatype = DATATYPE_FLOAT64
119     self.dtype = "<f8"
120    
121     class Test_writeBinaryGridRipley_LITTLE_INT32(WriteBinaryGridTestBase):
122     def setUp(self):
123     self.byteorder = BYTEORDER_LITTLE_ENDIAN
124     self.datatype = DATATYPE_INT32
125     self.dtype = "<i4"
126    
127     class Test_writeBinaryGridRipley_BIG_FLOAT32(WriteBinaryGridTestBase):
128     def setUp(self):
129     self.byteorder = BYTEORDER_BIG_ENDIAN
130     self.datatype = DATATYPE_FLOAT32
131     self.dtype = ">f4"
132    
133     class Test_writeBinaryGridRipley_BIG_FLOAT64(WriteBinaryGridTestBase):
134     def setUp(self):
135     self.byteorder = BYTEORDER_BIG_ENDIAN
136     self.datatype = DATATYPE_FLOAT64
137     self.dtype = ">f8"
138    
139     class Test_writeBinaryGridRipley_BIG_INT32(WriteBinaryGridTestBase):
140     def setUp(self):
141     self.byteorder = BYTEORDER_BIG_ENDIAN
142     self.datatype = DATATYPE_INT32
143     self.dtype = ">i4"
144    
145 sshaw 5540
146 sshaw 5528 class ReadBinaryGridTestBase(unittest.TestCase): #subclassing required
147     """
148     The reader tests work in several stages:
149     1) create numpy array and write to temporary file (ref)
150     2) call readBinaryGrid with that filename
151     3) write the resulting Data object using writeBinaryGrid (test)
152     4) read the result using numpy and compare (ref) and (test)
153     As such, it is important to note that a working writeBinaryGrid() method
154     is assumed!
155     """
156     # set defaults which may be overridden in subclasses
157     NX = 10
158     NZ = 8
159     fspaces = [(ReducedFunction,'RF'), (ContinuousFunction,'CF')]
160     byteorder = BYTEORDER_NATIVE
161     datatype = DATATYPE_FLOAT64
162     dtype = "f8"
163     shape = ()
164     fill = -42.57
165     first = [0,0,0]
166     multiplier = [1,1,1]
167     reverse = [0,0,0]
168    
169     def generateUniqueData(self, ftype):
170     dim = self.domain.getDim()
171     NE = adjust(self.Ndata, ftype)
172     nvals=NE[0]*NE[1]
173     if dim > 2:
174     nvals*=NE[2]
175     grid = np.array(range(nvals), dtype=self.dtype).reshape(tuple(reversed(NE)))
176     return grid
177    
178     def write(self, data, filename):
179     self.domain.writeBinaryGrid(data, filename, self.byteorder, self.datatype)
180    
181     def read(self, filename, ftype):
182     first = self.first[:self.domain.getDim()]
183     multiplier = self.multiplier[:self.domain.getDim()]
184     reverse = self.reverse[:self.domain.getDim()]
185     numValues=adjust(self.Ndata, ftype)
186     return readBinaryGrid(filename, ftype(self.domain),
187     shape=self.shape, fill=self.fill, byteOrder=self.byteorder,
188     dataType=self.datatype, first=first, numValues=numValues,
189     multiplier=multiplier, reverse=reverse)
190    
191     def numpy2Data2Numpy(self, ref, ftype, fcode):
192     filename = os.path.join(RIPLEY_WORKDIR, "_rgrid%dd%s"%(self.domain.getDim(),fcode))
193     filename = filename + self.dtype.replace('<','L').replace('>','B')
194     if mpiRank == 0:
195     ref.tofile(filename)
196     MPIBarrierWorld()
197     # step 2 - read
198     data = self.read(filename, ftype)
199     MPIBarrierWorld()
200     # step 3 - write
201     self.write(data, filename) # overwrite is ok
202     MPIBarrierWorld()
203     result = np.fromfile(filename, dtype=self.dtype).reshape(
204     tuple(reversed(adjust(self.NE,ftype))))
205     return result
206    
207     def test_readGrid2D(self):
208     if self.multiplier[0] == 1:
209     self.NE = [self.NX*mpiSize-1, self.NZ*self.multiplier[1]]
210     else:
211     self.NE = [self.NX*mpiSize*self.multiplier[0]-1, self.NZ*self.multiplier[1]]
212     self.domain = Rectangle(n0=self.NE[0], n1=self.NE[1], d0=mpiSize, d1=1)
213     for ftype,fcode in self.fspaces:
214     self.Ndata = [self.NX*mpiSize-1, self.NZ]
215     if ftype==ContinuousFunction:
216     self.Ndata[1] = self.NZ-1
217     # step 1 - generate
218     ref = self.generateUniqueData(ftype)
219     # step 2 & 3
220     with self.assertRaises(RuntimeError):
221     result = self.numpy2Data2Numpy(ref, ftype, fcode)
222     # apply transformations to be able to compare
223     if self.reverse[0]:
224     result = result[...,::-1]
225     if self.reverse[1]:
226     result = result[::-1,:]
227     for i in range(2):
228     ref = np.repeat(ref, self.multiplier[i], axis=1-i)
229    
230     # if domain larger than data: add column(s)/row(s) with fill value
231     fill=np.array(self.fill, dtype=ref.dtype)
232     realNE = adjust(self.NE,ftype)
233     for d in range(2):
234     excess = realNE[d]-ref.shape[1-d]
235     if excess > 0:
236     shape = list(ref.shape)
237     shape[1-d] = excess
238     extra = fill * np.ones(shape)
239     if self.reverse[d]:
240     ref = np.append(extra, ref, axis=1-d)
241     else:
242     ref = np.append(ref, extra, axis=1-d)
243    
244     # step 4 - compare
245     self.assertAlmostEquals(Lsup(ref-result), 0, delta=1e-9,
246     msg="Data doesn't match for "+str(ftype(self.domain)))
247    
248 sshaw 5540 @unittest.skipIf(mpiSize > 1, "3D Multiresolution domains don't support multiprocess yet")
249 sshaw 5528 def test_readGrid3D(self):
250     if self.multiplier[0] == 1:
251     self.NE = [self.NX*mpiSize-1, self.NX*self.multiplier[1], self.NZ*self.multiplier[2]]
252     else:
253     self.NE = [self.NX*mpiSize*self.multiplier[0]-1,
254     self.NX*self.multiplier[1], self.NZ*self.multiplier[2]]
255     self.domain = Brick(n0=self.NE[0], n1=self.NE[1], n2=self.NE[2], d0=mpiSize, d1=1, d2=1)
256     for ftype,fcode in self.fspaces:
257     self.Ndata = [self.NX*mpiSize-1, self.NX, self.NZ]
258     if ftype==ContinuousFunction:
259     self.Ndata[1] = self.NX-1
260     self.Ndata[2] = self.NZ-1
261     # step 1 - generate
262     ref = self.generateUniqueData(ftype)
263     # step 2 & 3
264     with self.assertRaises(RuntimeError):
265     result = self.numpy2Data2Numpy(ref, ftype, fcode)
266     # apply transformations to be able to compare
267     if self.reverse[0]:
268     result = result[...,::-1]
269     if self.reverse[1]:
270     result = result[...,::-1,:]
271     if self.reverse[2]:
272     result = result[::-1,:,:]
273     for i in range(3):
274     ref = np.repeat(ref, self.multiplier[i], axis=2-i)
275    
276     # if domain larger than data: add column(s)/row(s) with fill value
277     fill=np.array(self.fill, dtype=ref.dtype)
278     realNE = adjust(self.NE,ftype)
279     for d in range(3):
280     excess = realNE[d]-ref.shape[2-d]
281     if excess > 0:
282     shape = list(ref.shape)
283     shape[2-d] = excess
284     extra = fill * np.ones(shape)
285     if self.reverse[d]:
286     ref = np.append(extra, ref, axis=2-d)
287     else:
288     ref = np.append(ref, extra, axis=2-d)
289    
290     # step 4 - compare
291     self.assertAlmostEquals(Lsup(ref-result), 0, delta=1e-9,
292     msg="Data doesn't match for "+str(ftype(self.domain)))
293    
294    
295     # The following block tests the reader for different byte orders and data
296     # types with domain-filling data (i.e. multiplier=1, reverse=0 and N=NE)
297    
298     class Test_readBinaryGridRipley_LITTLE_FLOAT32(ReadBinaryGridTestBase):
299     def setUp(self):
300     self.byteorder = BYTEORDER_LITTLE_ENDIAN
301     self.datatype = DATATYPE_FLOAT32
302     self.dtype = "<f4"
303    
304     class Test_readBinaryGridRipley_LITTLE_FLOAT64(ReadBinaryGridTestBase):
305     def setUp(self):
306     self.byteorder = BYTEORDER_LITTLE_ENDIAN
307     self.datatype = DATATYPE_FLOAT64
308     self.dtype = "<f8"
309    
310     class Test_readBinaryGridRipley_LITTLE_INT32(ReadBinaryGridTestBase):
311     def setUp(self):
312     self.byteorder = BYTEORDER_LITTLE_ENDIAN
313     self.datatype = DATATYPE_INT32
314     self.dtype = "<i4"
315    
316     class Test_readBinaryGridRipley_BIG_FLOAT32(ReadBinaryGridTestBase):
317     def setUp(self):
318     self.byteorder = BYTEORDER_BIG_ENDIAN
319     self.datatype = DATATYPE_FLOAT32
320     self.dtype = ">f4"
321    
322     class Test_readBinaryGridRipley_BIG_FLOAT64(ReadBinaryGridTestBase):
323     def setUp(self):
324     self.byteorder = BYTEORDER_BIG_ENDIAN
325     self.datatype = DATATYPE_FLOAT64
326     self.dtype = ">f8"
327    
328     class Test_readBinaryGridRipley_BIG_INT32(ReadBinaryGridTestBase):
329     def setUp(self):
330     self.byteorder = BYTEORDER_BIG_ENDIAN
331     self.datatype = DATATYPE_INT32
332     self.dtype = ">i4"
333    
334     @unittest.skip("reverseX not supported yet")
335     class Test_readBinaryGridRipley_reverseX(ReadBinaryGridTestBase):
336     def setUp(self):
337     self.reverse = [1,0,0]
338    
339     @unittest.skip("reverseY not supported yet")
340     class Test_readBinaryGridRipley_reverseY(ReadBinaryGridTestBase):
341     def setUp(self):
342     self.reverse = [0,1,0]
343    
344     class Test_readBinaryGridRipley_reverseZ(ReadBinaryGridTestBase):
345     def setUp(self):
346     self.reverse = [0,0,1]
347    
348     class Test_readBinaryGridRipley_multiplierX(ReadBinaryGridTestBase):
349     def setUp(self):
350     self.multiplier = [2,1,1]
351    
352     class Test_readBinaryGridRipley_multiplierY(ReadBinaryGridTestBase):
353     def setUp(self):
354     self.multiplier = [1,2,1]
355    
356     class Test_readBinaryGridRipley_multiplierZ(ReadBinaryGridTestBase):
357     def setUp(self):
358     self.multiplier = [1,1,2]
359    
360     class Test_readBinaryGridRipley_multiplierXYZ(ReadBinaryGridTestBase):
361     def setUp(self):
362     self.multiplier = [2,3,4]
363    
364    
365     @unittest.skipIf(getMPISizeWorld() > 1,
366     "Skipping compressed binary grid tests due to element stretching")
367     class Test_readBinaryGridZippedRipley(unittest.TestCase):
368     # constants
369     byteorder = BYTEORDER_NATIVE
370     datatype = DATATYPE_FLOAT64
371    
372     def read(self, filename, FS, expected, zipped = False):
373     first = [0 for i in expected]
374     reverse = [0 for i in expected]
375     scale = [1 for i in expected]
376    
377     if not zipped:
378     return readBinaryGrid(filename, FS, (), 50000,
379     self.byteorder, self.datatype, first, expected, scale, reverse)
380    
381 caltinay 6141 if not HAVE_UNZIP:
382 sshaw 5528 raise unittest.SkipTest("unzip library not available (boost_iostreams)")
383     return ripleycpp._readBinaryGridFromZipped(filename, FS, (), 50000,
384     self.byteorder, self.datatype, first, expected, scale, reverse)
385    
386     def test_readCompressed2D(self):
387     NE = [9, 10]
388     domain = Rectangle(n0=NE[0], n1=NE[1], d1=1)
389     for filename, ftype in [("RectRedF%s.grid.gz", ReducedFunction),
390     ("RectConF%s.grid.gz", ContinuousFunction)]:
391     FS = ftype(domain)
392     filename = os.path.join("ref_data", filename%mpiSize)
393     with self.assertRaises(RuntimeError): #non-parent multidomain
394     unzipped = self.read(filename[:-3], FS, adjust(NE, ftype))
395     zipped = self.read(filename, FS, adjust(NE, ftype), True)
396     self.assertEqual(Lsup(zipped - unzipped), 0, "Data objects don't match for "+str(FS))
397    
398     def test_readCompressed3D(self):
399     NE = [9, 9, 10]
400     domain = Brick(n0=NE[0], n1=NE[1], n2=NE[2], d1=1, d2=1)
401     for filename, ftype in [("BrickRedF%s.grid.gz", ReducedFunction),
402     ("BrickConF%s.grid.gz", ContinuousFunction)]:
403     FS = ftype(domain)
404     filename = os.path.join("ref_data", filename%mpiSize)
405     with self.assertRaises(RuntimeError): #non-parent multidomain
406     unzipped = self.read(filename[:-3], FS, adjust(NE, ftype))
407     zipped = self.read(filename, FS, adjust(NE, ftype), True)
408     self.assertEqual(Lsup(zipped - unzipped), 0, "Data objects don't match for "+str(FS))
409    
410    
411     if __name__ == '__main__':
412     run_tests(__name__, exit_on_failure=True)
413    

  ViewVC Help
Powered by ViewVC 1.1.26