/[escript]/trunk/downunder/py_src/datasources.py
ViewVC logotype

Diff of /trunk/downunder/py_src/datasources.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 3957 by caltinay, Wed Sep 5 23:49:23 2012 UTC revision 4108 by caltinay, Thu Dec 13 06:38:11 2012 UTC
# Line 1  Line 1 
1    
2  ########################################################  ##############################################################################
3  #  #
4  # Copyright (c) 2003-2012 by University of Queensland  # Copyright (c) 2003-2012 by University of Queensland
5  # Earth Systems Science Computational Center (ESSCC)  # http://www.uq.edu.au
 # http://www.uq.edu.au/esscc  
6  #  #
7  # Primary Business: Queensland, Australia  # Primary Business: Queensland, Australia
8  # Licensed under the Open Software License version 3.0  # Licensed under the Open Software License version 3.0
9  # http://www.opensource.org/licenses/osl-3.0.php  # http://www.opensource.org/licenses/osl-3.0.php
10  #  #
11  ########################################################  # Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12    # Development since 2012 by School of Earth Sciences
13    #
14    ##############################################################################
15    
16    """Data readers/providers for inversions"""
17    
18  __copyright__="""Copyright (c) 2003-2012 by University of Queensland  __copyright__="""Copyright (c) 2003-2012 by University of Queensland
19  Earth Systems Science Computational Center (ESSCC)  http://www.uq.edu.au
 http://www.uq.edu.au/esscc  
20  Primary Business: Queensland, Australia"""  Primary Business: Queensland, Australia"""
21  __license__="""Licensed under the Open Software License version 3.0  __license__="""Licensed under the Open Software License version 3.0
22  http://www.opensource.org/licenses/osl-3.0.php"""  http://www.opensource.org/licenses/osl-3.0.php"""
23  __url__="https://launchpad.net/escript-finley"  __url__="https://launchpad.net/escript-finley"
24    
25  __all__ = ['DataSource','UBCDataSource','SyntheticDataSource','SmoothAnomaly']  __all__ = ['simpleBackgroundMagneticField', 'DataSource','ErMapperData','SyntheticFeatureData','SmoothAnomaly']
26    
27  import logging  import logging
28  import numpy as np  import numpy as np
29  from esys.escript import *  from esys.escript import ReducedFunction
30  from esys.escript.linearPDEs import *  from esys.escript.linearPDEs import LinearSinglePDE
31    from esys.escript.util import *
32  import esys.escript.unitsSI as U  import esys.escript.unitsSI as U
33    from esys.ripley import Brick, Rectangle, ripleycpp
34    
35  try:  try:
36      from scipy.io.netcdf import netcdf_file      from scipy.io.netcdf import netcdf_file
37      __all__ += ['NetCDFDataSource']      __all__ += ['NetCdfData']
38  except:  except:
39      pass      pass
40    
# Line 36  def LatLonToUTM(lon, lat, wkt_string=Non Line 42  def LatLonToUTM(lon, lat, wkt_string=Non
42      """      """
43      Converts one or more longitude,latitude pairs to the corresponding x,y      Converts one or more longitude,latitude pairs to the corresponding x,y
44      coordinates in the Universal Transverse Mercator projection.      coordinates in the Universal Transverse Mercator projection.
45      If wkt_string is not given or invalid or the gdal module is not available  
46      to convert the string, then the input values are assumed to be given in the      :note: If the ``pyproj`` module is not installed a warning is printed and
47      Clarke 1866 projection.             the input values are scaled by a constant and returned.
48        :note: If `wkt_string` is not given or invalid or the ``gdal`` module is
49               not available to convert the string, then the input values are
50               assumed to be given in the Clarke 1866 projection.
51    
52        :param lon: longitude value(s)
53        :type lon: `float`, `list`, `tuple`, or ``numpy.array``
54        :param lat: latitude value(s)
55        :type lat: `float`, `list`, `tuple`, or ``numpy.array``
56        :rtype: ``numpy.array``
57      """      """
58    
59      # not really optimal: if pyproj is not installed we return the input      # not really optimal: if pyproj is not installed we return the input
60      # values without modification.      # values scaled by a constant.
61      try:      try:
62          import pyproj          import pyproj
63      except:      except:
64          print("Warning, pyproj not available. Domain extents will be wrong")          print("Warning, pyproj not available. Domain extents will be wrong")
65          return lon,lat          return np.array(lon)*1000., np.array(lat)*1000.
66    
67      # determine UTM zone from the input data      # determine UTM zone from the input data
68      zone=int(np.median((np.floor((np.array(lon) + 180)/6) + 1) % 60))      zone=int(np.median((np.floor((np.array(lon) + 180)/6) + 1) % 60))
# Line 58  def LatLonToUTM(lon, lat, wkt_string=Non Line 73  def LatLonToUTM(lon, lat, wkt_string=Non
73          p_src = pyproj.Proj(srs.ExportToProj4())          p_src = pyproj.Proj(srs.ExportToProj4())
74      except:      except:
75          p_src = pyproj.Proj('+proj=longlat +ellps=clrk66 +no_defs')          p_src = pyproj.Proj('+proj=longlat +ellps=clrk66 +no_defs')
76      p_dest = pyproj.Proj(proj='utm', zone=zone) # ellps?      # we assume southern hemisphere here
77        p_dest = pyproj.Proj('+proj=utm +zone=%d +south +units=m'%zone)
78      x,y=pyproj.transform(p_src, p_dest, lon, lat)      x,y=pyproj.transform(p_src, p_dest, lon, lat)
79      return x,y      return x,y
80    
81    def simpleBackgroundMagneticField(latitude, longitude=0.):
82            theta = (90-latitude)/180.*np.pi
83            B_0=U.Mu_0  * U.Magnetic_Dipole_Moment_Earth / (4 * np.pi *  U.R_Earth**3)
84            B_theta= B_0 * sin(theta)
85            B_r= 2 * B_0 * cos(theta)
86            return B_r, B_theta, 0.
87    
88  class DataSource(object):  class DataSource(object):
89      """      """
90      A class that provides survey data for the inversion process.      A class that provides survey data for the inversion process.
91        This is an abstract base class that implements common functionality.
92        Methods to be overwritten by subclasses are marked as such.
93        This class assumes 2D data which is mapped to a slice of a 3D domain.
94        For other setups override the methods as required.
95      """      """
96      # this is currently specific to gravity inversion and should be generalised  
97        GRAVITY, MAGNETIC = list(range(2))
98    
99      def __init__(self):      def __init__(self):
100          """          """
101            Constructor. Sets some defaults and initializes logger.
102          """          """
         self._domain=None  
         self._pad_l=0.1  
         self._pad_h=0.1  
103          self.logger = logging.getLogger('inv.%s'%self.__class__.__name__)          self.logger = logging.getLogger('inv.%s'%self.__class__.__name__)
104            self.__subsampling_factor=1
105            self.__background_magnetic_field=None
106    
107      def _addPadding(self, pad_l, pad_h, NE, l, origin):      def getDataExtents(self):
108          """          """
109          Helper method that computes new number of elements, length and origin          returns a tuple of tuples ``( (x0, y0), (nx, ny), (dx, dy) )``, where
110          after adding padding to the input values.  
111            - ``x0``, ``y0`` = coordinates of data origin
112            - ``nx``, ``ny`` = number of data points in x and y
113            - ``dx``, ``dy`` = spacing of data points in x and y
114    
115            This method must be implemented in subclasses.
116          """          """
117          DIM=len(NE)          raise NotImplementedError
         frac=[0.]*DIM  
         # padding is applied to each side so multiply by 2 to get the total  
         # amount of padding per dimension  
         if pad_l>0 and pad_l<1:  
             for i in xrange(DIM-1):  
                 frac[i]=2*pad_l  
         elif pad_l>=1:  
             for i in xrange(DIM-1):  
                 frac[i]=2*pad_l/float(NE[i])  
         if pad_h>0 and pad_h<1:  
             frac[DIM-1]=2*pad_h  
         elif pad_h>=1:  
             frac[DIM-1]=2*pad_h/(float(NE[DIM-1]))  
         # calculate new number of elements  
         NE_new=[int(NE[i]*(1+frac[i])) for i in xrange(DIM)]  
         NEdiff=[NE_new[i]-NE[i] for i in xrange(DIM)]  
         spacing=[l[i]/NE[i] for i in xrange(DIM)]  
         l_new=[NE_new[i]*spacing[i] for i in xrange(DIM)]  
         origin_new=[origin[i]-NEdiff[i]/2.*spacing[i] for i in xrange(DIM)]  
         return NE_new, l_new, origin_new  
   
     def _interpolateOnDomain(self, data, shape, origin, spacing, length):  
         """  
         Helper method that interpolates data arrays onto the domain.  
         """  
         dim=len(shape)  
         arrays=np.zeros(((len(data[0])-dim),)+tuple(shape))  
         for entry in data:  
             index=()  
             for i in range(dim):  
                 index=((entry[i]-origin[i])/spacing[i],)+index  
             for i in range(arrays.shape[0]):  
                 arrays[i][index]=entry[dim+i]  
         dom=self.getDomain()  
         x=dom.getX()  
         delta=[length[i]/(shape[dim-i-1]-1) for i in xrange(dim)]  
         realorigin=[inf(x[i]) for i in xrange(dim)]  
         res=[]  
         for i in range(arrays.shape[0]):  
             res.append(interpolateTable(arrays[i], x[:dim], realorigin, delta, 1e9))  
         return res  
   
     def setPadding(self, pad_l=0.1, pad_h=0.1):  
         """  
         Sets the amount of padding around the dataset. If pad_l/pad_h is >=1  
         they are treated as number of elements to be added to the domain.  
         If 0 < pad_l;pad_h < 1, the padding amount is relative.  
         """  
         self._pad_l=pad_l  
         self._pad_h=pad_h  
   
     def getDomain(self):  
         """  
         Returns a domain that spans the data area plus padding.  
         """  
         if self._domain is None:  
             self._domain=self._createDomain(self._pad_l, self._pad_h)  
         return self._domain  
118    
119      def getDensityMask(self):      def getDataType(self):
120          """          """
121          Returns the density mask data object, where mask has value 1 on the          Returns the type of survey data managed by this source.
122          padding area, 0 elsewhere.          Subclasses must return `GRAVITY` or `MAGNETIC` as appropriate.
123          """          """
124          raise NotImplementedError          raise NotImplementedError
125    
126      def getGravityAndStdDev(self):      def getSurveyData(self, domain, origin, NE, spacing):
127          """          """
128          Returns the gravity anomaly and standard deviation data objects as a          This method is called by the `DomainBuilder` to retrieve the survey
129          tuple.          data as `Data` objects on the given domain.
130            Subclasses should return one or more `Data` objects with survey data
131            interpolated on the given ripley domain. The exact return type
132            depends on the type of data.
133    
134            :param domain: the escript domain to use
135            :type domain: `esys.escript.Domain`
136            :param origin: the origin coordinates of the domain
137            :type origin: ``tuple`` or ``list``
138            :param NE: the number of domain elements in each dimension
139            :type NE: ``tuple`` or ``list``
140            :param spacing: the cell sizes (node spacing) in the domain
141            :type spacing: ``tuple`` or ``list``
142          """          """
143          raise NotImplementedError          raise NotImplementedError
144    
145      def _createDomain(self, padding_l, padding_h):      def setSubsamplingFactor(self, f):
146          """          """
147          creates and returns an escript domain that spans the entire area of          Sets the data subsampling factor (default=1).
148          available data plus a buffer zone.          The factor is applied in all dimensions. For example a 2D dataset
149            with 300 x 150 data points will be reduced to 150 x 75 when a
150            subsampling factor of 2 is used.
151            This becomes important when adding data of varying resolution to
152            a `DomainBuilder`.
153          """          """
154          raise NotImplementedError          self.__subsampling_factor=f
155    
156        def getSubsamplingFactor(self):
157            """
158            Returns the subsampling factor that was set via `setSubsamplingFactor`
159            (see there).
160            """
161            return self.__subsampling_factor
162    
163    
164  ##############################################################################  ##############################################################################
165  class UBCDataSource(DataSource):  class ErMapperData(DataSource):
166      def __init__(self, domainclass, meshfile, gravfile, topofile=None):      """
167          super(UBCDataSource,self).__init__()      Data Source for ER Mapper raster data.
168          self._meshfile=meshfile      Note that this class only accepts a very specific type of ER Mapper data
169          self._gravfile=gravfile      input and will raise an exception if other data is found.
170          self._topofile=topofile      """
171          self._domainclass=domainclass      def __init__(self, datatype, headerfile, datafile=None, altitude=0.):
172          self._readMesh()          """
173            :param datatype: type of data, must be `GRAVITY` or `MAGNETIC`
174      def getDensityMask(self):          :type datatype: ``int``
175          #topodata=self._readTopography()          :param headerfile: ER Mapper header file (usually ends in .ers)
176          #shape=[self.NE[1]+1, self.NE[0]+1]          :type headerfile: ``str``
177          #mask=self._interpolateOnDomain(topodata, shape, self._origin, self._spacing, self._meshlen)          :param datafile: ER Mapper binary data file name. If not supplied the
178          #mask=wherePositive(self.getDomain().getX()[2]-mask[0])                           name of the header file without '.ers' is assumed
179          return self._mask          :type datafile: ``str``
180            :param altitude: altitude of measurements above ground in meters
181      def getGravityAndStdDev(self):          :type altitude: ``float``
182          gravlist=self._readGravity() # x,y,z,g,s          """
183          shape=[self.NE[2]+1, self.NE[1]+1, self.NE[0]+1]          super(ErMapperData,self).__init__()
184          g_and_sigma=self._interpolateOnDomain(gravlist, shape, self._origin, self._spacing, self._meshlen)          self.__headerfile=headerfile
185          return g_and_sigma[0]*[0,0,1], g_and_sigma[1]          if datafile is None:
186                self.__datafile=headerfile[:-4]
187      def _readMesh(self):          else:
188          meshdata=open(self._meshfile).readlines()              self.__datafile=datafile
189          numDataPoints=meshdata[0].split()          self.__altitude=altitude
190          origin=meshdata[1].split()          self.__datatype=datatype
191          self._numDataPoints=[int(x) for x in numDataPoints]          self.__readHeader()
192          self._origin=[float(x) for x in origin]  
193          self._spacing=[float(X.split('*')[1]) for X in meshdata[2:]]      def __readHeader(self):
194          # vertical data is upside down          self.logger.debug("Checking Data Source: %s (header: %s)"%(self.__datafile, self.__headerfile))
195          self._origin[2]-=(self._numDataPoints[2]-1)*self._spacing[2]          metadata=open(self.__headerfile, 'r').readlines()
196            # parse metadata
197      def _createDomain(self, padding_l, padding_h):          start=-1
198          NE=[self._numDataPoints[i]-1 for i in xrange(3)]          for i in range(len(metadata)):
199          l=[NE[i]*self._spacing[i] for i in xrange(3)]              if metadata[i].strip() == 'DatasetHeader Begin':
200          NE_new, l_new, origin_new = self._addPadding(padding_l, padding_h, \                  start=i+1
201                  NE, l, self._origin)          if start==-1:
202                raise RuntimeError('Invalid ERS file (DatasetHeader not found)')
203          self._meshlen=l_new  
204          self.NE=NE_new          md_dict={}
205          self._origin=origin_new          section=[]
206          lo=[(self._origin[i], self._origin[i]+l_new[i]) for i in xrange(3)]          for i in range(start, len(metadata)):
207          NEdiff=[NE_new[i]-NE[i] for i in xrange(3)]              line=metadata[i]
208                if line[-6:].strip() == 'Begin':
209                    section.append(line[:-6].strip())
210                elif line[-4:].strip() == 'End':
211                    if len(section)>0:
212                        section.pop()
213                else:
214                    vals=line.split('=')
215                    if len(vals)==2:
216                        key = vals[0].strip()
217                        value = vals[1].strip()
218                        fullkey='.'.join(section+[key])
219                        md_dict[fullkey]=value
220    
221          try:          try:
222              dom=self._domainclass(*self.NE, l0=lo[0], l1=lo[1], l2=lo[2])              if md_dict['RasterInfo.CellType'] != 'IEEE4ByteReal':
223              x=dom.getX()-[self._origin[i]+NEdiff[i]/2.*self._spacing[i] for i in xrange(3)]                  raise RuntimeError('Unsupported data type '+md_dict['RasterInfo.CellType'])
224              mask=wherePositive(dom.getX()[2])          except KeyError:
225                self.logger.warn("Cell type not specified. Assuming IEEE4ByteReal.")
         except TypeError:  
             dom=self._domainclass(*self.NE, l0=l_new[0], l1=l_new[1], l2=l_new[2])  
             x=dom.getX()-[NEdiff[i]/2.*self._spacing[i] for i in xrange(3)]  
             mask=wherePositive(x[2]+self._origin[2])  
   
         M=2 # do not constrain bottom  
         #M=3 # constrain bottom  
         for i in xrange(M):  
             mask=mask + whereNegative(x[i]) + \  
                     wherePositive(x[i]-l_new[i]+NEdiff[i]*self._spacing[i])  
         self._mask=wherePositive(mask)  
   
         self.logger.debug("Domain size: %d x %d x %d elements"%(self.NE[0],self.NE[1],self.NE[2]))  
         self.logger.debug("     length: %g x %g x %g"%(l_new[0],l_new[1],l_new[2]))  
         self.logger.debug("     origin: %g x %g x %g"%(origin_new[0],origin_new[1],origin_new[2]))  
   
         return dom  
   
     def _readTopography(self):  
         f=open(self._topofile)  
         n=int(f.readline())  
         topodata=np.zeros((n,3))  
         for i in xrange(n):  
             x=f.readline().split()  
             x[0]=float(x[0])  
             x[1]=float(x[1])  
             x[2]=float(x[2])  
             topodata[i]=x  
         f.close()  
         return topodata  
226    
227      def _readGravity(self):          try:
228          f=open(self._gravfile)              NX = int(md_dict['RasterInfo.NrOfCellsPerLine'])
229          n=int(f.readline())              NY = int(md_dict['RasterInfo.NrOfLines'])
230          gravdata=np.zeros((n,5))          except:
231          for i in xrange(n):              raise RuntimeError("Could not determine extents of data")
             x=f.readline().split()  
             x[0]=float(x[0]) # x  
             x[1]=float(x[1]) # y  
             x[2]=float(x[2]) # z  
             x[3]=float(x[3]) # gravity anomaly in mGal  
             x[4]=float(x[4]) # stddev  
             # convert gravity anomaly units to m/s^2 and rescale error  
             x[3]*=-1e-5  
             x[4]*=1e-5  
             gravdata[i]=x  
         f.close()  
         return gravdata  
232    
233  ##############################################################################          try:
234  class NetCDFDataSource(DataSource):              maskval = float(md_dict['RasterInfo.NullCellValue'])
235      def __init__(self, domainclass, gravfile, topofile=None, vertical_extents=(-40000,10000,26), alt_of_data=1.):          except:
236                maskval = 99999
237    
238            try:
239                spacingX = float(md_dict['RasterInfo.CellInfo.Xdimension'])
240                spacingY = float(md_dict['RasterInfo.CellInfo.Ydimension'])
241            except:
242                raise RuntimeError("Could not determine cell dimensions")
243    
244            try:
245                if md_dict['CoordinateSpace.CoordinateType']=='EN':
246                    originX = float(md_dict['RasterInfo.RegistrationCoord.Eastings'])
247                    originY = float(md_dict['RasterInfo.RegistrationCoord.Northings'])
248                elif md_dict['CoordinateSpace.CoordinateType']=='LL':
249                    originX = float(md_dict['RasterInfo.RegistrationCoord.Longitude'])
250                    originY = float(md_dict['RasterInfo.RegistrationCoord.Latitude'])
251                else:
252                    raise RuntimeError("Unknown CoordinateType")
253            except:
254                self.logger.warn("Could not determine coordinate origin. Setting to (0.0, 0.0)")
255                originX,originY = 0.0, 0.0
256    
257            if 'GEODETIC' in md_dict['CoordinateSpace.Projection']:
258                # it appears we have lat/lon coordinates so need to convert
259                # origin and spacing. Try using gdal to get the wkt if available:
260                try:
261                    from osgeo import gdal
262                    ds=gdal.Open(self.__headerfile)
263                    wkt=ds.GetProjection()
264                except:
265                    wkt='GEOGCS["GEOCENTRIC DATUM of AUSTRALIA",DATUM["GDA94",SPHEROID["GRS80",6378137,298.257222101]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]]'
266                    self.logger.warn('GDAL not available or file read error, assuming GDA94 data')
267                originX_UTM,originY_UTM=LatLonToUTM(originX, originY, wkt)
268                op1X,op1Y=LatLonToUTM(originX+spacingX, originY+spacingY, wkt)
269                # we are rounding to avoid interpolation issues
270                spacingX=np.round(op1X-originX_UTM)
271                spacingY=np.round(op1Y-originY_UTM)
272                originX=np.round(originX_UTM)
273                originY=np.round(originY_UTM)
274    
275            # data sets have origin in top-left corner so y runs top-down
276            self.__dataorigin=[originX, originY]
277            originY-=(NY-1)*spacingY
278            self.__delta = [spacingX, spacingY]
279            self.__maskval = maskval
280            self.__nPts = [NX, NY]
281            self.__origin = [originX, originY]
282            if self.__datatype == self.GRAVITY:
283                self.logger.info("Assuming gravity data scale is 1e-6 m/s^2.")
284                self.__scalefactor = 1e-6
285            else:
286                self.logger.info("Assuming magnetic data units are 'nT'.")
287                self.__scalefactor = 1e-9
288    
289        def getDataExtents(self):
290          """          """
291          vertical_extents - (alt_min, alt_max, num_elements)          returns ( (x0, y0), (nx, ny), (dx, dy) )
         alt_of_data - altitude of measurements  
292          """          """
293          super(NetCDFDataSource,self).__init__()          return (list(self.__origin), list(self.__nPts), list(self.__delta))
         self._topofile=topofile  
         self._gravfile=gravfile  
         self._domainclass=domainclass  
         self._determineExtents(vertical_extents)  
         self._altOfData=alt_of_data  
   
     def getDensityMask(self):  
         #topodata=self._readTopography()  
         #shape=self._numDataPoints[1::-1]  
         #mask=self._interpolateOnDomain(topodata, shape, self._origin, self._spacing, self._meshlen)  
         #mask=wherePositive(self.getDomain().getX()[2]-mask[0])  
         #rho=fill*(1.-mask) + RHO_AIR*mask  
         return self._mask  
   
     def getGravityAndStdDev(self):  
         gravlist=self._readGravity() # x,y,z,g,s  
         shape=[self.NE[2]+1, self.NE[1]+1, self.NE[0]+1]  
         g_and_sigma=self._interpolateOnDomain(gravlist, shape, self._origin, self._spacing, self._meshlen)  
         return g_and_sigma[0]*[0,0,1], g_and_sigma[1]  
294    
295      def _determineExtents(self, ve):      def getDataType(self):
296          f=netcdf_file(self._gravfile, 'r')          return self.__datatype
297    
298        def getSurveyData(self, domain, origin, NE, spacing):
299            nValues=self.__nPts
300            # determine base location of this dataset within the domain
301            first=[int((self.__origin[i]-origin[i])/spacing[i]) for i in range(len(self.__nPts))]
302            if domain.getDim()==3:
303                first.append(int((self.__altitude-origin[2])/spacing[2]))
304                nValues=nValues+[1]
305    
306            data=ripleycpp._readBinaryGrid(self.__datafile,
307                    ReducedFunction(domain),
308                    first, nValues, (), self.__maskval)
309            sigma = whereNonZero(data-self.__maskval)
310            data = data*self.__scalefactor
311            sigma = sigma * 2. * self.__scalefactor
312            return data, sigma
313    
314    
315    ##############################################################################
316    class NetCdfData(DataSource):
317        """
318        Data Source for gridded netCDF data that use CF/COARDS conventions.
319        """
320        def __init__(self, datatype, filename, altitude=0.):
321            """
322            :param filename: file name for survey data in netCDF format
323            :type filename: ``str``
324            :param datatype: type of data, must be `GRAVITY` or `MAGNETIC`
325            :type datatype: ``int``
326            :param altitude: altitude of measurements in meters
327            :type altitude: ``float``
328            """
329            super(NetCdfData,self).__init__()
330            self.__filename=filename
331            if not datatype in [self.GRAVITY,self.MAGNETIC]:
332                raise ValueError("Invalid value for datatype parameter")
333            self.__datatype=datatype
334            self.__altitude=altitude
335            self.__readMetadata()
336    
337        def __readMetadata(self):
338            self.logger.debug("Checking Data Source: %s"%self.__filename)
339            f=netcdf_file(self.__filename, 'r')
340          NX=0          NX=0
341          for n in ['lon','longitude','x']:          for n in ['lon','longitude','x']:
342              if n in f.dimensions:              if n in f.dimensions:
# Line 324  class NetCDFDataSource(DataSource): Line 370  class NetCDFDataSource(DataSource):
370          if lat_name is None:          if lat_name is None:
371              raise RuntimeError("Could not determine latitude variable")              raise RuntimeError("Could not determine latitude variable")
372    
373          # try to figure out gravity variable name          # try to figure out data variable name
374          grav_name=None          data_name=None
375          if len(f.variables)==1:          if len(f.variables)==1:
376              grav_name=f.variables.keys()[0]              data_name=f.variables.keys()[0]
377          else:          else:
378              for n in f.variables.keys():              for n in f.variables.keys():
379                  dims=f.variables[n].dimensions                  dims=f.variables[n].dimensions
380                  if (lat_name in dims) and (lon_name in dims):                  if (lat_name in dims) and (lon_name in dims):
381                      grav_name=n                      data_name=n
382                      break                      break
383          if grav_name is None:          if data_name is None:
384              raise RuntimeError("Could not determine gravity variable")              raise RuntimeError("Could not determine data variable")
385    
386            # try to determine value for unused data
387            if hasattr(f.variables[data_name], 'missing_value'):
388                maskval = float(f.variables[data_name].missing_value)
389            elif hasattr(f.variables[data_name], '_FillValue'):
390                maskval = float(f.variables[data_name]._FillValue)
391            else:
392                self.logger.debug("missing_value attribute not found, using default.")
393                maskval = 99999
394    
395            # try to determine units of data - this is disabled for now
396            #if hasattr(f.variables[data_name], 'units'):
397            #   units=f.variables[data_name].units
398            if self.__datatype == self.GRAVITY:
399                self.logger.info("Assuming gravity data scale is 1e-6 m/s^2.")
400                self.__scalefactor = 1e-6
401            else:
402                self.logger.info("Assuming magnetic data units are 'nT'.")
403                self.__scalefactor = 1e-9
404    
405          # see if there is a wkt string to convert coordinates          # see if there is a wkt string to convert coordinates
406          try:          try:
407              wkt_string=f.variables[grav_name].esri_pe_string              wkt_string=f.variables[data_name].esri_pe_string
408          except:          except:
409              wkt_string=None              wkt_string=None
410    
411          origin=[0.,0.,ve[0]]          # we don't trust actual_range & geospatial_lon_min/max since subset
412          lengths=[100000.,100000.,ve[1]-ve[0]]          # data does not seem to have these fields updated.
413            # Getting min/max from the arrays is obviously not very efficient but..
414          try:          #lon_range=longitude.actual_range
415              lon_range=longitude.actual_range          #lat_range=latitude.actual_range
416              lat_range=latitude.actual_range          #lon_range=[f.geospatial_lon_min,f.geospatial_lon_max]
417            #lat_range=[f.geospatial_lat_min,f.geospatial_lat_max]
418            lon_range=longitude.data.min(),longitude.data.max()
419            lat_range=latitude.data.min(),latitude.data.max()
420            if lon_range[1]<180:
421              lon_range,lat_range=LatLonToUTM(lon_range, lat_range, wkt_string)              lon_range,lat_range=LatLonToUTM(lon_range, lat_range, wkt_string)
422              origin[:2]=lon_range[0],lat_range[0]          lengths=[lon_range[1]-lon_range[0], lat_range[1]-lat_range[0]]
             lengths[:2]=[lon_range[1]-lon_range[0], lat_range[1]-lat_range[0]]  
         except:  
             try:  
                 lon_range=[f.geospatial_lon_min,f.geospatial_lon_max]  
                 lat_range=[f.geospatial_lat_min,f.geospatial_lat_max]  
                 lon_range,lat_range=LatLonToUTM(lon_range, lat_range, wkt_string)  
                 origin[:2]=lon_range[0],lat_range[0]  
                 lengths[:2]=[lon_range[1]-lon_range[0], lat_range[1]-lat_range[0]]  
             except:  
                 pass  
   
423          f.close()          f.close()
424    
425          self._numDataPoints=[NX, NY, ve[2]]          self.__nPts=[NX, NY]
426          self._origin=origin          self.__origin=[lon_range[0],lat_range[0]]
427          self._spacing=[np.round(lengths[i]/(self._numDataPoints[i]-1)) for i in xrange(3)]          # we are rounding to avoid interpolation issues
428          self._meshlen=[self._numDataPoints[i]*self._spacing[i] for i in xrange(3)]          self.__delta=[np.round(lengths[i]/self.__nPts[i]) for i in range(2)]
429          self._wkt_string=wkt_string          #self.__wkt_string=wkt_string
430          self._lon=lon_name          #self.__lon_name=lon_name
431          self._lat=lat_name          #self.__lat_name=lat_name
432          self._grv=grav_name          self.__data_name=data_name
433            self.__maskval=maskval
434      def _createDomain(self, padding_l, padding_h):  
435          NE=[self._numDataPoints[i]-1 for i in xrange(3)]      def getDataExtents(self):
436          l=self._meshlen          """
437          NE_new, l_new, origin_new = self._addPadding(padding_l, padding_h, \          returns ( (x0, y0), (nx, ny), (dx, dy) )
438                  NE, l, self._origin)          """
439            return (list(self.__origin), list(self.__nPts), list(self.__delta))
440          self._meshlen=l_new  
441          self.NE=NE_new      def getDataType(self):
442          self._origin=origin_new          return self.__datatype
443          lo=[(self._origin[i], self._origin[i]+l_new[i]) for i in xrange(3)]  
444          NEdiff=[NE_new[i]-NE[i] for i in xrange(3)]      def getSurveyData(self, domain, origin, NE, spacing):
445          try:          nValues=self.__nPts
446              dom=self._domainclass(*self.NE, l0=lo[0], l1=lo[1], l2=lo[2])          # determine base location of this dataset within the domain
447              # ripley may adjust NE and length          first=[int((self.__origin[i]-origin[i])/spacing[i]) for i in range(len(self.__nPts))]
448              self._meshlen=[sup(dom.getX()[i])-inf(dom.getX()[i]) for i in xrange(3)]          if domain.getDim()==3:
449              self.NE=[self._meshlen[i]/self._spacing[i] for i in xrange(3)]              first.append(int((self.__altitude-origin[2])/spacing[2]))
450              x=dom.getX()-[self._origin[i]+NEdiff[i]/2.*self._spacing[i] for i in xrange(3)]              nValues=nValues+[1]
451              mask=wherePositive(dom.getX()[2])  
452            data=ripleycpp._readNcGrid(self.__filename, self.__data_name,
453          except TypeError:                    ReducedFunction(domain), first, nValues, (), self.__maskval)
454              dom=self._domainclass(*self.NE, l0=l_new[0], l1=l_new[1], l2=l_new[2])          sigma=whereNonZero(data-self.__maskval)
455              x=dom.getX()-[NEdiff[i]/2.*self._spacing[i] for i in xrange(3)]          data=data*self.__scalefactor
456              mask=wherePositive(x[2]+self._origin[2])          sigma=sigma * 2. * self.__scalefactor
457            return data, sigma
         M=2 # do not constrain bottom  
         #M=3 # constrain bottom  
         for i in xrange(M):  
             mask=mask + whereNegative(x[i]) + \  
                     wherePositive(x[i]-l_new[i]+NEdiff[i]*self._spacing[i])  
         self._mask=wherePositive(mask)  
   
         self.logger.debug("Domain size: %d x %d x %d elements"%(self.NE[0],self.NE[1],self.NE[2]))  
         self.logger.debug("     length: %s x %s x %s"%(self._meshlen[0],self._meshlen[1],self._meshlen[2]))  
         self.logger.debug("     origin: %s x %s x %s"%(self._origin[0],self._origin[1],self._origin[2]))  
   
         return dom  
   
     def _readTopography(self):  
         f=netcdf_file(self._topofile, 'r')  
         lon=None  
         for n in ['lon','longitude']:  
             if n in f.variables:  
                 lon=f.variables[n][:]  
                 break  
         if lon is None:  
             raise RuntimeError("Could not determine longitude variable")  
         lat=None  
         for n in ['lat','latitude']:  
             if n in f.variables:  
                 lat=f.variables[n][:]  
                 break  
         if lat is None:  
             raise RuntimeError("Could not determine latitude variable")  
         alt=None  
         for n in ['altitude','alt']:  
             if n in f.variables:  
                 alt=f.variables[n][:]  
                 break  
         if alt is None:  
             raise RuntimeError("Could not determine altitude variable")  
458    
         topodata=np.column_stack((lon,lat,alt))  
         f.close()  
         return topodata  
459    
460      def _readGravity(self):  ##############################################################################
461          f=netcdf_file(self._gravfile, 'r')  class SourceFeature(object):
462          lon=f.variables[self._lon][:]      """
463          lat=f.variables[self._lat][:]      A feature adds a density distribution to (parts of) a domain of a synthetic
464          lon,lat=np.meshgrid(lon,lat)      data source, for example a layer of a specific rock type or a simulated
465          lon,lat=LatLonToUTM(lon, lat, self._wkt_string)      ore body.
466          grav=f.variables[self._grv][:]      """
467          f.close()      def getValue(self):
468          lon=lon.flatten()          """
469          lat=lat.flatten()          Returns the value for the area covered by mask. It can be constant
470          grav=grav.flatten()          or a data object with spatial dependency.
471          alt=self._altOfData*np.ones(grav.shape)          """
472          # error value is an assumption          raise NotImplementedError
473          try:      def getMask(self, x):
474              missing=grav.missing_value          """
475              err=np.where(grav==missing, 20.0, 0.0)          Returns the mask of the area of interest for this feature. That is,
476          except:          mask is non-zero where the density returned by getDensity() should be
477              err=20.0*np.ones(lon.shape)          applied, zero elsewhere.
478          # convert units          """
479          err=1e-6*err          raise NotImplementedError
         grav=1e-6*grav  
         gravdata=np.column_stack((lon,lat,alt,grav,err))  
         return gravdata  
480    
481  class SmoothAnomaly(object):  class SmoothAnomaly(SourceFeature):
482      def __init__(self, lx, ly, lz, x, y, depth, rho_inner, rho_outer):      def __init__(self, lx, ly, lz, x, y, depth, v_inner=None, v_outer=None):
483          self.x=x          self.x=x
484          self.y=y          self.y=y
485          self.lx=lx          self.lx=lx
486          self.ly=ly          self.ly=ly
487          self.lz=lz          self.lz=lz
488          self.depth=depth          self.depth=depth
489          self.rho_inner=rho_inner          self.v_inner=v_inner
490          self.rho_outer=rho_outer          self.v_outer=v_outer
491          self.rho=None          self.value=None
492            self.mask=None
493    
494        def getValue(self,x):
495            if self.value is None:
496                if self.v_outer is None or self.v_inner is None:
497                    self.value=0
498                else:
499                    DIM=x.getDomain().getDim()
500                    alpha=-log(abs(self.v_outer/self.v_inner))*4
501                    value=exp(-alpha*((x[0]-self.x)/self.lx)**2)
502                    value=value*exp(-alpha*((x[DIM-1]+self.depth)/self.lz)**2)
503                    self.value=maximum(abs(self.v_outer), abs(self.v_inner*value))
504                    if self.v_inner<0: self.value=-self.value
505    
506            return self.value
507    
508      def getMask(self, x):      def getMask(self, x):
509          DIM=x.getDomain().getDim()          DIM=x.getDomain().getDim()
510          m=whereNonNegative(x[DIM-1]-(sup(x[DIM-1])-self.depth-self.lz/2)) * whereNonPositive(x[DIM-1]-(sup(x[DIM-1])-self.depth+self.lz/2)) \          m=whereNonNegative(x[DIM-1]+self.depth+self.lz/2) * whereNonPositive(x[DIM-1]+self.depth-self.lz/2) \
511              *whereNonNegative(x[0]-(self.x-self.lx/2)) * whereNonPositive(x[0]-(self.x+self.lx/2))              *whereNonNegative(x[0]-(self.x-self.lx/2)) * whereNonPositive(x[0]-(self.x+self.lx/2))
512          if DIM>2:          if DIM>2:
513              m*=whereNonNegative(x[1]-(self.y-self.ly/2)) * whereNonPositive(x[1]-(self.y+self.ly/2))              m*=whereNonNegative(x[1]-(self.y-self.ly/2)) * whereNonPositive(x[1]-(self.y+self.ly/2))
514          if self.rho is None:          self.mask = m
515              alpha=-log(abs(self.rho_outer/self.rho_inner))*4          return m
516              rho=exp(-alpha*((x[0]-self.x)/self.lx)**2)  
517              rho=rho*exp(-alpha*((x[DIM-1]-(sup(x[DIM-1])-self.depth))/self.lz)**2)  ##############################################################################
518              self.rho=maximum(abs(self.rho_outer), abs(self.rho_inner*rho))  class SyntheticFeatureData(DataSource):
519              if self.rho_inner<0: self.rho=-self.rho      def __init__(self, datatype, DIM, NE, l, features, B_b=None):
520          return m              super(SyntheticFeatureData,self).__init__()
521            if not datatype in [self.GRAVITY,self.MAGNETIC]:
522  class SyntheticDataSource(DataSource):              raise ValueError("Invalid value for datatype parameter")
523      def __init__(self, DIM, NE, l, h, features):          self.__datatype = datatype
         super(SyntheticDataSource,self).__init__()  
524          self._features = features          self._features = features
525            self.__origin = [0]*(DIM-1)
526            self.__nPts = [NE]*(DIM-1)
527            self.__delta = [float(l)/NE]*(DIM-1)
528            self.__B_b =None
529          self.DIM=DIM          self.DIM=DIM
530          self.NE=NE          self.NE=NE
531          self.l=l          self.l=l
532          self.h=h          # this is for Cartesian (FIXME ?)
533            if datatype  ==  self.MAGNETIC:
534                if self.DIM<3:
535                   self.__B_b =  np.array([-B_b[2],  -B_b[0]])
536                else:
537                   self.__B_b = ([-B_b[1],  -B_b[2],  -B_b[0]])
538    
539      def _createDomain(self, padding_l, padding_h):      def getDataExtents(self):
540          NE_H=self.NE          return (list(self.__origin), list(self.__nPts), list(self.__delta))
         NE_L=int((self.l/self.h)*NE_H+0.5)  
         l=[self.l]*(self.DIM-1)+[self.h]  
         NE=[NE_L]*(self.DIM-1)+[NE_H]  
         origin=[0.]*self.DIM  
         NE_new, l_new, origin_new = self._addPadding(padding_l, padding_h, \  
                 NE, l, origin)  
   
         self.NE=NE_new  
         self.l=l_new[0]  
         self.h=l_new[self.DIM-1]  
   
         if self.DIM==2:  
             from esys.finley import Rectangle  
             dom = Rectangle(n0=NE_new[0], n1=NE_new[1], l0=l_new[0], l1=l_new[1])  
             self._x = dom.getX() + origin_new  
             self.logger.debug("Domain size: %d x %d elements"%(NE_new[0], NE_new[1]))  
             self.logger.debug("     length: %g x %g"%(l_new[0],l_new[1]))  
             self.logger.debug("     origin: %g x %g"%(origin_new[0],origin_new[1]))  
         else:    
             from esys.finley import Brick  
             dom = Brick(n0=NE_new[0], n1=NE_new[1], n2=NE_new[2], l0=l_new[0], l1=l_new[1], l2=l_new[2])  
             self._x = dom.getX() + origin_new  
             self.logger.debug("Domain size: %d x %d x %d elements"%(self.NE[0],self.NE[1],self.NE[2]))  
             self.logger.debug("     length: %g x %g x %g"%(l_new[0],l_new[1],l_new[2]))  
             self.logger.debug("     origin: %g x %g x %g"%(origin_new[0],origin_new[1],origin_new[2]))  
   
         dz=l_new[self.DIM-1]/NE_new[self.DIM-1]  
         self._g_mask=wherePositive(dom.getX()[0]-origin_new[0]) \  
                 * whereNegative(dom.getX()[0]-(l_new[0]-origin_new[0])) \  
                 * whereNonNegative(dom.getX()[self.DIM-1]-(l_new[self.DIM-1]+origin_new[self.DIM-1])) \  
                 * whereNonPositive(dom.getX()[self.DIM-1]-(l_new[self.DIM-1]+(origin_new[self.DIM-1]+dz)))  
         self._mask=whereNegative(self._x[self.DIM-1]) + \  
                 wherePositive(self._x[self.DIM-1]-l[self.DIM-1])  
         for i in xrange(self.DIM-1):  
             self._mask=self._mask + whereNegative(self._x[i]) + \  
                     wherePositive(self._x[i]-l[i])  
         self._mask=wherePositive(self._mask)  
   
         rho_ref=0.  
         for f in self._features:  
             m=f.getMask(self._x)  
             rho_ref = rho_ref * (1-m) + f.rho * m  
         self._rho=rho_ref  
541    
542          return dom      def getDataType(self):
543            return self.__datatype
     def getDensityMask(self):  
         return self._mask  
544    
545      def getReferenceDensity(self):      def getReferenceDensity(self):
546            """
547            Returns the reference density Data object that was used to generate
548            the gravity anomaly data.
549            """
550          return self._rho          return self._rho
551    
552      def getGravityAndStdDev(self):      def getReferenceSusceptibility(self):
553          pde=LinearSinglePDE(self.getDomain())          """
554          G=6.6742e-11*U.m**3/(U.kg*U.sec**2)          Returns the reference magnetic susceptibility Data objects that was
555          m_psi_ref=0.          used to generate the magnetic field data.
556          for i in range(self.DIM):          """
557              m_psi_ref=m_psi_ref + whereZero(self._x[i]-inf(self._x[i])) \          return self._k
                     + whereZero(self._x[i]-sup(self._x[i]))  
558    
559          pde.setValue(A=kronecker(self.getDomain()), Y=-4*np.pi*G*self._rho, q=m_psi_ref)      def getSurveyData(self, domain, origin, NE, spacing):
560            pde=LinearSinglePDE(domain)
561            G=U.Gravitational_Constant
562            m_psi_ref=0.
563            x=domain.getX()
564            DIM=domain.getDim()
565            m_psi_ref=whereZero(x[DIM-1]-sup(x[DIM-1])) # + whereZero(x[DIM-1]-inf(x[DIM-1]))
566            if self.getDataType()==DataSource.GRAVITY:
567                rho_ref=0.
568                for f in self._features:
569                    m=f.getMask(x)
570                    rho_ref = rho_ref * (1-m) + f.getValue(x) * m
571                self._rho=rho_ref
572                pde.setValue(A=kronecker(domain), Y=-4*np.pi*G*rho_ref, q=m_psi_ref)
573            else:
574                k_ref=0.
575                for f in self._features:
576                    m=f.getMask(x)
577                    k_ref = k_ref * (1-m) + f.getValue(x) * m
578                self._k=k_ref
579                pde.setValue(A=kronecker(domain), X=k_ref*self.__B_b, q=m_psi_ref)
580          pde.setSymmetryOn()          pde.setSymmetryOn()
581          psi_ref=pde.getSolution()          psi_ref=pde.getSolution()
582          del pde          del pde
583          g=-grad(psi_ref)          if self.getDataType()==DataSource.GRAVITY:
584          sigma=self._g_mask              data = -grad(psi_ref, ReducedFunction(domain))
585          return g,sigma          else:
586                data = self._k*self.__B_b-grad(psi_ref, ReducedFunction(domain))
587    
588            sigma=1.
589            # limit mask to non-padding in horizontal area
590            x=ReducedFunction(domain).getX()
591            for i in range(self.DIM-1):
592                sigma=sigma * wherePositive(x[i]) \
593                            * whereNegative(x[i]-(sup(x[i])+inf(x[i])))
594            # limit mask to one cell thickness at z=0
595            sigma = sigma * whereNonNegative(x[self.DIM-1]) \
596                    * whereNonPositive(x[self.DIM-1]-spacing[self.DIM-1])
597            return data,sigma
598    
599    
600    

Legend:
Removed from v.3957  
changed lines
  Added in v.4108

  ViewVC Help
Powered by ViewVC 1.1.26