Django

Code

Changeset 6687

Show
Ignore:
Timestamp:
11/17/07 15:57:12 (8 months ago)
Author:
jbronn
Message:

gis: LayerMapping? refactor

(1) Moved all routines into LayerMapping? class (for easier subclassing) and modularized the routines.
(2) OFTString and OFTReal OGR fields are verified w/the Django fields prior to insertion, thus avoiding invalidating a large transaction.
(3) Added keyword options for specifying the transaction mode, not performing transformations, and status printing.
(4) Created unit tests.

Other Changes:

Updated ogrinfo for GDAL refactor and fixed an iterating bug; simplified a few lines in geoapp model tests.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • django/branches/gis/django/contrib/gis/tests/geoapp/tests.py

    r6596 r6687  
    7878 
    7979        # Creating a State object using a built Polygon 
    80         ply = Polygon(shell.clone(), inner.clone()) 
     80        ply = Polygon(shell, inner) 
    8181        nullstate = State(name='NullState', poly=ply) 
    8282        self.assertEqual(4326, nullstate.poly.srid) # SRID auto-set from None 
    …… 
    9595        # Changing the interior ring on the poly attribute. 
    9696        new_inner = LinearRing((30, 30), (30, 70), (70, 70), (70, 30), (30, 30)) 
    97         nullstate.poly[1] = new_inner.clone() 
     97        ns.poly[1] = new_inner 
    9898        ply[1] = new_inner 
    99         self.assertEqual(4326, nullstate.poly.srid) 
    100         nullstate.save() 
     99        self.assertEqual(4326, ns.poly.srid) 
     100        ns.save() 
    101101        self.assertEqual(ply, State.objects.get(name='NullState').poly) 
    102         nullstate.delete() 
     102        ns.delete() 
    103103 
    104104    @no_oracle # Oracle does not support KML. 

  • django/branches/gis/django/contrib/gis/tests/__init__.py

    r6527 r6687  
    33from unittest import TestSuite, TextTestRunner 
    44from django.contrib.gis.gdal import HAS_GDAL 
    5 from django.contrib.gis.tests.utils import mysql 
     5try: 
     6    from django.contrib.gis.tests.utils import mysql 
     7except: 
     8    mysql = False 
     9 
     10# Tests that require use of a spatial database (e.g., creation of models) 
     11test_models = ['geoapp'] 
    612 
    713# Tests that do not require setting up and tearing down a spatial database. 
    …… 
    1117] 
    1218if HAS_GDAL: 
     19    test_models += ['layermap'] 
    1320    test_suite_names += [ 
    1421        'test_gdal_driver', 
    …… 
    2128else: 
    2229    print >>sys.stderr, "GDAL not available - no GDAL tests will be run." 
    23  
    24 test_models = ['geoapp'] 
    2530 
    2631def suite(): 
    …… 
    4146     
    4247    In order to run geographic model tests the DATABASE_USER will require 
    43     superuser priviliges.  To accomplish this outside the `postgres` user, 
    44     create your own PostgreSQL database as a user: 
     48    superuser priviliges.  To accomplish this outside the `postgres` user, 
     49    create your own PostgreSQL database as a user: 
    4550     (1) Initialize database: `initdb -D /path/to/user/db` 
    4651     (2) If there's already a Postgres instance on the machine, it will need 
    …… 
    5055 
    5156    On Windows platforms simply use the pgAdmin III utility to add superuser  
    52     priviliges to your database user. 
     57    priviliges to your database user. 
    5358 
    5459    Make sure your settings.py matches the settings of the user database.  
    55     For example, set the same port number (`DATABASE_PORT=5433`).   
    56     DATABASE_NAME or TEST_DATABSE_NAME must be set, along with DATABASE_USER. 
     60    For example, set the same port number (`DATABASE_PORT=5433`).   
     61    DATABASE_NAME or TEST_DATABSE_NAME must be set, along with DATABASE_USER. 
    5762       
    5863    In settings.py set TEST_RUNNER='django.contrib.gis.tests.run_tests'. 
    5964 
    6065    Finally, this assumes that the PostGIS SQL files (lwpostgis.sql and  
    61     spatial_ref_sys.sql) are installed in the directory specified by  
    62     `pg_config --sharedir` (and defaults to /usr/local/share if that fails). 
    63     This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings. 
     66    spatial_ref_sys.sql) are installed in the directory specified by  
     67    `pg_config --sharedir` (and defaults to /usr/local/share if that fails). 
     68    This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings. 
    6469     
    65     Windows users should use the POSTGIS_SQL_PATH because the output 
    66     of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'. 
     70    Windows users should set POSTGIS_SQL_PATH manually because the output 
     71    of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'. 
    6772 
    6873    Finally, the tests may be run by invoking `./manage.py test`. 
    …… 
    9499        test_suite.addTest(tsuite.suite()) 
    95100 
    96     # Resetting the loaded flag to take into account what we appended to the INSTALLED_APPS 
    97     #  (since this routine is invoked through django/core/management, it caches the apps, 
    98     #   this ensures that syncdb will see our appended models) 
     101    # Resetting the loaded flag to take into account what we appended to  
     102    # the INSTALLED_APPS (since this routine is invoked through  
     103    # django/core/management, it caches the apps; this ensures that syncdb  
     104    # will see our appended models) 
    99105    from django.db.models import loading 
    100106    loading._loaded = False 

  • django/branches/gis/django/contrib/gis/utils/layermapping.py

    r6524 r6687  
    11# LayerMapping -- A Django Model/OGR Layer Mapping Utility 
    22""" 
    3 The LayerMapping class provides a way to map the contents of OGR 
     3 The LayerMapping class provides a way to map the contents of OGR 
    44 vector files (e.g. SHP files) to Geographic-enabled Django models. 
    55 
    6 This grew out of my personal needs, specifically the code repetition 
     6 This grew out of my personal needs, specifically the code repetition 
    77 that went into pulling geometries and fields out of an OGR layer, 
    88 converting to another coordinate system (e.g. WGS84), and then inserting 
    9  into a Geographic Django model. 
    10  
    11 This utility is still in early stages of development, so its usage 
     9 into a GeoDjango model. 
     10 
     11 This utility is still in early stages of development, so its usage 
    1212 is subject to change -- please report any bugs. 
    1313 
    14 TODO: Unit tests and documentation. 
    15  
    16 Requirements:  OGR C Library (from GDAL) required. 
    17  
    18 Usage:  
     14 Requirements:  OGR C Library (from GDAL) required. 
     15 
     16 Usage:  
    1917  lm = LayerMapping(model, source_file, mapping) where, 
    2018 
    …… 
    3331   geometry type, e.g. 'POINT', 'LINESTRING', 'POLYGON'. 
    3432 
    35 Keyword Args: 
     33 Keyword Args: 
    3634  layer: 
    3735   The index of the layer to use from the Data Source (defaults to 0) 
    …… 
    4644   For example, 'latin-1', 'utf-8', and 'cp437' are all valid 
    4745   encoding parameters. 
     46 
     47  check: 
     48   By default, LayerMapping increments through each feature in the 
     49   layer to ensure that it is compatible with the given model and 
     50   mapping.  Setting this keyword to False, disables this action, 
     51   which will speed up execution time for very large files. 
     52 
     53  silent: 
     54   By default, non-fatal error notifications are printed to stdout; this 
     55   keyword may be set in order to disable these notifications. 
     56 
     57  strict: 
     58   Setting this keyword to True will instruct the save() method to 
     59   cease execution on the first error encountered. 
     60 
     61  transaction_mode: 
     62   May be 'commit_on_success' (default) or 'autocommit'. 
     63 
     64  transform: 
     65   Setting this to False will disable all coordinate transformations.   
    4866 
    4967Example: 
    …… 
    92110 
    93111 LayerMapping just transformed the three geometries from the SHP file from their 
    94   source spatial reference system (WGS84) to the spatial reference system of 
    95   the GeoDjango model (NAD83).  If no spatial reference system is defined for 
    96   the layer, use the `source_srs` keyword with a SpatialReference object to 
    97   specify one. Further, data is selectively imported from the given data source  
    98   fields into the model fields. 
     112 source spatial reference system (WGS84) to the spatial reference system of 
     113 the GeoDjango model (NAD83).  If no spatial reference system is defined for 
     114 the layer, use the `source_srs` keyword with a SpatialReference object to 
     115 specify one. 
    99116""" 
    100 from types import StringType, TupleType 
    101 from datetime import datetime 
     117from datetime import date, datetime 
     118from decimal import Decimal 
     119from django.core.exceptions import ObjectDoesNotExist 
     120from django.db import connection, transaction 
     121from django.db.models.fields.related import ForeignKey 
    102122from django.contrib.gis.db.backend import SPATIAL_BACKEND 
    103 from django.contrib.gis.gdal import \ 
    104      OGRGeometry, OGRGeomType, SpatialReference, CoordTransform, \ 
    105      DataSource, OGRException 
    106 from django.contrib.gis.gdal.field import Field, OFTInteger, OFTReal, OFTString, OFTDateTime 
     123from django.contrib.gis.gdal import CoordTransform, DataSource, \ 
     124    OGRException, OGRGeometry, OGRGeomType, SpatialReference 
     125from django.contrib.gis.gdal.field import OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime 
    107126from django.contrib.gis.models import GeometryColumns, SpatialRefSys 
    108 from django.db import connection, transaction 
    109 from django.core.exceptions import ObjectDoesNotExist 
    110  
    111 # A mapping of given geometry types to their OGR integer type. 
    112 ogc_types = {'POINT' : OGRGeomType('Point'), 
    113              'LINESTRING' : OGRGeomType('LineString'), 
    114              'POLYGON' : OGRGeomType('Polygon'), 
    115              'MULTIPOINT' : OGRGeomType('MultiPoint'), 
    116              'MULTILINESTRING' : OGRGeomType('MultiLineString'), 
    117              'MULTIPOLYGON' : OGRGeomType('MultiPolygon'), 
    118              'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'), 
    119              } 
    120  
    121 # The django.contrib.gis model types. 
    122 gis_fields = {'PointField' : 'POINT', 
    123               'LineStringField': 'LINESTRING', 
    124               'PolygonField': 'POLYGON', 
    125               'MultiPointField' : 'MULTIPOINT', 
    126               'MultiLineStringField' : 'MULTILINESTRING', 
    127               'MultiPolygonField' : 'MULTIPOLYGON', 
    128               } 
    129  
    130 # Acceptable 'base' types for a multi-geometry type. 
    131 multi_types = {'POINT' : OGRGeomType('MultiPoint'), 
    132                'LINESTRING' : OGRGeomType('MultiLineString'), 
    133                'POLYGON' : OGRGeomType('MultiPolygon'), 
    134                } 
    135  
    136 def map_foreign_key(django_field): 
    137     from django.db.models.fields.related import ForeignKey 
    138  
    139     if not django_field.__class__ is ForeignKey: 
    140         return django_field.__class__.__name__ 
    141  
    142       
    143     rf=django_field.rel.get_related_field() 
    144  
    145     return rf.get_internal_type() 
    146                  
    147 # The acceptable Django field types that map to OGR fields. 
    148 field_types = { 
    149                'AutoField' : OFTInteger, 
    150                'IntegerField' : OFTInteger, 
    151                'FloatField' : OFTReal, 
    152                'DateTimeField' : OFTDateTime, 
    153                'DecimalField' : OFTReal, 
    154                'CharField' : OFTString, 
    155                'SmallIntegerField' : OFTInteger, 
    156                'PositiveSmallIntegerField' : OFTInteger, 
    157                } 
    158  
    159 def make_multi(geom_name, model_type): 
    160     "Determines whether the geometry should be made into a GeometryCollection." 
    161     if (geom_name in multi_types) and (model_type.startswith('Multi')): 
    162         return True 
    163     else: 
    164         return False 
    165  
    166 def check_feature(feat, model_fields, mapping): 
    167     "Checks the OGR layer feature." 
    168  
    169     HAS_GEO = False 
    170  
    171     # Incrementing through each model_field & ogr_field in the given mapping. 
    172     for model_field, ogr_field in mapping.items(): 
    173  
    174         # Making sure the given mapping model field is in the given model fields. 
    175         if model_field in model_fields: 
    176             model_type = model_fields[model_field] 
    177         elif model_field[:-3] in model_fields: #foreign key 
    178             model_type = model_fields[model_field[:-3]] 
    179         else: 
    180             raise Exception('Given mapping field "%s" not in given Model fields!' % model_field) 
    181  
    182         ### Handling if we get a geometry in the Field ### 
    183         if ogr_field in ogc_types: 
    184             # At this time, no more than one geographic field per model =( 
    185             if HAS_GEO: 
    186                 raise Exception('More than one geographic field in mapping not allowed (yet).') 
    187             else: 
    188                 HAS_GEO = ogr_field 
    189  
    190             # Making sure this geometry field type is a valid Django GIS field. 
    191             if not model_type in gis_fields: 
    192                 raise Exception('Unknown Django GIS field type "%s"' % model_type) 
    193              
    194             # Getting the OGRGeometry, it's type (an integer) and it's name (a string) 
    195             geom  = feat.geom 
    196             gtype = geom.geom_type 
    197             gname = geom.geom_name 
    198  
    199             if make_multi(gname, model_type): 
    200                 # Do we have to 'upsample' into a Geometry Collection? 
    201                 pass 
    202             elif gtype == ogc_types[gis_fields[model_type]]: 
    203                 # The geometry type otherwise was expected 
    204                 pass 
    205             else: 
    206                 raise Exception('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype)) 
    207  
    208         ## Handling other fields  
    209         else: 
    210             # Making sure the model field is 
    211             if not model_type in field_types: 
    212                 raise Exception('Django field type "%s" has no OGR mapping (yet).' % model_type) 
    213  
    214             # Otherwise, we've got an OGR Field.  Making sure that an 
    215             # index exists for the mapping OGR field. 
    216             try: 
    217                 fi = feat.index(ogr_field) 
    218             except: 
    219                 raise Exception('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field) 
    220                      
    221 def check_layer(layer, fields, mapping): 
    222     "Checks the OGR layer by incrementing through and checking each feature." 
    223     # Incrementing through each feature in the layer. 
    224     for feat in layer: 
    225         check_feature(feat, fields, mapping) 
    226  
    227 def check_srs(layer, source_srs): 
    228     "Checks the compatibility of the given spatial reference object." 
    229     if isinstance(source_srs, SpatialReference): 
    230         sr = source_srs 
    231     elif isinstance(source_srs, SpatialRefSys): 
    232         sr = source_srs.srs 
    233     elif isinstance(source_srs, (int, str)): 
    234         sr = SpatialReference(source_srs) 
    235     else: 
    236         sr = layer.srs 
    237     if not sr: 
    238         raise Exception('No source reference system defined.') 
    239     else: 
    240         return sr 
    241  
    242 class LayerMapping: 
    243     "A class that maps OGR Layers to Django Models." 
    244  
    245     def __init__(self, model, data, mapping, layer=0, source_srs=None, encoding=None): 
     127 
     128# LayerMapping exceptions. 
     129class LayerMapError(Exception): pass 
     130class InvalidString(LayerMapError): pass 
     131class InvalidDecimal(LayerMapError): pass 
     132 
     133class LayerMapping(object): 
     134    "A class that maps OGR Layers to GeoDjango Models." 
     135     
     136    # A mapping of given geometry types to their OGR integer type. 
     137    OGC_TYPES = {'POINT' : OGRGeomType('Point'), 
     138                 'LINESTRING' : OGRGeomType('LineString'), 
     139                 'POLYGON' : OGRGeomType('Polygon'), 
     140                 'MULTIPOINT' : OGRGeomType('MultiPoint'), 
     141                 'MULTILINESTRING' : OGRGeomType('MultiLineString'), 
     142                 'MULTIPOLYGON' : OGRGeomType('MultiPolygon'), 
     143                 'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'), 
     144                 } 
     145 
     146    # The django.contrib.gis model types. 
     147    GIS_FIELDS = {'PointField' : 'POINT', 
     148                  'LineStringField': 'LINESTRING', 
     149                  'PolygonField': 'POLYGON', 
     150                  'MultiPointField' : 'MULTIPOINT', 
     151                  'MultiLineStringField' : 'MULTILINESTRING', 
     152                  'MultiPolygonField' : 'MULTIPOLYGON', 
     153                  'GeometryCollectionField' : 'GEOMETRYCOLLECTION', 
     154                  } 
     155 
     156    # Acceptable 'base' types for a multi-geometry type. 
     157    MULTI_TYPES = {'POINT' : OGRGeomType('MultiPoint'), 
     158                   'LINESTRING' : OGRGeomType('MultiLineString'), 
     159                   'POLYGON' : OGRGeomType('MultiPolygon'), 
     160                   } 
     161 
     162    # The acceptable Django field types that map to OGR fields. 
     163    FIELD_TYPES = { 
     164        'AutoField' : OFTInteger, 
     165        'IntegerField' : OFTInteger, 
     166        'FloatField' : OFTReal, 
     167        'DateField' : OFTDate, 
     168        'DateTimeField' : OFTDateTime, 
     169        'TimeField' : OFTTime, 
     170        'DecimalField' : OFTReal, 
     171        'CharField' : OFTString, 
     172        'TextField' : OFTString, 
     173        'SmallIntegerField' : OFTInteger, 
     174        'PositiveSmallIntegerField' : OFTInteger, 
     175        } 
     176 
     177    # The acceptable transaction modes. 
     178    TRANSACTION_MODES = {'autocommit' : transaction.autocommit, 
     179                         'commit_on_success' : transaction.commit_on_success, 
     180                         } 
     181 
     182    def __init__(self, model, data, mapping, layer=0,  
     183                 source_srs=None, encoding=None, check=True,  
     184                 progress=False, interval=1000, strict=False, silent=False, 
     185                 transaction_mode='commit_on_success', transform=True): 
    246186        "Takes the Django model, the data source, and the mapping (dictionary)" 
    247187 
    248188        # Getting the field names and types from the model 
    249         fields = dict((f.name, map_foreign_key(f)) for f in model._meta.fields) 
     189        self.fields = dict((f.name, self.map_foreign_key(f)) for f in model._meta.fields) 
     190        self.field_classes = dict((f.name, f) for f in model._meta.fields) 
     191 
    250192        # Getting the DataSource and its Layer 
    251193        if isinstance(data, basestring): 
    …… 
    255197        self.layer = self.ds[layer] 
    256198 
     199        # Setting the mapping 
     200        self.mapping = mapping 
     201 
     202        # Setting the model, and getting the geometry column associated  
     203        # with the model (an exception will be raised if there is no  
     204        # geometry column). 
     205        self.model = model 
     206        self.geo_col = self.geometry_column() 
     207 
     208        # Checking the source spatial reference system, and getting 
     209        # the coordinate transformation object (unless the `transform` 
     210        # keyword is set to False) 
     211        self.source_srs = self.check_srs(source_srs) 
     212        self.transform = transform and self.coord_transform() 
     213 
    257214        # Checking the layer -- intitialization of the object will fail if 
    258         #  things don't check out before hand. 
    259         check_layer(self.layer, fields, mapping) 
    260          
    261         # Since the layer checked out, setting the fields and the mapping. 
    262         self.fields = fields 
    263         self.mapping = mapping 
    264         self.model = model 
    265         self.source_srs = check_srs(self.layer, source_srs) 
     215        # things don't check out before hand.  This may be time-consuming, 
     216        # and disabled by setting the `check` keyword to False. 
     217        if check: self.check_layer() 
     218 
     219        # The silent, strict, progress, and interval flags. 
     220        self.silent = silent 
     221        self.strict = strict 
     222        self.progress = progress 
     223        self.interval = interval 
    266224 
    267225        # Setting the encoding for OFTString fields, if specified. 
    268226        if encoding: 
    269227            # Making sure the encoding exists, if not a LookupError 
    270             # exception will be thrown. 
     228            # exception will be thrown. 
    271229            from codecs import lookup 
    272230            lookup(encoding) 
    …… 
    275233            self.encoding = None 
    276234 
    277     # Either the import will work, or it won't be committed. 
    278     @transaction.commit_on_success 
    279     def save(self, verbose=False): 
    280         "Runs the layer mapping on the given SHP file, and saves to the database." 
    281  
     235        # Setting the transaction decorator with the function in the  
     236        # transaction modes dictionary. 
     237        if transaction_mode in self.TRANSACTION_MODES: 
     238            self.transaction_decorator = self.TRANSACTION_MODES[transaction_mode] 
     239            self.transaction_mode = transaction_mode 
     240        else: 
     241            raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode) 
     242 
     243    def check_feature(self, feat): 
     244        "Checks the OGR feature against the model fields and mapping." 
     245        HAS_GEO = False    
     246 
     247        # Incrementing through each model_field & ogr_field in the given mapping. 
     248        for model_field, ogr_field in self.mapping.items(): 
     249            # Making sure the given mapping model field is in the given model fields. 
     250            if model_field in self.fields: 
     251                model_type = self.fields[model_field] 
     252            elif model_field[:-3] in self.fields: #foreign key 
     253                model_type = self.fields[model_field[:-3]] 
     254            else: 
     255                raise LayerMapError('Given mapping field "%s" not in given Model fields!' % model_field) 
     256 
     257            ### Handling if we get a geometry in the Field ### 
     258            if ogr_field in self.OGC_TYPES: 
     259                # At this time, no more than one geographic field per model =( 
     260                if HAS_GEO: 
     261                    raise LayerMapError('More than one geographic field in mapping not allowed (yet).') 
     262                else: 
     263                    HAS_GEO = ogr_field 
     264 
     265                # Making sure this geometry field type is a valid Django GIS field. 
     266                if not model_type in self.GIS_FIELDS: 
     267                    raise LayerMapError('Unknown Django GIS field type "%s"' % model_type) 
     268 
     269                # Getting the OGRGeometry, it's type (an integer) and it's name (a string) 
     270                geom  = feat.geom 
     271                gtype = geom.geom_type 
     272                gname = geom.geom_name 
     273 
     274                if self.make_multi(gname, model_type): 
     275                    # Do we have to 'upsample' into a Geometry Collection? 
     276                    pass 
     277                elif gtype == self.OGC_TYPES[self.GIS_FIELDS[model_type]]: 
     278                    # The geometry type otherwise was expected 
     279                    pass 
     280                else: 
     281                    raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s' % (model_type, gtype)) 
     282            ### Handling other fields ### 
     283            else: 
     284                # Making sure the model field is supported. 
     285                if not model_type in self.FIELD_TYPES: 
     286                    raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % model_type) 
     287 
     288                # Otherwise, we've got an OGR Field.  Making sure that an 
     289                # index exists for the mapping OGR field. 
     290                try: 
     291                    fi = feat.index(ogr_field) 
     292                except: 
     293                    raise LayerMapError('Given mapping OGR field "%s" not in given OGR layer feature!' % ogr_field) 
     294 
     295    def check_layer(self): 
     296        "Checks every feature in this object's layer." 
     297        for feat in self.layer: 
     298            self.check_feature(feat) 
     299 
     300    def check_srs(self, source_srs): 
     301        "Checks the compatibility of the given spatial reference object." 
     302        if isinstance(source_srs, SpatialReference): 
     303            sr = source_srs 
     304        elif isinstance(source_srs, SpatialRefSys): 
     305            sr = source_srs.srs 
     306        elif isinstance(source_srs, (int, str)): 
     307            sr = SpatialReference(source_srs) 
     308        else: 
     309            # Otherwise just pulling the SpatialReference from the layer 
     310            sr = self.layer.srs 
     311             
     312        if not sr: 
     313            raise LayerMapError('No source reference system defined.') 
     314        else: 
     315            return sr 
     316 
     317    def coord_transform(self): 
     318        "Returns the coordinate transformation object." 
     319        try: 
     320            # Getting the target spatial reference system 
     321            target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs 
     322         
     323            # Creating the CoordTransform object 
     324            return CoordTransform(self.source_srs, target_srs) 
     325        except Exception, msg: 
     326            raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg) 
     327 
     328    def feature_kwargs(self, feat): 
     329        "Returns the keyword arguments needed for saving a feature." 
     330         
     331        # The keyword arguments for model construction. 
     332        kwargs = {} 
     333 
     334        # The all_prepped flagged, will be set to False if there's a 
     335        # problem w/a ForeignKey that doesn't exist. 
     336        all_prepped = True 
     337 
     338        # Incrementing through each model field and OGR field in the 
     339        # dictionary mapping. 
     340        for model_field, ogr_field in self.mapping.items(): 
     341            is_fk = False 
     342            try: 
     343                model_type = self.fields[model_field] 
     344            except KeyError: #foreign key 
     345                # The -3 index is b/c foreign keys are appended w/'_id'. 
     346                model_type = self.fields[model_field[:-3]] 
     347                is_fk = True 
     348             
     349            if ogr_field in self.OGC_TYPES: 
     350                # Verify OGR geometry. 
     351                val = self.verify_geom(feat.geom, model_type) 
     352            else: 
     353                # Otherwise, verify OGR Field type. 
     354                val = self.verify_field(feat[ogr_field], model_field) 
     355 
     356            if is_fk: 
     357                # Handling if foreign key. 
     358                rel_obj = None 
     359                field_name = model_field[:-3] 
     360                try: 
     361                    # FIXME: refactor to efficiently fetch FKs. 
     362                    #  Requires significant re-work. :-/ 
     363                    rel = self.model._meta.get_field(field_name).rel 
     364                    rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val}) 
     365                except ObjectDoesNotExist: 
     366                    all_prepped = False 
     367 
     368                kwargs[model_field[:-3]] = rel_obj 
     369            else: 
     370                kwargs[model_field] = val 
     371             
     372        return kwargs, all_prepped 
     373 
     374    def verify_field(self, fld, model_field): 
     375        """ 
     376        Verifies if the OGR Field contents are acceptable to the Django 
     377        model field.  If they are, the verified value is returned,  
     378        otherwise the proper exception is raised. 
     379        """ 
     380        field_class = self.field_classes[model_field] 
     381        if isinstance(fld, OFTString): 
     382            if self.encoding: 
     383                # The encoding for OGR data sources may be specified here 
     384                # (e.g., 'cp437' for Census Bureau boundary files). 
     385                val = unicode(fld.value, self.encoding) 
     386            else: 
     387                val = fld.value 
     388                if len(val) > field_class.max_length: 
     389                    raise InvalidString('%s model field maximum string length is %s, given %s characters.' % 
     390                                        (model_field, field_class.max_length, len(val))) 
     391        elif isinstance(fld, OFTReal): 
     392            try: 
     393                # Creating an instance of the Decimal value to use. 
     394                d = Decimal(str(fld.value)) 
     395            except: 
     396                raise InvalidDecimal('Could not construct decimal from: %s' % fld) 
     397            dtup = d.as_tuple() 
     398            if len(dtup[1]) > field_class.max_digits: 
     399                raise InvalidDecimal('More than the maximum # of digits encountered.') 
     400            elif len(dtup[1][dtup[2]:]) > field_class.decimal_places: 
     401                raise InvalidDecimal('More than the maximum # of decimal places encountered.') 
     402            val = d 
     403        else: 
     404            val = fld.value 
     405        return val 
     406 
     407    def verify_geom(self, geom, model_type): 
     408        "Verifies the geometry." 
     409        if self.make_multi(geom.geom_name, model_type): 
     410            # Constructing a multi-geometry type to contain the single geometry 
     411            multi_type = self.MULTI_TYPES[geom.geom_name] 
     412            g = OGRGeometry(multi_type) 
     413            g.add(geom) 
     414        else: 
     415            g = geom 
     416 
     417        # Transforming the geometry with our Coordinate Transformation object, 
     418        # but only if the class variable `transform` is set w/a CoordTransform  
     419        # object. 
     420        if self.transform: g.transform(self.transform) 
     421         
     422        # Returning the WKT of the geometry. 
     423        return g.wkt 
     424         
     425    def geometry_column(self): 
     426        "Returns the GeometryColumn model associated with the geographic column." 
    282427        # Getting the GeometryColumn object. 
    283428        try: 
    …… 
    285430            if SPATIAL_BACKEND == 'oracle': db_table = db_table.upper() 
    286431            gc_kwargs = {GeometryColumns.table_name_col() : db_table} 
    287             geo_col = GeometryColumns.objects.get(**gc_kwargs) 
    288         except: 
    289             raise Exception('Geometry column does not exist. (did you run syncdb?)') 
     432            return GeometryColumns.objects.get(**gc_kwargs) 
     433        except Exception, msg: 
     434            raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg) 
     435 
     436    def make_multi(self, geom_name, model_type): 
     437        "Determines whether the geometry should be made into a GeometryCollection." 
     438        return (geom_name in self.MULTI_TYPES) and (model_type.startswith('Multi')) 
     439 
     440    def map_foreign_key(self, django_field): 
     441        "Handles fields within foreign keys for the given field." 
     442        if not django_field.__class__ is ForeignKey: 
     443            # Returning the field's class name. 
     444            return django_field.__class__.__name__ 
     445        else: 
     446            # Otherwise, getting the type of the related field's 
     447            # from the Foreign key. 
     448            rf = django_field.rel.get_related_field() 
     449            return rf.get_internal_type() 
     450 
     451    def save(self, verbose=False): 
     452        "Runs the layer mapping on the given SHP file, and saves to the database." 
    290453         
    291         # Getting the coordinate system needed for transformation (with CoordTransform)   
    292         try: 
    293             # Getting the target spatial reference system 
    294             target_srs = SpatialRefSys.objects.get(srid=geo_col.srid).srs 
    295  
    296             # Creating the CoordTransform object 
    297             ct = CoordTransform(self.source_srs, target_srs) 
    298         except Exception, msg: 
    299             raise Exception('Could not translate between the data source and model geometry: %s' % msg) 
    300  
    301         for feat in self.layer: 
    302             # The keyword arguments for model construction 
    303             kwargs = {} 
    304  
    305             # Incrementing through each model field and the OGR field in the mapping 
    306             all_prepped = True 
    307  
    308             for model_field, ogr_field in self.mapping.items(): 
    309                 is_fk = False 
     454        @self.transaction_decorator 
     455        def _save(): 
     456            num_feat = 0 
     457            num_saved = 0 
     458 
     459            for feat in self.layer: 
     460                num_feat += 1 
     461                # Getting the keyword arguments 
    310462                try: 
    311                     model_type = self.fields[model_field] 
    312                 except KeyError: #foreign key 
    313                     model_type = self.fields[model_field[:-3]] 
    314                     is_fk = True 
    315  
    316                 if ogr_field in ogc_types: 
    317                     ## Getting the OGR geometry from the field 
    318                     geom = feat.geom 
    319  
    320                     if make_multi(geom.geom_name, model_type): 
    321                         # Constructing a multi-geometry type to contain the single geometry 
    322                         multi_type = multi_types[geom.geom_name] 
    323                         g = OGRGeometry(multi_type) 
    324                         g.add(geom) 
     463                    kwargs, all_prepped = self.feature_kwargs(feat) 
     464                except LayerMapError, msg: 
     465                    # Something borked the validation 
     466                    if self.strict: raise 
     467                    elif not self.silent:  
     468                        print 'Ignoring Feature ID %s because: %s' % (feat.fid, msg) 
     469                else: 
     470                    # Constructing the model using the constructed keyword args 
     471                    if all_prepped: 
     472                        m = self.model(**kwargs) 
     473                        try: 
     474                            m.save() 
     475                            num_saved += 1 
     476                            if verbose: print 'Saved: %s' % m 
     477                        except SystemExit: 
     478                            raise 
     479                        except Exception, msg: 
     480                            if self.transaction_mode == 'autocommit': 
     481                                # Rolling back the transaction so that other model saves 
     482                                # will work. 
     483                                transaction.rollback_unless_managed() 
     484                            if self.strict:  
     485                                # Bailing out if the `strict` keyword is set. 
     486                                if not self.silent: 
     487                                    print 'Failed to save the feature (id: %s) into the model with the keyword arguments:' % feat.fid 
     488                                    print kwargs 
     489                                raise 
     490                            elif not self.silent: 
     491                                print 'Failed to save %s:\n %s\nContinuing' % (kwargs, msg) 
    325492                    else: 
    326                         g = geom 
    327  
    328                     # Transforming the geometry with our Coordinate Transformation object. 
    329                     g.transform(ct) 
    330  
    331                     # Updating the keyword args with the WKT of the transformed model. 
    332                     val = g.wkt 
    333                 else: 
    334                     ## Otherwise, this is an OGR field type 
    335                     fld = feat[ogr_field] 
    336  
    337                     if isinstance(fld, OFTString) and self.encoding: 
    338                         # The encoding for OGR data sources may be specified here 
    339                         #  (e.g., 'cp437' for Census Bureau boundary files). 
    340                         val = unicode(fld.value, self.encoding) 
    341                     else: 
    342                         val = fld.value 
    343  
    344                 if is_fk: 
    345                     rel_obj = None 
    346                     field_name = model_field[:-3] 
    347                     try: 
    348                         #FIXME: refactor to efficiently fetch FKs. 
    349                         #  Requires significant re-work. :-/ 
    350                         rel = self.model._meta.get_field(field_name).rel 
    351                         rel_obj = rel.to._default_manager.get(**{('%s__exact' % rel.field_name):val}) 
    352                     except ObjectDoesNotExist: 
    353                         all_prepped = False 
    354                      
    355                     kwargs[model_field[:-3]] = rel_obj 
    356                 else: 
    357                     kwargs[model_field] = val 
    358  
    359             # Constructing the model using the constructed keyword args 
    360             if all_prepped: 
    361                 m = self.model(**kwargs) 
    362  
    363                 # Saving the model 
    364                 try: 
    365                     if all_prepped: 
    366                         m.save() 
    367                         if verbose: print 'Saved: %s' % str(m)                         
    368                     else: 
    369                         print "Skipping %s due to missing relation." % kwargs 
    370                 except SystemExit: 
    371                     raise 
    372                 except Exception, e: 
    373                     print "Failed to save %s\n  Continuing" % kwargs 
     493                        print 'Skipping %s due to missing relation.' % kwargs 
     494 
     495                # Printing progress information, if requested. 
     496                if self.progress and num_feat % self.interval == 0: 
     497                    print 'Processed %d features, saved %d ...' % (num_feat, num_saved) 
     498                
     499        # Calling our defined function, which will use the specified 
     500        # trasaction mode. 
     501        _save() 

  • django/branches/gis/django/contrib/gis/utils/ogrinfo.py

    r6423 r6687  
    3434        width = max(*map(len,layer.fields)) 
    3535        fmt = " %%%ss: %%s" % width 
    36         for i, feature in enumerate(layer[:num_features]): 
    37             print "=== Feature %s" % i 
    38             for field in layer.fields: 
    39                 fld_typ = feature[field].__class__.__name__.replace('OFT', '') 
    40                 output = fmt % (field, fld_typ) 
    41                 val = feature.get(field) 
     36        for j, feature in enumerate(layer[:num_features]): 
     37            print "=== Feature %s" % j 
     38            for fld_name in layer.fields: 
     39                type_name = feature[fld_name].type_name 
     40                output = fmt % (fld_name, type_name) 
     41                val = feature.get(fld_name) 
    4242                if val: 
    4343                    if isinstance(val, str):