Changeset 7013
- Timestamp:
- 01/10/08 11:11:02 (8 months ago)
- Files:
Legend:
- Unmodified
- Added
- Removed
- Modified
- Copied
- Moved
django/branches/gis/django/contrib/gis/tests/layermap/tests.py
r6992 r7013 11 11 co_shp = os.path.join(shp_path, 'counties/counties.shp') 12 12 inter_shp = os.path.join(shp_path, 'interstates/interstates.shp') 13 14 # Dictionaries to hold what's expected in the county shapefile. 15 NAMES = ['Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo'] 16 NUMS = [1, 2, 1, 19, 1] # Number of polygons for each. 17 STATES = ['Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado'] 13 18 14 19 class LayerMapTest(unittest.TestCase): … … 78 83 # the importation to stop. 79 84 try: 80 lm = LayerMapping(Interstate, inter_shp, inter_mapping, 81 strict=True, silent=True) 82 lm.save() 85 lm = LayerMapping(Interstate, inter_shp, inter_mapping) 86 lm.save(silent=True, strict=True) 83 87 except InvalidDecimal: 84 88 pass … … 87 91 88 92 # This LayerMapping should work b/c `strict` is not set. 89 lm = LayerMapping(Interstate, inter_shp, inter_mapping , silent=True)90 lm.save( )93 lm = LayerMapping(Interstate, inter_shp, inter_mapping) 94 lm.save(silent=True) 91 95 92 96 # Two interstate should have imported correctly. … … 112 116 self.assertAlmostEqual(p1[1], p2[1], 6) 113 117 118 def county_helper(self, county_feat=True): 119 "Helper function for ensuring the integrity of the mapped County models." 120 121 for name, n, st in zip(NAMES, NUMS, STATES): 122 # Should only be one record b/c of `unique` keyword. 123 c = County.objects.get(name=name) 124 self.assertEqual(n, len(c.mpoly)) 125 self.assertEqual(st, c.state.name) # Checking ForeignKey mapping. 126 127 # Multiple records because `unique` was not set. 128 if county_feat: 129 qs = CountyFeat.objects.filter(name=name) 130 self.assertEqual(n, qs.count()) 131 114 132 def test04_layermap_unique_multigeometry_fk(self): 115 133 "Testing the `unique`, and `transform`, geometry collection conversion, and ForeignKey mappings." … … 146 164 # a MissingForeignKey exception (this error would be ignored if the `strict` 147 165 # keyword is not set). 148 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name' , silent=True, strict=True)149 self.assertRaises(MissingForeignKey, lm.save )166 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name') 167 self.assertRaises(MissingForeignKey, lm.save, silent=True, strict=True) 150 168 151 169 # Now creating the state models so the ForeignKey mapping may work. … … 166 184 # all of the various islands in Honolulu county will be in in one 167 185 # database record with a MULTIPOLYGON type. 168 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name' , silent=True, strict=True)169 lm.save( )186 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name') 187 lm.save(silent=True, strict=True) 170 188 171 189 # A reference that doesn't use the unique keyword; a new database record will 172 190 # created for each polygon. 173 lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False, silent=True, strict=True) 174 lm.save() 175 176 # Dictionary to hold what's expected in the shapefile. 177 names = ('Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo') 178 nums = (1, 2, 1, 19, 1) # Number of polygons for each. 179 states = ('Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado') 180 181 for name, n, st in zip(names, nums, states): 182 # Should only be one record b/c of `unique` keyword. 183 c = County.objects.get(name=name) 184 self.assertEqual(n, len(c.mpoly)) 185 self.assertEqual(st, c.state.name) # Checking ForeignKey mapping. 186 187 # Multiple records because `unique` was not set. 188 qs = CountyFeat.objects.filter(name=name) 189 self.assertEqual(n, qs.count()) 190 191 lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False) 192 lm.save(silent=True, strict=True) 193 194 # The county helper is called to ensure integrity of County models. 195 self.county_helper() 196 197 def test05_test_fid_range_step(self): 198 "Tests the `fid_range` keyword and the `step` keyword of .save()." 199 200 # Function for clearing out all the counties before testing. 201 def clear_counties(): County.objects.all().delete() 202 203 # Initializing the LayerMapping object to use in these tests. 204 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name') 205 206 # Bad feature id ranges should raise a type error. 207 clear_counties() 208 bad_ranges = (5.0, 'foo', co_shp) 209 for bad in bad_ranges: 210 self.assertRaises(TypeError, lm.save, fid_range=bad) 211 212 # Step keyword should not be allowed w/`fid_range`. 213 fr = (3, 5) # layer[3:5] 214 self.assertRaises(LayerMapError, lm.save, fid_range=fr, step=10) 215 lm.save(fid_range=fr) 216 217 # Features IDs 3 & 4 are for Galveston County, Texas -- only 218 # one model is returned because the `unique` keyword was set. 219 qs = County.objects.all() 220 self.assertEqual(1, qs.count()) 221 self.assertEqual('Galveston', qs[0].name) 222 223 # Features IDs 5 and beyond for Honolulu County, Hawaii, and 224 # FID 0 is for Pueblo County, Colorado. 225 clear_counties() 226 lm.save(fid_range=slice(5, None), silent=True, strict=True) # layer[5:] 227 lm.save(fid_range=slice(None, 1), silent=True, strict=True) # layer[:1] 228 229 # Only Pueblo & Honolulu counties should be present because of 230 # the `unique` keyword. 231 qs = County.objects.all() 232 self.assertEqual(2, qs.count()) 233 hi, co = tuple(qs) 234 hi_idx, co_idx = tuple(map(NAMES.index, ('Honolulu', 'Pueblo'))) 235 self.assertEqual('Pueblo', co.name); self.assertEqual(NUMS[co_idx], len(co.mpoly)) 236 self.assertEqual('Honolulu', hi.name); self.assertEqual(NUMS[hi_idx], len(hi.mpoly)) 237 238 # Testing the `step` keyword -- should get the same counties 239 # regardless of we use a step that divides equally, that is odd, 240 # or that is larger than the dataset. 241 for st in (4,7,1000): 242 clear_counties() 243 lm.save(step=st, strict=True) 244 self.county_helper(county_feat=False) 245 191 246 def suite(): 192 247 s = unittest.TestSuite() django/branches/gis/django/contrib/gis/utils/layermapping.py
r6992 r7013 44 44 For example, 'latin-1', 'utf-8', and 'cp437' are all valid 45 45 encoding parameters. 46 47 check:48 Due to optimizations, this keyword argument is deprecated and will49 be removed in future revisions.50 51 pipe:52 Status information will be written to this file handle. Defaults53 to using `sys.stdout`, but any object with a `write` method is54 supported.55 56 silent:57 By default, non-fatal error notifications are printed to stdout; this58 keyword may be set in order to disable these notifications.59 60 strict:61 Setting this keyword to True will instruct the save() method to62 cease execution on the first error encountered. The default behavior63 is to attempt to continue even if errors are encountered.64 46 65 47 transaction_mode: … … 176 158 177 159 def __init__(self, model, data, mapping, layer=0, 178 source_srs=None, encoding=None, check=True, pipe=sys.stdout, 179 progress=False, interval=1000, strict=False, silent=False, 180 transaction_mode='commit_on_success', transform=True, 181 unique=False): 160 source_srs=None, encoding=None, 161 transaction_mode='commit_on_success', 162 transform=True, unique=None): 182 163 """ 183 164 A LayerMapping object is initialized using the given Model (not an instance), … … 214 195 # things don't check out before hand. 215 196 self.check_layer() 216 217 # The strict flag -- if it is set, exceptions will be propagated up.218 self.strict = strict219 220 # Setting the keyword arguments related to status printing.221 self.silent = silent222 self.progress = progress223 self.pipe = pipe224 self.interval = interval225 197 226 198 # Setting the encoding for OFTString fields, if specified. … … 250 222 251 223 #### Checking routines used during initialization #### 224 def check_fid_range(self, fid_range): 225 "This checks the `fid_range` keyword." 226 if fid_range: 227 if isinstance(fid_range, (tuple, list)): 228 return slice(*fid_range) 229 elif isinstance(fid_range, slice): 230 return fid_range 231 else: 232 raise TypeError 233 else: 234 return None 235 252 236 def check_layer(self): 253 237 """ … … 368 352 """ 369 353 Given an OGR Feature, this will return a dictionary of keyword arguments 370 for constructing the mapped model. Also returned is the `all_prepped` 371 flag, which is used to signal that a model corresponding to a ForeignKey 372 mapping does not exist. 354 for constructing the mapped model. 373 355 """ 374 356 # The keyword arguments for model construction. 375 357 kwargs = {} 376 377 # The all_prepped flagged, will be set to False if there's a378 # problem w/a ForeignKey that doesn't exist.379 all_prepped = True380 358 381 359 # Incrementing through each model field and OGR field in the … … 391 369 # another mapping for the related Model. 392 370 val = self.verify_fk(feat, model_field, ogr_name) 393 if not val: all_prepped = False394 371 else: 395 372 # Otherwise, verify OGR Field type. … … 400 377 kwargs[field_name] = val 401 378 402 return kwargs , all_prepped379 return kwargs 403 380 404 381 def unique_kwargs(self, kwargs): … … 481 458 return rel_model.objects.get(**fk_kwargs) 482 459 except ObjectDoesNotExist: 483 if self.strict: raise MissingForeignKey('No %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs)) 484 else: return None 485 460 raise MissingForeignKey('No ForeignKey %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs)) 461 486 462 def verify_geom(self, geom, model_field): 487 463 """ … … 537 513 model_field.__class__.__name__ == 'Multi%s' % geom_type.django) 538 514 539 def save(self, verbose=False): 515 def save(self, verbose=False, fid_range=False, step=False, 516 progress=False, silent=False, stream=sys.stdout, strict=False): 540 517 """ 541 518 Saves the contents from the OGR DataSource Layer into the database 542 according to the mapping dictionary given at initialization. If 543 the `verbose` keyword is set, information will be printed subsequent 544 to each model save executed on the database. 545 """ 519 according to the mapping dictionary given at initialization. 520 521 Keyword Parameters: 522 verbose: 523 If set, information will be printed subsequent to each model save 524 executed on the database. 525 526 fid_range: 527 May be set with a slice or tuple of (begin, end) feature ID's to map 528 from the data source. In other words, this keyword enables the user 529 to selectively import a subset range of features in the geographic 530 data source. 531 532 step: 533 If set with an integer, transactions will occur at every step 534 interval. For example, if step=1000, a commit would occur after 535 the 1,000th feature, the 2,000th feature etc. 536 537 progress: 538 When this keyword is set, status information will be printed giving 539 the number of features processed and sucessfully saved. By default, 540 progress information will pe printed every 1000 features processed, 541 however, this default may be overridden by setting this keyword with an 542 integer for the desired interval. 543 544 stream: 545 Status information will be written to this file handle. Defaults to 546 using `sys.stdout`, but any object with a `write` method is supported. 547 548 silent: 549 By default, non-fatal error notifications are printed to stdout, but 550 this keyword may be set to disable these notifications. 551 552 strict: 553 Execution of the model mapping will cease upon the first error 554 encountered. The default behavior is to attempt to continue. 555 """ 556 # Getting the default Feature ID range. 557 default_range = self.check_fid_range(fid_range) 558 559 # Setting the progress interval, if requested. 560 if progress: 561 if progress is True or not isinstance(progress, int): 562 progress_interval = 1000 563 else: 564 progress_interval = progress 565 546 566 # Defining the 'real' save method, utilizing the transaction 547 567 # decorator created during initialization. 548 568 @self.transaction_decorator 549 def _save(): 550 num_feat = 0 551 num_saved = 0 552 553 for feat in self.layer: 569 def _save(feat_range=default_range, num_feat=0, num_saved=0): 570 if feat_range: 571 layer_iter = self.layer[feat_range] 572 else: 573 layer_iter = self.layer 574 575 for feat in layer_iter: 554 576 num_feat += 1 555 577 # Getting the keyword arguments 556 578 try: 557 kwargs , all_prepped= self.feature_kwargs(feat)579 kwargs = self.feature_kwargs(feat) 558 580 except LayerMapError, msg: 559 581 # Something borked the validation 560 if s elf.strict: raise561 elif not s elf.silent:562 s elf.pipe.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg))582 if strict: raise 583 elif not silent: 584 stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg)) 563 585 else: 564 586 # Constructing the model using the keyword args 565 if all_prepped: 566 if self.unique: 567 # If we want unique models on a particular field, handle the 568 # geometry appropriately. 569 try: 570 # Getting the keyword arguments and retrieving 571 # the unique model. 572 u_kwargs = self.unique_kwargs(kwargs) 573 m = self.model.objects.get(**u_kwargs) 574 575 # Getting the geometry (in OGR form), creating 576 # one from the kwargs WKT, adding in additional 577 # geometries, and update the attribute with the 578 # just-updated geometry WKT. 579 geom = getattr(m, self.geom_field).ogr 580 new = OGRGeometry(kwargs[self.geom_field]) 581 for g in new: geom.add(g) 582 setattr(m, self.geom_field, geom.wkt) 583 except ObjectDoesNotExist: 584 # No unique model exists yet, create. 585 m = self.model(**kwargs) 586 else: 587 if self.unique: 588 # If we want unique models on a particular field, handle the 589 # geometry appropriately. 590 try: 591 # Getting the keyword arguments and retrieving 592 # the unique model. 593 u_kwargs = self.unique_kwargs(kwargs) 594 m = self.model.objects.get(**u_kwargs) 595 596 # Getting the geometry (in OGR form), creating 597 # one from the kwargs WKT, adding in additional 598 # geometries, and update the attribute with the 599 # just-updated geometry WKT. 600 geom = getattr(m, self.geom_field).ogr 601 new = OGRGeometry(kwargs[self.geom_field]) 602 for g in new: geom.add(g) 603 setattr(m, self.geom_field, geom.wkt) 604 except ObjectDoesNotExist: 605 # No unique model exists yet, create. 587 606 m = self.model(**kwargs) 588 589 try: 590 # Attempting to save. 591 m.save() 592 num_saved += 1 593 if verbose: self.pipe.write('Saved: %s\n' % m) 594 except SystemExit: 607 else: 608 m = self.model(**kwargs) 609 610 try: 611 # Attempting to save. 612 m.save() 613 num_saved += 1 614 if verbose: stream.write('Saved: %s\n' % m) 615 except SystemExit: 616 raise 617 except Exception, msg: 618 if self.transaction_mode == 'autocommit': 619 # Rolling back the transaction so that other model saves 620 # will work. 621 transaction.rollback_unless_managed() 622 if strict: 623 # Bailing out if the `strict` keyword is set. 624 if not silent: 625 stream.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid) 626 stream.write('%s\n' % kwargs) 595 627 raise 596 except Exception, msg: 597 if self.transaction_mode == 'autocommit': 598 # Rolling back the transaction so that other model saves 599 # will work. 600 transaction.rollback_unless_managed() 601 if self.strict: 602 # Bailing out if the `strict` keyword is set. 603 if not self.silent: 604 self.pipe.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid) 605 self.pipe.write('%s\n' % kwargs) 606 raise 607 elif not self.silent: 608 self.pipe.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg)) 609 else: 610 if not self.silent: self.pipe.write('Skipping due to missing relation:\n%s\n' % kwargs) 611 628 elif not silent: 629 stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg)) 612 630 613 631 # Printing progress information, if requested. 614 if self.progress and num_feat % self.interval == 0: 615 self.pipe.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved)) 616 617 # Calling our defined function, which will use the specified 618 # trasaction mode. 619 _save() 632 if progress and num_feat % progress_interval == 0: 633 stream.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved)) 634 635 # Only used for status output purposes -- incremental saving uses the 636 # values returned here. 637 return num_saved, num_feat 638 639 nfeat = self.layer.num_feat 640 if step and isinstance(step, int) and step < nfeat: 641 # Incremental saving is requested at the given interval (step) 642 if default_range: 643 raise LayerMapError('The `step` keyword may not be used in conjunction with the `fid_range` keyword.') 644 beg, num_feat, num_saved = (0, 0, 0) 645 indices = range(step, nfeat, step) 646 n_i = len(indices) 647 648 for i, end in enumerate(indices): 649 # Constructing the slice to use for this step; the last slice is 650 # special (e.g, [100:] instead of [90:100]). 651 if i+1 == n_i: step_slice = slice(beg, None) 652 else: step_slice = slice(beg, end) 653 654 try: 655 num_feat, num_saved = _save(step_slice, num_feat, num_saved) 656 beg = end 657 except: 658 stream.write('%s\nFailed to save slice: %s\n' % ('=-' * 20, step_slice)) 659 raise 660 else: 661 # Otherwise, just calling the previously defined _save() function. 662 _save()
