001// License: GPL. See LICENSE file for details. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.InputStream; 007import java.io.InputStreamReader; 008import java.text.MessageFormat; 009import java.util.ArrayList; 010import java.util.Collection; 011import java.util.List; 012import java.util.regex.Matcher; 013import java.util.regex.Pattern; 014 015import javax.xml.stream.Location; 016import javax.xml.stream.XMLInputFactory; 017import javax.xml.stream.XMLStreamConstants; 018import javax.xml.stream.XMLStreamException; 019import javax.xml.stream.XMLStreamReader; 020 021import org.openstreetmap.josm.Main; 022import org.openstreetmap.josm.data.Bounds; 023import org.openstreetmap.josm.data.DataSource; 024import org.openstreetmap.josm.data.coor.LatLon; 025import org.openstreetmap.josm.data.osm.Changeset; 026import org.openstreetmap.josm.data.osm.DataSet; 027import org.openstreetmap.josm.data.osm.Node; 028import org.openstreetmap.josm.data.osm.NodeData; 029import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 030import org.openstreetmap.josm.data.osm.PrimitiveData; 031import org.openstreetmap.josm.data.osm.Relation; 032import org.openstreetmap.josm.data.osm.RelationData; 033import org.openstreetmap.josm.data.osm.RelationMemberData; 034import org.openstreetmap.josm.data.osm.Tagged; 035import org.openstreetmap.josm.data.osm.User; 036import org.openstreetmap.josm.data.osm.Way; 037import org.openstreetmap.josm.data.osm.WayData; 038import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 039import org.openstreetmap.josm.gui.progress.ProgressMonitor; 040import org.openstreetmap.josm.tools.CheckParameterUtil; 041import org.openstreetmap.josm.tools.date.DateUtils; 042 043/** 044 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it. 045 * 046 * For each xml element, there is a dedicated method. 047 * The XMLStreamReader cursor points to the start of the element, when the method is 048 * entered, and it must point to the end of the same element, when it is exited. 049 */ 050public class OsmReader extends AbstractReader { 051 052 protected XMLStreamReader parser; 053 054 protected boolean cancel; 055 056 /** Used by plugins to register themselves as data postprocessors. */ 057 private static List<OsmServerReadPostprocessor> postprocessors; 058 059 /** register a new postprocessor */ 060 public static void registerPostprocessor(OsmServerReadPostprocessor pp) { 061 if (postprocessors == null) { 062 postprocessors = new ArrayList<>(); 063 } 064 postprocessors.add(pp); 065 } 066 067 /** deregister a postprocessor previously registered with registerPostprocessor */ 068 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) { 069 if (postprocessors != null) { 070 postprocessors.remove(pp); 071 } 072 } 073 074 /** 075 * constructor (for private and subclasses use only) 076 * 077 * @see #parseDataSet(InputStream, ProgressMonitor) 078 */ 079 protected OsmReader() { 080 } 081 082 protected void setParser(XMLStreamReader parser) { 083 this.parser = parser; 084 } 085 086 protected void throwException(String msg, Throwable th) throws XMLStreamException { 087 throw new OsmParsingException(msg, parser.getLocation(), th); 088 } 089 090 protected void throwException(String msg) throws XMLStreamException { 091 throw new OsmParsingException(msg, parser.getLocation()); 092 } 093 094 protected void parse() throws XMLStreamException { 095 int event = parser.getEventType(); 096 while (true) { 097 if (event == XMLStreamConstants.START_ELEMENT) { 098 parseRoot(); 099 } else if (event == XMLStreamConstants.END_ELEMENT) 100 return; 101 if (parser.hasNext()) { 102 event = parser.next(); 103 } else { 104 break; 105 } 106 } 107 parser.close(); 108 } 109 110 protected void parseRoot() throws XMLStreamException { 111 if ("osm".equals(parser.getLocalName())) { 112 parseOsm(); 113 } else { 114 parseUnknown(); 115 } 116 } 117 118 private void parseOsm() throws XMLStreamException { 119 String v = parser.getAttributeValue(null, "version"); 120 if (v == null) { 121 throwException(tr("Missing mandatory attribute ''{0}''.", "version")); 122 } 123 if (!"0.6".equals(v)) { 124 throwException(tr("Unsupported version: {0}", v)); 125 } 126 ds.setVersion(v); 127 String upload = parser.getAttributeValue(null, "upload"); 128 if (upload != null) { 129 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload)); 130 } 131 String generator = parser.getAttributeValue(null, "generator"); 132 Long uploadChangesetId = null; 133 if (parser.getAttributeValue(null, "upload-changeset") != null) { 134 uploadChangesetId = getLong("upload-changeset"); 135 } 136 while (true) { 137 int event = parser.next(); 138 139 if (cancel) { 140 cancel = false; 141 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation()); 142 } 143 144 if (event == XMLStreamConstants.START_ELEMENT) { 145 switch (parser.getLocalName()) { 146 case "bounds": 147 parseBounds(generator); 148 break; 149 case "node": 150 parseNode(); 151 break; 152 case "way": 153 parseWay(); 154 break; 155 case "relation": 156 parseRelation(); 157 break; 158 case "changeset": 159 parseChangeset(uploadChangesetId); 160 break; 161 default: 162 parseUnknown(); 163 } 164 } else if (event == XMLStreamConstants.END_ELEMENT) 165 return; 166 } 167 } 168 169 private void parseBounds(String generator) throws XMLStreamException { 170 String minlon = parser.getAttributeValue(null, "minlon"); 171 String minlat = parser.getAttributeValue(null, "minlat"); 172 String maxlon = parser.getAttributeValue(null, "maxlon"); 173 String maxlat = parser.getAttributeValue(null, "maxlat"); 174 String origin = parser.getAttributeValue(null, "origin"); 175 if (minlon != null && maxlon != null && minlat != null && maxlat != null) { 176 if (origin == null) { 177 origin = generator; 178 } 179 Bounds bounds = new Bounds( 180 Double.parseDouble(minlat), Double.parseDouble(minlon), 181 Double.parseDouble(maxlat), Double.parseDouble(maxlon)); 182 if (bounds.isOutOfTheWorld()) { 183 Bounds copy = new Bounds(bounds); 184 bounds.normalize(); 185 Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds); 186 } 187 DataSource src = new DataSource(bounds, origin); 188 ds.dataSources.add(src); 189 } else { 190 throwException(tr( 191 "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.", 192 minlon, minlat, maxlon, maxlat, origin 193 )); 194 } 195 jumpToEnd(); 196 } 197 198 protected Node parseNode() throws XMLStreamException { 199 NodeData nd = new NodeData(); 200 String lat = parser.getAttributeValue(null, "lat"); 201 String lon = parser.getAttributeValue(null, "lon"); 202 if (lat != null && lon != null) { 203 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon))); 204 } 205 readCommon(nd); 206 Node n = new Node(nd.getId(), nd.getVersion()); 207 n.setVisible(nd.isVisible()); 208 n.load(nd); 209 externalIdMap.put(nd.getPrimitiveId(), n); 210 while (true) { 211 int event = parser.next(); 212 if (event == XMLStreamConstants.START_ELEMENT) { 213 if ("tag".equals(parser.getLocalName())) { 214 parseTag(n); 215 } else { 216 parseUnknown(); 217 } 218 } else if (event == XMLStreamConstants.END_ELEMENT) 219 return n; 220 } 221 } 222 223 protected Way parseWay() throws XMLStreamException { 224 WayData wd = new WayData(); 225 readCommon(wd); 226 Way w = new Way(wd.getId(), wd.getVersion()); 227 w.setVisible(wd.isVisible()); 228 w.load(wd); 229 externalIdMap.put(wd.getPrimitiveId(), w); 230 231 Collection<Long> nodeIds = new ArrayList<>(); 232 while (true) { 233 int event = parser.next(); 234 if (event == XMLStreamConstants.START_ELEMENT) { 235 switch (parser.getLocalName()) { 236 case "nd": 237 nodeIds.add(parseWayNode(w)); 238 break; 239 case "tag": 240 parseTag(w); 241 break; 242 default: 243 parseUnknown(); 244 } 245 } else if (event == XMLStreamConstants.END_ELEMENT) { 246 break; 247 } 248 } 249 if (w.isDeleted() && !nodeIds.isEmpty()) { 250 Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId())); 251 nodeIds = new ArrayList<>(); 252 } 253 ways.put(wd.getUniqueId(), nodeIds); 254 return w; 255 } 256 257 private long parseWayNode(Way w) throws XMLStreamException { 258 if (parser.getAttributeValue(null, "ref") == null) { 259 throwException( 260 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId()) 261 ); 262 } 263 long id = getLong("ref"); 264 if (id == 0) { 265 throwException( 266 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id) 267 ); 268 } 269 jumpToEnd(); 270 return id; 271 } 272 273 protected Relation parseRelation() throws XMLStreamException { 274 RelationData rd = new RelationData(); 275 readCommon(rd); 276 Relation r = new Relation(rd.getId(), rd.getVersion()); 277 r.setVisible(rd.isVisible()); 278 r.load(rd); 279 externalIdMap.put(rd.getPrimitiveId(), r); 280 281 Collection<RelationMemberData> members = new ArrayList<>(); 282 while (true) { 283 int event = parser.next(); 284 if (event == XMLStreamConstants.START_ELEMENT) { 285 switch (parser.getLocalName()) { 286 case "member": 287 members.add(parseRelationMember(r)); 288 break; 289 case "tag": 290 parseTag(r); 291 break; 292 default: 293 parseUnknown(); 294 } 295 } else if (event == XMLStreamConstants.END_ELEMENT) { 296 break; 297 } 298 } 299 if (r.isDeleted() && !members.isEmpty()) { 300 Main.info(tr("Deleted relation {0} contains members", r.getUniqueId())); 301 members = new ArrayList<>(); 302 } 303 relations.put(rd.getUniqueId(), members); 304 return r; 305 } 306 307 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException { 308 String role = null; 309 OsmPrimitiveType type = null; 310 long id = 0; 311 String value = parser.getAttributeValue(null, "ref"); 312 if (value == null) { 313 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId())); 314 } 315 try { 316 id = Long.parseLong(value); 317 } catch(NumberFormatException e) { 318 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value), e); 319 } 320 value = parser.getAttributeValue(null, "type"); 321 if (value == null) { 322 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId()))); 323 } 324 try { 325 type = OsmPrimitiveType.fromApiTypeName(value); 326 } catch(IllegalArgumentException e) { 327 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value), e); 328 } 329 value = parser.getAttributeValue(null, "role"); 330 role = value; 331 332 if (id == 0) { 333 throwException(tr("Incomplete <member> specification with ref=0")); 334 } 335 jumpToEnd(); 336 return new RelationMemberData(role, type, id); 337 } 338 339 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException { 340 341 Long id = null; 342 if (parser.getAttributeValue(null, "id") != null) { 343 id = getLong("id"); 344 } 345 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value 346 if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) { 347 uploadChangeset = new Changeset(id != null ? id.intValue() : 0); 348 while (true) { 349 int event = parser.next(); 350 if (event == XMLStreamConstants.START_ELEMENT) { 351 if ("tag".equals(parser.getLocalName())) { 352 parseTag(uploadChangeset); 353 } else { 354 parseUnknown(); 355 } 356 } else if (event == XMLStreamConstants.END_ELEMENT) 357 return; 358 } 359 } else { 360 jumpToEnd(false); 361 } 362 } 363 364 private void parseTag(Tagged t) throws XMLStreamException { 365 String key = parser.getAttributeValue(null, "k"); 366 String value = parser.getAttributeValue(null, "v"); 367 if (key == null || value == null) { 368 throwException(tr("Missing key or value attribute in tag.")); 369 } 370 t.put(key.intern(), value.intern()); 371 jumpToEnd(); 372 } 373 374 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 375 if (printWarning) { 376 Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName())); 377 } 378 while (true) { 379 int event = parser.next(); 380 if (event == XMLStreamConstants.START_ELEMENT) { 381 parseUnknown(false); /* no more warning for inner elements */ 382 } else if (event == XMLStreamConstants.END_ELEMENT) 383 return; 384 } 385 } 386 387 protected void parseUnknown() throws XMLStreamException { 388 parseUnknown(true); 389 } 390 391 /** 392 * When cursor is at the start of an element, moves it to the end tag of that element. 393 * Nested content is skipped. 394 * 395 * This is basically the same code as parseUnknown(), except for the warnings, which 396 * are displayed for inner elements and not at top level. 397 */ 398 private void jumpToEnd(boolean printWarning) throws XMLStreamException { 399 while (true) { 400 int event = parser.next(); 401 if (event == XMLStreamConstants.START_ELEMENT) { 402 parseUnknown(printWarning); 403 } else if (event == XMLStreamConstants.END_ELEMENT) 404 return; 405 } 406 } 407 408 private void jumpToEnd() throws XMLStreamException { 409 jumpToEnd(true); 410 } 411 412 private User createUser(String uid, String name) throws XMLStreamException { 413 if (uid == null) { 414 if (name == null) 415 return null; 416 return User.createLocalUser(name); 417 } 418 try { 419 long id = Long.parseLong(uid); 420 return User.createOsmUser(id, name); 421 } catch(NumberFormatException e) { 422 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e); 423 } 424 return null; 425 } 426 427 /** 428 * Read out the common attributes and put them into current OsmPrimitive. 429 */ 430 private void readCommon(PrimitiveData current) throws XMLStreamException { 431 current.setId(getLong("id")); 432 if (current.getUniqueId() == 0) { 433 throwException(tr("Illegal object with ID=0.")); 434 } 435 436 String time = parser.getAttributeValue(null, "timestamp"); 437 if (time != null && time.length() != 0) { 438 current.setTimestamp(DateUtils.fromString(time)); 439 } 440 441 String user = parser.getAttributeValue(null, "user"); 442 String uid = parser.getAttributeValue(null, "uid"); 443 current.setUser(createUser(uid, user)); 444 445 String visible = parser.getAttributeValue(null, "visible"); 446 if (visible != null) { 447 current.setVisible(Boolean.parseBoolean(visible)); 448 } 449 450 String versionString = parser.getAttributeValue(null, "version"); 451 int version = 0; 452 if (versionString != null) { 453 try { 454 version = Integer.parseInt(versionString); 455 } catch(NumberFormatException e) { 456 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", 457 Long.toString(current.getUniqueId()), versionString), e); 458 } 459 switch (ds.getVersion()) { 460 case "0.6": 461 if (version <= 0 && !current.isNew()) { 462 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", 463 Long.toString(current.getUniqueId()), versionString)); 464 } else if (version < 0 && current.isNew()) { 465 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", 466 current.getUniqueId(), version, 0, "0.6")); 467 version = 0; 468 } 469 break; 470 default: 471 // should not happen. API version has been checked before 472 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion())); 473 } 474 } else { 475 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6 476 if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) { 477 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId()))); 478 } 479 } 480 current.setVersion(version); 481 482 String action = parser.getAttributeValue(null, "action"); 483 if (action == null) { 484 // do nothing 485 } else if ("delete".equals(action)) { 486 current.setDeleted(true); 487 current.setModified(current.isVisible()); 488 } else if ("modify".equals(action)) { 489 current.setModified(true); 490 } 491 492 String v = parser.getAttributeValue(null, "changeset"); 493 if (v == null) { 494 current.setChangesetId(0); 495 } else { 496 try { 497 current.setChangesetId(Integer.parseInt(v)); 498 } catch (IllegalArgumentException e) { 499 Main.debug(e.getMessage()); 500 if (current.isNew()) { 501 // for a new primitive we just log a warning 502 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId())); 503 current.setChangesetId(0); 504 } else { 505 // for an existing primitive this is a problem 506 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e); 507 } 508 } catch (IllegalStateException e) { 509 // thrown for positive changeset id on new primitives 510 Main.info(e.getMessage()); 511 current.setChangesetId(0); 512 } 513 if (current.getChangesetId() <= 0) { 514 if (current.isNew()) { 515 // for a new primitive we just log a warning 516 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId())); 517 current.setChangesetId(0); 518 } else { 519 // for an existing primitive this is a problem 520 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); 521 } 522 } 523 } 524 } 525 526 private long getLong(String name) throws XMLStreamException { 527 String value = parser.getAttributeValue(null, name); 528 if (value == null) { 529 throwException(tr("Missing required attribute ''{0}''.",name)); 530 } 531 try { 532 return Long.parseLong(value); 533 } catch(NumberFormatException e) { 534 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value), e); 535 } 536 return 0; // should not happen 537 } 538 539 private static class OsmParsingException extends XMLStreamException { 540 541 public OsmParsingException(String msg, Location location) { 542 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */ 543 this.location = location; 544 } 545 546 public OsmParsingException(String msg, Location location, Throwable th) { 547 super(msg, th); 548 this.location = location; 549 } 550 551 @Override 552 public String getMessage() { 553 String msg = super.getMessage(); 554 if (msg == null) { 555 msg = getClass().getName(); 556 } 557 if (getLocation() == null) 558 return msg; 559 msg += " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber()); 560 int offset = getLocation().getCharacterOffset(); 561 if (offset > -1) { 562 msg += ". "+ tr("{0} bytes have been read", offset); 563 } 564 return msg; 565 } 566 } 567 568 /** 569 * Exception thrown after user cancelation. 570 */ 571 private static final class OsmParsingCanceledException extends OsmParsingException implements ImportCancelException { 572 /** 573 * Constructs a new {@code OsmParsingCanceledException}. 574 * @param msg The error message 575 * @param location The parser location 576 */ 577 public OsmParsingCanceledException(String msg, Location location) { 578 super(msg, location); 579 } 580 } 581 582 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 583 if (progressMonitor == null) { 584 progressMonitor = NullProgressMonitor.INSTANCE; 585 } 586 ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() { 587 @Override public void operationCanceled() { 588 cancel = true; 589 } 590 }; 591 progressMonitor.addCancelListener(cancelListener); 592 CheckParameterUtil.ensureParameterNotNull(source, "source"); 593 try { 594 progressMonitor.beginTask(tr("Prepare OSM data...", 2)); 595 progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); 596 597 try (InputStreamReader ir = UTFInputStreamReader.create(source)) { 598 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir); 599 setParser(parser); 600 parse(); 601 } 602 progressMonitor.worked(1); 603 604 progressMonitor.indeterminateSubTask(tr("Preparing data set...")); 605 prepareDataSet(); 606 progressMonitor.worked(1); 607 608 // iterate over registered postprocessors and give them each a chance 609 // to modify the dataset we have just loaded. 610 if (postprocessors != null) { 611 for (OsmServerReadPostprocessor pp : postprocessors) { 612 pp.postprocessDataSet(getDataSet(), progressMonitor); 613 } 614 } 615 return getDataSet(); 616 } catch(IllegalDataException e) { 617 throw e; 618 } catch(OsmParsingException e) { 619 throw new IllegalDataException(e.getMessage(), e); 620 } catch(XMLStreamException e) { 621 String msg = e.getMessage(); 622 Pattern p = Pattern.compile("Message: (.+)"); 623 Matcher m = p.matcher(msg); 624 if (m.find()) { 625 msg = m.group(1); 626 } 627 if (e.getLocation() != null) 628 throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); 629 else 630 throw new IllegalDataException(msg, e); 631 } catch(Exception e) { 632 throw new IllegalDataException(e); 633 } finally { 634 progressMonitor.finishTask(); 635 progressMonitor.removeCancelListener(cancelListener); 636 } 637 } 638 639 /** 640 * Parse the given input source and return the dataset. 641 * 642 * @param source the source input stream. Must not be null. 643 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 644 * 645 * @return the dataset with the parsed data 646 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source 647 * @throws IllegalArgumentException thrown if source is null 648 */ 649 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 650 return new OsmReader().doParseDataSet(source, progressMonitor); 651 } 652}