001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.net.Authenticator.RequestorType; 009import java.net.HttpURLConnection; 010import java.net.MalformedURLException; 011import java.net.URL; 012import java.util.List; 013 014import javax.xml.parsers.ParserConfigurationException; 015 016import org.openstreetmap.josm.data.gpx.GpxData; 017import org.openstreetmap.josm.data.notes.Note; 018import org.openstreetmap.josm.data.osm.DataSet; 019import org.openstreetmap.josm.gui.progress.ProgressMonitor; 020import org.openstreetmap.josm.io.auth.CredentialsAgentException; 021import org.openstreetmap.josm.io.auth.CredentialsManager; 022import org.openstreetmap.josm.spi.preferences.Config; 023import org.openstreetmap.josm.tools.HttpClient; 024import org.openstreetmap.josm.tools.Logging; 025import org.openstreetmap.josm.tools.Utils; 026import org.openstreetmap.josm.tools.XmlParsingException; 027import org.openstreetmap.josm.tools.XmlUtils; 028import org.w3c.dom.Document; 029import org.w3c.dom.Node; 030import org.xml.sax.SAXException; 031 032/** 033 * This DataReader reads directly from the REST API of the osm server. 034 * 035 * It supports plain text transfer as well as gzip or deflate encoded transfers; 036 * if compressed transfers are unwanted, set property osm-server.use-compression 037 * to false. 038 * 039 * @author imi 040 */ 041public abstract class OsmServerReader extends OsmConnection { 042 private final OsmApi api = OsmApi.getOsmApi(); 043 private boolean doAuthenticate; 044 protected boolean gpxParsedProperly; 045 protected String contentType; 046 047 /** 048 * Constructs a new {@code OsmServerReader}. 049 */ 050 public OsmServerReader() { 051 try { 052 doAuthenticate = OsmApi.isUsingOAuth() && CredentialsManager.getInstance().lookupOAuthAccessToken() != null; 053 } catch (CredentialsAgentException e) { 054 Logging.warn(e); 055 } 056 } 057 058 /** 059 * Open a connection to the given url and return a reader on the input stream 060 * from that connection. In case of user cancel, return <code>null</code>. 061 * Relative URL's are directed to API base URL. 062 * @param urlStr The url to connect to. 063 * @param progressMonitor progress monitoring and abort handler 064 * @return A reader reading the input stream (servers answer) or <code>null</code>. 065 * @throws OsmTransferException if data transfer errors occur 066 */ 067 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException { 068 return getInputStream(urlStr, progressMonitor, null); 069 } 070 071 /** 072 * Open a connection to the given url and return a reader on the input stream 073 * from that connection. In case of user cancel, return <code>null</code>. 074 * Relative URL's are directed to API base URL. 075 * @param urlStr The url to connect to. 076 * @param progressMonitor progress monitoring and abort handler 077 * @param reason The reason to show on console. Can be {@code null} if no reason is given 078 * @return A reader reading the input stream (servers answer) or <code>null</code>. 079 * @throws OsmTransferException if data transfer errors occur 080 */ 081 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException { 082 try { 083 api.initialize(progressMonitor); 084 String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr); 085 return getInputStreamRaw(url, progressMonitor, reason); 086 } finally { 087 progressMonitor.invalidate(); 088 } 089 } 090 091 /** 092 * Return the base URL for relative URL requests 093 * @return base url of API 094 */ 095 protected String getBaseUrl() { 096 return api.getBaseUrl(); 097 } 098 099 /** 100 * Open a connection to the given url and return a reader on the input stream 101 * from that connection. In case of user cancel, return <code>null</code>. 102 * @param urlStr The exact url to connect to. 103 * @param progressMonitor progress monitoring and abort handler 104 * @return An reader reading the input stream (servers answer) or <code>null</code>. 105 * @throws OsmTransferException if data transfer errors occur 106 */ 107 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException { 108 return getInputStreamRaw(urlStr, progressMonitor, null); 109 } 110 111 /** 112 * Open a connection to the given url and return a reader on the input stream 113 * from that connection. In case of user cancel, return <code>null</code>. 114 * @param urlStr The exact url to connect to. 115 * @param progressMonitor progress monitoring and abort handler 116 * @param reason The reason to show on console. Can be {@code null} if no reason is given 117 * @return An reader reading the input stream (servers answer) or <code>null</code>. 118 * @throws OsmTransferException if data transfer errors occur 119 */ 120 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException { 121 return getInputStreamRaw(urlStr, progressMonitor, reason, false); 122 } 123 124 /** 125 * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream 126 * from that connection. In case of user cancel, return <code>null</code>. 127 * @param urlStr The exact url to connect to. 128 * @param progressMonitor progress monitoring and abort handler 129 * @param reason The reason to show on console. Can be {@code null} if no reason is given 130 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition} 131 * for {@code filename} and uncompress a gzip/bzip2/xz/zip stream. 132 * @return An reader reading the input stream (servers answer) or <code>null</code>. 133 * @throws OsmTransferException if data transfer errors occur 134 */ 135 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 136 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 137 return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null); 138 } 139 140 /** 141 * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream 142 * from that connection. In case of user cancel, return <code>null</code>. 143 * @param urlStr The exact url to connect to. 144 * @param progressMonitor progress monitoring and abort handler 145 * @param reason The reason to show on console. Can be {@code null} if no reason is given 146 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition} 147 * for {@code filename} and uncompress a gzip/bzip2/xz/zip stream. 148 * @param httpMethod HTTP method ("GET", "POST" or "PUT") 149 * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method. 150 * @return An reader reading the input stream (servers answer) or <code>null</code>. 151 * @throws OsmTransferException if data transfer errors occur 152 * @since 12596 153 */ 154 @SuppressWarnings("resource") 155 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 156 boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException { 157 try { 158 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Config.getUrls().getJOSMWebsite()); 159 OnlineResource.OSM_API.checkOfflineAccess(urlStr, OsmApi.getOsmApi().getServerUrl()); 160 161 URL url = null; 162 try { 163 url = new URL(urlStr.replace(" ", "%20")); 164 } catch (MalformedURLException e) { 165 throw new OsmTransferException(e); 166 } 167 168 String protocol = url.getProtocol(); 169 if ("file".equals(protocol) || "jar".equals(protocol)) { 170 try { 171 return Utils.openStream(url); 172 } catch (IOException e) { 173 throw new OsmTransferException(e); 174 } 175 } 176 177 final HttpClient client = HttpClient.create(url, httpMethod) 178 .setFinishOnCloseOutput(false) 179 .setReasonForRequest(reason) 180 .setOutputMessage(tr("Downloading data...")) 181 .setRequestBody(requestBody); 182 activeConnection = client; 183 adaptRequest(client); 184 if (doAuthenticate) { 185 addAuth(client); 186 } 187 if (cancel) 188 throw new OsmTransferCanceledException("Operation canceled"); 189 190 final HttpClient.Response response; 191 try { 192 response = client.connect(progressMonitor); 193 contentType = response.getContentType(); 194 } catch (IOException e) { 195 Logging.error(e); 196 OsmTransferException ote = new OsmTransferException( 197 tr("Could not connect to the OSM server. Please check your internet connection."), e); 198 ote.setUrl(url.toString()); 199 throw ote; 200 } 201 try { 202 if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) { 203 CredentialsManager.getInstance().purgeCredentialsCache(RequestorType.SERVER); 204 throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null); 205 } 206 207 if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH) 208 throw new OsmTransferCanceledException("Proxy Authentication Required"); 209 210 if (response.getResponseCode() != HttpURLConnection.HTTP_OK) { 211 String errorHeader = response.getHeaderField("Error"); 212 String errorBody = fetchResponseText(response); 213 throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString(), null, 214 contentType); 215 } 216 217 response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition); 218 return response.getContent(); 219 } catch (OsmTransferException e) { 220 throw e; 221 } catch (IOException e) { 222 throw new OsmTransferException(e); 223 } 224 } finally { 225 progressMonitor.invalidate(); 226 } 227 } 228 229 private static String fetchResponseText(final HttpClient.Response response) { 230 try { 231 return response.fetchContent(); 232 } catch (IOException e) { 233 Logging.error(e); 234 return tr("Reading error text failed."); 235 } 236 } 237 238 /** 239 * Allows subclasses to modify the request. 240 * @param request the prepared request 241 * @since 9308 242 */ 243 protected void adaptRequest(HttpClient request) { 244 } 245 246 /** 247 * Download OSM files from somewhere 248 * @param progressMonitor The progress monitor 249 * @return The corresponding dataset 250 * @throws OsmTransferException if any error occurs 251 */ 252 public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException; 253 254 /** 255 * Download compressed OSM files from somewhere 256 * @param progressMonitor The progress monitor 257 * @param compression compression to use 258 * @return The corresponding dataset 259 * @throws OsmTransferException if any error occurs 260 * @since 13352 261 */ 262 public DataSet parseOsm(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 263 return null; 264 } 265 266 /** 267 * Download OSM Change uncompressed files from somewhere 268 * @param progressMonitor The progress monitor 269 * @return The corresponding dataset 270 * @throws OsmTransferException if any error occurs 271 */ 272 public DataSet parseOsmChange(ProgressMonitor progressMonitor) throws OsmTransferException { 273 return null; 274 } 275 276 /** 277 * Download OSM Change compressed files from somewhere 278 * @param progressMonitor The progress monitor 279 * @param compression compression to use 280 * @return The corresponding dataset 281 * @throws OsmTransferException if any error occurs 282 * @since 13352 283 */ 284 public DataSet parseOsmChange(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 285 return null; 286 } 287 288 /** 289 * Retrieve raw gps waypoints from the server API. 290 * @param progressMonitor The progress monitor 291 * @return The corresponding GPX tracks 292 * @throws OsmTransferException if any error occurs 293 */ 294 public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException { 295 return null; 296 } 297 298 /** 299 * Retrieve compressed GPX files from somewhere. 300 * @param progressMonitor The progress monitor 301 * @param compression compression to use 302 * @return The corresponding GPX tracks 303 * @throws OsmTransferException if any error occurs 304 * @since 13352 305 */ 306 public GpxData parseRawGps(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 307 return null; 308 } 309 310 /** 311 * Returns true if this reader is adding authentication credentials to the read 312 * request sent to the server. 313 * 314 * @return true if this reader is adding authentication credentials to the read 315 * request sent to the server 316 */ 317 public boolean isDoAuthenticate() { 318 return doAuthenticate; 319 } 320 321 /** 322 * Sets whether this reader adds authentication credentials to the read 323 * request sent to the server. 324 * 325 * @param doAuthenticate true if this reader adds authentication credentials to the read 326 * request sent to the server 327 */ 328 public void setDoAuthenticate(boolean doAuthenticate) { 329 this.doAuthenticate = doAuthenticate; 330 } 331 332 /** 333 * Determines if the GPX data has been parsed properly. 334 * @return true if the GPX data has been parsed properly, false otherwise 335 * @see GpxReader#parse 336 */ 337 public final boolean isGpxParsedProperly() { 338 return gpxParsedProperly; 339 } 340 341 /** 342 * Downloads notes from the API, given API limit parameters 343 * 344 * @param noteLimit How many notes to download. 345 * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes. 346 * @param progressMonitor Progress monitor for user feedback 347 * @return List of notes returned by the API 348 * @throws OsmTransferException if any errors happen 349 */ 350 public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException { 351 return null; 352 } 353 354 /** 355 * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added 356 * 357 * @param progressMonitor progress monitor 358 * @return A list of notes parsed from the URL 359 * @throws OsmTransferException if any error occurs during dialog with OSM API 360 */ 361 public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException { 362 return null; 363 } 364 365 /** 366 * Download notes from a URL that contains a compressed notes dump file 367 * @param progressMonitor progress monitor 368 * @param compression compression to use 369 * @return A list of notes parsed from the URL 370 * @throws OsmTransferException if any error occurs during dialog with OSM API 371 * @since 13352 372 */ 373 public List<Note> parseRawNotes(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 374 return null; 375 } 376 377 /** 378 * Returns an attribute from the given DOM node. 379 * @param node DOM node 380 * @param name attribute name 381 * @return attribute value for the given attribute 382 * @since 12510 383 */ 384 protected static String getAttribute(Node node, String name) { 385 return node.getAttributes().getNamedItem(name).getNodeValue(); 386 } 387 388 /** 389 * DOM document parser. 390 * @param <R> resulting type 391 * @since 12510 392 */ 393 @FunctionalInterface 394 protected interface DomParser<R> { 395 /** 396 * Parses a given DOM document. 397 * @param doc DOM document 398 * @return parsed data 399 * @throws XmlParsingException if an XML parsing error occurs 400 */ 401 R parse(Document doc) throws XmlParsingException; 402 } 403 404 /** 405 * Fetches generic data from the DOM document resulting an API call. 406 * @param api the OSM API call 407 * @param subtask the subtask translated message 408 * @param parser the parser converting the DOM document (OSM API result) 409 * @param <T> data type 410 * @param monitor The progress monitor 411 * @param reason The reason to show on console. Can be {@code null} if no reason is given 412 * @return The converted data 413 * @throws OsmTransferException if something goes wrong 414 * @since 12510 415 */ 416 public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason) 417 throws OsmTransferException { 418 try { 419 monitor.beginTask(""); 420 monitor.indeterminateSubTask(subtask); 421 try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) { 422 return parser.parse(XmlUtils.parseSafeDOM(in)); 423 } 424 } catch (OsmTransferException e) { 425 throw e; 426 } catch (IOException | ParserConfigurationException | SAXException e) { 427 throw new OsmTransferException(e); 428 } finally { 429 monitor.finishTask(); 430 } 431 } 432}