001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.awt.Rectangle; 007import java.io.BufferedReader; 008import java.io.IOException; 009import java.io.InputStream; 010import java.io.Reader; 011import java.io.StringReader; 012import java.lang.reflect.Method; 013import java.text.MessageFormat; 014import java.util.ArrayList; 015import java.util.Collection; 016import java.util.Collections; 017import java.util.HashMap; 018import java.util.HashSet; 019import java.util.Iterator; 020import java.util.LinkedHashMap; 021import java.util.LinkedHashSet; 022import java.util.LinkedList; 023import java.util.List; 024import java.util.Locale; 025import java.util.Map; 026import java.util.Objects; 027import java.util.Optional; 028import java.util.Set; 029import java.util.function.Predicate; 030import java.util.regex.Matcher; 031import java.util.regex.Pattern; 032 033import org.openstreetmap.josm.command.ChangePropertyCommand; 034import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 035import org.openstreetmap.josm.command.Command; 036import org.openstreetmap.josm.command.DeleteCommand; 037import org.openstreetmap.josm.command.SequenceCommand; 038import org.openstreetmap.josm.data.coor.LatLon; 039import org.openstreetmap.josm.data.osm.DataSet; 040import org.openstreetmap.josm.data.osm.IPrimitive; 041import org.openstreetmap.josm.data.osm.OsmPrimitive; 042import org.openstreetmap.josm.data.osm.OsmUtils; 043import org.openstreetmap.josm.data.osm.Relation; 044import org.openstreetmap.josm.data.osm.Tag; 045import org.openstreetmap.josm.data.osm.Way; 046import org.openstreetmap.josm.data.preferences.sources.SourceEntry; 047import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 048import org.openstreetmap.josm.data.validation.OsmValidator; 049import org.openstreetmap.josm.data.validation.Severity; 050import org.openstreetmap.josm.data.validation.Test; 051import org.openstreetmap.josm.data.validation.TestError; 052import org.openstreetmap.josm.gui.mappaint.Environment; 053import org.openstreetmap.josm.gui.mappaint.Keyword; 054import org.openstreetmap.josm.gui.mappaint.MultiCascade; 055import org.openstreetmap.josm.gui.mappaint.mapcss.Condition; 056import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition; 057import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition; 058import org.openstreetmap.josm.gui.mappaint.mapcss.Expression; 059import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction; 060import org.openstreetmap.josm.gui.mappaint.mapcss.Functions; 061import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction; 062import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression; 063import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule; 064import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration; 065import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource; 066import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex; 067import org.openstreetmap.josm.gui.mappaint.mapcss.Selector; 068import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector; 069import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector; 070import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector; 071import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser; 072import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException; 073import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError; 074import org.openstreetmap.josm.gui.progress.ProgressMonitor; 075import org.openstreetmap.josm.io.CachedFile; 076import org.openstreetmap.josm.io.FileWatcher; 077import org.openstreetmap.josm.io.IllegalDataException; 078import org.openstreetmap.josm.io.UTFInputStreamReader; 079import org.openstreetmap.josm.spi.preferences.Config; 080import org.openstreetmap.josm.tools.CheckParameterUtil; 081import org.openstreetmap.josm.tools.DefaultGeoProperty; 082import org.openstreetmap.josm.tools.GeoProperty; 083import org.openstreetmap.josm.tools.GeoPropertyIndex; 084import org.openstreetmap.josm.tools.I18n; 085import org.openstreetmap.josm.tools.Logging; 086import org.openstreetmap.josm.tools.MultiMap; 087import org.openstreetmap.josm.tools.Territories; 088import org.openstreetmap.josm.tools.Utils; 089 090/** 091 * MapCSS-based tag checker/fixer. 092 * @since 6506 093 */ 094public class MapCSSTagChecker extends Test.TagTest { 095 MapCSSTagCheckerIndex indexData; 096 final Set<OsmPrimitive> tested = new HashSet<>(); 097 098 099 /** 100 * A grouped MapCSSRule with multiple selectors for a single declaration. 101 * @see MapCSSRule 102 */ 103 public static class GroupedMapCSSRule { 104 /** MapCSS selectors **/ 105 public final List<Selector> selectors; 106 /** MapCSS declaration **/ 107 public final Declaration declaration; 108 109 /** 110 * Constructs a new {@code GroupedMapCSSRule}. 111 * @param selectors MapCSS selectors 112 * @param declaration MapCSS declaration 113 */ 114 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) { 115 this.selectors = selectors; 116 this.declaration = declaration; 117 } 118 119 @Override 120 public int hashCode() { 121 return Objects.hash(selectors, declaration); 122 } 123 124 @Override 125 public boolean equals(Object obj) { 126 if (this == obj) return true; 127 if (obj == null || getClass() != obj.getClass()) return false; 128 GroupedMapCSSRule that = (GroupedMapCSSRule) obj; 129 return Objects.equals(selectors, that.selectors) && 130 Objects.equals(declaration, that.declaration); 131 } 132 133 @Override 134 public String toString() { 135 return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']'; 136 } 137 } 138 139 /** 140 * The preference key for tag checker source entries. 141 * @since 6670 142 */ 143 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries"; 144 145 /** 146 * Constructs a new {@code MapCSSTagChecker}. 147 */ 148 public MapCSSTagChecker() { 149 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values.")); 150 } 151 152 /** 153 * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}. 154 */ 155 @FunctionalInterface 156 interface FixCommand { 157 /** 158 * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders 159 * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}). 160 * @param p OSM primitive 161 * @param matchingSelector matching selector 162 * @return fix command 163 */ 164 Command createCommand(OsmPrimitive p, Selector matchingSelector); 165 166 /** 167 * Checks that object is either an {@link Expression} or a {@link String}. 168 * @param obj object to check 169 * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String} 170 */ 171 static void checkObject(final Object obj) { 172 CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String, 173 () -> "instance of Exception or String expected, but got " + obj); 174 } 175 176 /** 177 * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}. 178 * @param obj object to evaluate ({@link Expression} or {@link String}) 179 * @param p OSM primitive 180 * @param matchingSelector matching selector 181 * @return result string 182 */ 183 static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) { 184 final String s; 185 if (obj instanceof Expression) { 186 s = (String) ((Expression) obj).evaluate(new Environment(p)); 187 } else if (obj instanceof String) { 188 s = (String) obj; 189 } else { 190 return null; 191 } 192 return TagCheck.insertArguments(matchingSelector, s, p); 193 } 194 195 /** 196 * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag. 197 * @param obj object to evaluate ({@link Expression} or {@link String}) 198 * @return created fix command 199 */ 200 static FixCommand fixAdd(final Object obj) { 201 checkObject(obj); 202 return new FixCommand() { 203 @Override 204 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 205 final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector)); 206 return new ChangePropertyCommand(p, tag.getKey(), tag.getValue()); 207 } 208 209 @Override 210 public String toString() { 211 return "fixAdd: " + obj; 212 } 213 }; 214 } 215 216 /** 217 * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key. 218 * @param obj object to evaluate ({@link Expression} or {@link String}) 219 * @return created fix command 220 */ 221 static FixCommand fixRemove(final Object obj) { 222 checkObject(obj); 223 return new FixCommand() { 224 @Override 225 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 226 final String key = evaluateObject(obj, p, matchingSelector); 227 return new ChangePropertyCommand(p, key, ""); 228 } 229 230 @Override 231 public String toString() { 232 return "fixRemove: " + obj; 233 } 234 }; 235 } 236 237 /** 238 * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys. 239 * @param oldKey old key 240 * @param newKey new key 241 * @return created fix command 242 */ 243 static FixCommand fixChangeKey(final String oldKey, final String newKey) { 244 return new FixCommand() { 245 @Override 246 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 247 return new ChangePropertyKeyCommand(p, 248 TagCheck.insertArguments(matchingSelector, oldKey, p), 249 TagCheck.insertArguments(matchingSelector, newKey, p)); 250 } 251 252 @Override 253 public String toString() { 254 return "fixChangeKey: " + oldKey + " => " + newKey; 255 } 256 }; 257 } 258 } 259 260 final MultiMap<String, TagCheck> checks = new MultiMap<>(); 261 262 /** 263 * Result of {@link TagCheck#readMapCSS} 264 * @since 8936 265 */ 266 public static class ParseResult { 267 /** Checks successfully parsed */ 268 public final List<TagCheck> parseChecks; 269 /** Errors that occurred during parsing */ 270 public final Collection<Throwable> parseErrors; 271 272 /** 273 * Constructs a new {@code ParseResult}. 274 * @param parseChecks Checks successfully parsed 275 * @param parseErrors Errors that occurred during parsing 276 */ 277 public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) { 278 this.parseChecks = parseChecks; 279 this.parseErrors = parseErrors; 280 } 281 } 282 283 /** 284 * Tag check. 285 */ 286 public static class TagCheck implements Predicate<OsmPrimitive> { 287 /** The selector of this {@code TagCheck} */ 288 protected final GroupedMapCSSRule rule; 289 /** Commands to apply in order to fix a matching primitive */ 290 protected final List<FixCommand> fixCommands = new ArrayList<>(); 291 /** Tags (or arbitraty strings) of alternatives to be presented to the user */ 292 protected final List<String> alternatives = new ArrayList<>(); 293 /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair. 294 * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */ 295 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>(); 296 /** Unit tests */ 297 protected final Map<String, Boolean> assertions = new HashMap<>(); 298 /** MapCSS Classes to set on matching primitives */ 299 protected final Set<String> setClassExpressions = new HashSet<>(); 300 /** Denotes whether the object should be deleted for fixing it */ 301 protected boolean deletion; 302 /** A string used to group similar tests */ 303 protected String group; 304 305 TagCheck(GroupedMapCSSRule rule) { 306 this.rule = rule; 307 } 308 309 private static final String POSSIBLE_THROWS = possibleThrows(); 310 311 static final String possibleThrows() { 312 StringBuilder sb = new StringBuilder(); 313 for (Severity s : Severity.values()) { 314 if (sb.length() > 0) { 315 sb.append('/'); 316 } 317 sb.append("throw") 318 .append(s.name().charAt(0)) 319 .append(s.name().substring(1).toLowerCase(Locale.ENGLISH)); 320 } 321 return sb.toString(); 322 } 323 324 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException { 325 final TagCheck check = new TagCheck(rule); 326 for (Instruction i : rule.declaration.instructions) { 327 if (i instanceof Instruction.AssignmentInstruction) { 328 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i; 329 if (ai.isSetInstruction) { 330 check.setClassExpressions.add(ai.key); 331 continue; 332 } 333 try { 334 final String val = ai.val instanceof Expression 335 ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null) 336 : ai.val instanceof String 337 ? (String) ai.val 338 : ai.val instanceof Keyword 339 ? ((Keyword) ai.val).val 340 : null; 341 if (ai.key.startsWith("throw")) { 342 try { 343 check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH))); 344 } catch (IllegalArgumentException e) { 345 Logging.log(Logging.LEVEL_WARN, 346 "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e); 347 } 348 } else if ("fixAdd".equals(ai.key)) { 349 check.fixCommands.add(FixCommand.fixAdd(ai.val)); 350 } else if ("fixRemove".equals(ai.key)) { 351 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")), 352 "Unexpected '='. Please only specify the key to remove in: " + ai); 353 check.fixCommands.add(FixCommand.fixRemove(ai.val)); 354 } else if (val != null && "fixChangeKey".equals(ai.key)) { 355 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!"); 356 final String[] x = val.split("=>", 2); 357 check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1]))); 358 } else if (val != null && "fixDeleteObject".equals(ai.key)) { 359 CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'"); 360 check.deletion = true; 361 } else if (val != null && "suggestAlternative".equals(ai.key)) { 362 check.alternatives.add(val); 363 } else if (val != null && "assertMatch".equals(ai.key)) { 364 check.assertions.put(val, Boolean.TRUE); 365 } else if (val != null && "assertNoMatch".equals(ai.key)) { 366 check.assertions.put(val, Boolean.FALSE); 367 } else if (val != null && "group".equals(ai.key)) { 368 check.group = val; 369 } else if (ai.key.startsWith("-")) { 370 Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val); 371 } else { 372 throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!'); 373 } 374 } catch (IllegalArgumentException e) { 375 throw new IllegalDataException(e); 376 } 377 } 378 } 379 if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) { 380 throw new IllegalDataException( 381 "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors); 382 } else if (check.errors.size() > 1) { 383 throw new IllegalDataException( 384 "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " 385 + rule.selectors); 386 } 387 return check; 388 } 389 390 static ParseResult readMapCSS(Reader css) throws ParseException { 391 CheckParameterUtil.ensureParameterNotNull(css, "css"); 392 393 final MapCSSStyleSource source = new MapCSSStyleSource(""); 394 final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR); 395 final StringReader mapcss = new StringReader(preprocessor.pp_root(source)); 396 final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT); 397 parser.sheet(source); 398 // Ignore "meta" rule(s) from external rules of JOSM wiki 399 source.removeMetaRules(); 400 // group rules with common declaration block 401 Map<Declaration, List<Selector>> g = new LinkedHashMap<>(); 402 for (MapCSSRule rule : source.rules) { 403 if (!g.containsKey(rule.declaration)) { 404 List<Selector> sels = new ArrayList<>(); 405 sels.add(rule.selector); 406 g.put(rule.declaration, sels); 407 } else { 408 g.get(rule.declaration).add(rule.selector); 409 } 410 } 411 List<TagCheck> parseChecks = new ArrayList<>(); 412 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) { 413 try { 414 parseChecks.add(TagCheck.ofMapCSSRule( 415 new GroupedMapCSSRule(map.getValue(), map.getKey()))); 416 } catch (IllegalDataException e) { 417 Logging.error("Cannot add MapCss rule: "+e.getMessage()); 418 source.logError(e); 419 } 420 } 421 return new ParseResult(parseChecks, source.getErrors()); 422 } 423 424 @Override 425 public boolean test(OsmPrimitive primitive) { 426 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker. 427 return whichSelectorMatchesPrimitive(primitive) != null; 428 } 429 430 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) { 431 return whichSelectorMatchesEnvironment(new Environment(primitive)); 432 } 433 434 Selector whichSelectorMatchesEnvironment(Environment env) { 435 for (Selector i : rule.selectors) { 436 env.clearSelectorMatchingInformation(); 437 if (i.matches(env)) { 438 return i; 439 } 440 } 441 return null; 442 } 443 444 /** 445 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the 446 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}. 447 * @param matchingSelector matching selector 448 * @param index index 449 * @param type selector type ("key", "value" or "tag") 450 * @param p OSM primitive 451 * @return argument value, can be {@code null} 452 */ 453 static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) { 454 try { 455 final Condition c = matchingSelector.getConditions().get(index); 456 final Tag tag = c instanceof Condition.ToTagConvertable 457 ? ((Condition.ToTagConvertable) c).asTag(p) 458 : null; 459 if (tag == null) { 460 return null; 461 } else if ("key".equals(type)) { 462 return tag.getKey(); 463 } else if ("value".equals(type)) { 464 return tag.getValue(); 465 } else if ("tag".equals(type)) { 466 return tag.toString(); 467 } 468 } catch (IndexOutOfBoundsException ignore) { 469 Logging.debug(ignore); 470 } 471 return null; 472 } 473 474 /** 475 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding 476 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}. 477 * @param matchingSelector matching selector 478 * @param s any string 479 * @param p OSM primitive 480 * @return string with arguments inserted 481 */ 482 static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) { 483 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) { 484 return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p); 485 } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) { 486 return s; 487 } 488 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s); 489 final StringBuffer sb = new StringBuffer(); 490 while (m.find()) { 491 final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector, 492 Integer.parseInt(m.group(1)), m.group(2), p); 493 try { 494 // Perform replacement with null-safe + regex-safe handling 495 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", "")); 496 } catch (IndexOutOfBoundsException | IllegalArgumentException e) { 497 Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e); 498 } 499 } 500 m.appendTail(sb); 501 return sb.toString(); 502 } 503 504 /** 505 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive} 506 * if the error is fixable, or {@code null} otherwise. 507 * 508 * @param p the primitive to construct the fix for 509 * @return the fix or {@code null} 510 */ 511 Command fixPrimitive(OsmPrimitive p) { 512 if (fixCommands.isEmpty() && !deletion) { 513 return null; 514 } 515 try { 516 final Selector matchingSelector = whichSelectorMatchesPrimitive(p); 517 Collection<Command> cmds = new LinkedList<>(); 518 for (FixCommand fixCommand : fixCommands) { 519 cmds.add(fixCommand.createCommand(p, matchingSelector)); 520 } 521 if (deletion && !p.isDeleted()) { 522 cmds.add(new DeleteCommand(p)); 523 } 524 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds); 525 } catch (IllegalArgumentException e) { 526 Logging.error(e); 527 return null; 528 } 529 } 530 531 /** 532 * Constructs a (localized) message for this deprecation check. 533 * @param p OSM primitive 534 * 535 * @return a message 536 */ 537 String getMessage(OsmPrimitive p) { 538 if (errors.isEmpty()) { 539 // Return something to avoid NPEs 540 return rule.declaration.toString(); 541 } else { 542 final Object val = errors.keySet().iterator().next().val; 543 return String.valueOf( 544 val instanceof Expression 545 ? ((Expression) val).evaluate(new Environment(p)) 546 : val 547 ); 548 } 549 } 550 551 /** 552 * Constructs a (localized) description for this deprecation check. 553 * @param p OSM primitive 554 * 555 * @return a description (possibly with alternative suggestions) 556 * @see #getDescriptionForMatchingSelector 557 */ 558 String getDescription(OsmPrimitive p) { 559 if (alternatives.isEmpty()) { 560 return getMessage(p); 561 } else { 562 /* I18N: {0} is the test error message and {1} is an alternative */ 563 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives)); 564 } 565 } 566 567 /** 568 * Constructs a (localized) description for this deprecation check 569 * where any placeholders are replaced by values of the matched selector. 570 * 571 * @param matchingSelector matching selector 572 * @param p OSM primitive 573 * @return a description (possibly with alternative suggestions) 574 */ 575 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) { 576 return insertArguments(matchingSelector, getDescription(p), p); 577 } 578 579 Severity getSeverity() { 580 return errors.isEmpty() ? null : errors.values().iterator().next(); 581 } 582 583 @Override 584 public String toString() { 585 return getDescription(null); 586 } 587 588 /** 589 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error. 590 * 591 * @param p the primitive to construct the error for 592 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error. 593 */ 594 List<TestError> getErrorsForPrimitive(OsmPrimitive p) { 595 final Environment env = new Environment(p); 596 return getErrorsForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null); 597 } 598 599 private List<TestError> getErrorsForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) { 600 List<TestError> res = new ArrayList<>(); 601 if (matchingSelector != null && !errors.isEmpty()) { 602 final Command fix = fixPrimitive(p); 603 final String description = getDescriptionForMatchingSelector(p, matchingSelector); 604 final String description1 = group == null ? description : group; 605 final String description2 = group == null ? null : description; 606 TestError.Builder errorBuilder = TestError.builder(tester, getSeverity(), 3000) 607 .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString()); 608 if (fix != null) { 609 errorBuilder = errorBuilder.fix(() -> fix); 610 } 611 if (env.child instanceof OsmPrimitive) { 612 res.add(errorBuilder.primitives(p, (OsmPrimitive) env.child).build()); 613 } else if (env.children != null) { 614 for (IPrimitive c : env.children) { 615 if (c instanceof OsmPrimitive) { 616 errorBuilder = TestError.builder(tester, getSeverity(), 3000) 617 .messageWithManuallyTranslatedDescription(description1, description2, 618 matchingSelector.toString()); 619 if (fix != null) { 620 errorBuilder = errorBuilder.fix(() -> fix); 621 } 622 res.add(errorBuilder.primitives(p, (OsmPrimitive) c).build()); 623 } 624 } 625 } else { 626 res.add(errorBuilder.primitives(p).build()); 627 } 628 } 629 return res; 630 } 631 632 /** 633 * Returns the set of tagchecks on which this check depends on. 634 * @param schecks the collection of tagcheks to search in 635 * @return the set of tagchecks on which this check depends on 636 * @since 7881 637 */ 638 public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) { 639 Set<TagCheck> result = new HashSet<>(); 640 Set<String> classes = getClassesIds(); 641 if (schecks != null && !classes.isEmpty()) { 642 for (TagCheck tc : schecks) { 643 if (this.equals(tc)) { 644 continue; 645 } 646 for (String id : tc.setClassExpressions) { 647 if (classes.contains(id)) { 648 result.add(tc); 649 break; 650 } 651 } 652 } 653 } 654 return result; 655 } 656 657 /** 658 * Returns the list of ids of all MapCSS classes referenced in the rule selectors. 659 * @return the list of ids of all MapCSS classes referenced in the rule selectors 660 * @since 7881 661 */ 662 public Set<String> getClassesIds() { 663 Set<String> result = new HashSet<>(); 664 for (Selector s : rule.selectors) { 665 if (s instanceof AbstractSelector) { 666 for (Condition c : ((AbstractSelector) s).getConditions()) { 667 if (c instanceof ClassCondition) { 668 result.add(((ClassCondition) c).id); 669 } 670 } 671 } 672 } 673 return result; 674 } 675 } 676 677 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker { 678 public final GroupedMapCSSRule rule; 679 680 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) { 681 this.rule = rule; 682 } 683 684 @Override 685 public synchronized boolean equals(Object obj) { 686 return super.equals(obj) 687 || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule)) 688 || (obj instanceof GroupedMapCSSRule && rule.equals(obj)); 689 } 690 691 @Override 692 public synchronized int hashCode() { 693 return Objects.hash(super.hashCode(), rule); 694 } 695 696 @Override 697 public String toString() { 698 return "MapCSSTagCheckerAndRule [rule=" + rule + ']'; 699 } 700 } 701 702 /** 703 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}. 704 * @param p The OSM primitive 705 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned 706 * @return all errors for the given primitive, with or without those of "info" severity 707 */ 708 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) { 709 final List<TestError> res = new ArrayList<>(); 710 if (indexData == null) { 711 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverity, MapCSSTagCheckerIndex.ALL_TESTS); 712 } 713 714 MapCSSRuleIndex matchingRuleIndex = indexData.get(p); 715 716 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 717 // the declaration indices are sorted, so it suffices to save the last used index 718 Declaration lastDeclUsed = null; 719 720 Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p); 721 while (candidates.hasNext()) { 722 MapCSSRule r = candidates.next(); 723 env.clearSelectorMatchingInformation(); 724 if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector) 725 TagCheck check = indexData.getCheck(r); 726 if (check != null) { 727 if (r.declaration == lastDeclUsed) 728 continue; // don't apply one declaration more than once 729 lastDeclUsed = r.declaration; 730 731 r.declaration.execute(env); 732 if (!check.errors.isEmpty()) { 733 for (TestError e: check.getErrorsForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule))) { 734 addIfNotSimilar(e, res); 735 } 736 } 737 } 738 } 739 } 740 return res; 741 } 742 743 /** 744 * See #12627 745 * Add error to given list if list doesn't already contain a similar error. 746 * Similar means same code and description and same combination of primitives and same combination of highlighted objects, 747 * but maybe with different orders. 748 * @param toAdd the error to add 749 * @param errors the list of errors 750 */ 751 private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) { 752 boolean isDup = false; 753 if (toAdd.getPrimitives().size() >= 2) { 754 for (TestError e : errors) { 755 if (e.getCode() == toAdd.getCode() && e.getMessage().equals(toAdd.getMessage()) 756 && e.getPrimitives().size() == toAdd.getPrimitives().size() 757 && e.getPrimitives().containsAll(toAdd.getPrimitives()) 758 && e.getHighlighted().size() == toAdd.getHighlighted().size() 759 && e.getHighlighted().containsAll(toAdd.getHighlighted())) { 760 isDup = true; 761 break; 762 } 763 } 764 } 765 if (!isDup) 766 errors.add(toAdd); 767 } 768 769 private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity, 770 Collection<Set<TagCheck>> checksCol) { 771 final List<TestError> r = new ArrayList<>(); 772 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 773 for (Set<TagCheck> schecks : checksCol) { 774 for (TagCheck check : schecks) { 775 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity; 776 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class 777 if (ignoreError && check.setClassExpressions.isEmpty()) { 778 continue; 779 } 780 final Selector selector = check.whichSelectorMatchesEnvironment(env); 781 if (selector != null) { 782 check.rule.declaration.execute(env); 783 if (!ignoreError && !check.errors.isEmpty()) { 784 r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule))); 785 } 786 } 787 } 788 } 789 return r; 790 } 791 792 /** 793 * Visiting call for primitives. 794 * 795 * @param p The primitive to inspect. 796 */ 797 @Override 798 public void check(OsmPrimitive p) { 799 for (TestError e : getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())) { 800 addIfNotSimilar(e, errors); 801 } 802 if (partialSelection && p.isTagged()) { 803 tested.add(p); 804 } 805 } 806 807 /** 808 * Adds a new MapCSS config file from the given URL. 809 * @param url The unique URL of the MapCSS config file 810 * @return List of tag checks and parsing errors, or null 811 * @throws ParseException if the config file does not match MapCSS syntax 812 * @throws IOException if any I/O error occurs 813 * @since 7275 814 */ 815 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException { 816 CheckParameterUtil.ensureParameterNotNull(url, "url"); 817 ParseResult result; 818 try (CachedFile cache = new CachedFile(url); 819 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", ""); 820 InputStream s = zip != null ? zip : cache.getInputStream(); 821 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) { 822 if (zip != null) 823 I18n.addTexts(cache.getFile()); 824 result = TagCheck.readMapCSS(reader); 825 checks.remove(url); 826 checks.putAll(url, result.parseChecks); 827 indexData = null; 828 // Check assertions, useful for development of local files 829 if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) { 830 for (String msg : checkAsserts(result.parseChecks)) { 831 Logging.warn(msg); 832 } 833 } 834 } 835 return result; 836 } 837 838 @Override 839 public synchronized void initialize() throws Exception { 840 checks.clear(); 841 indexData = null; 842 for (SourceEntry source : new ValidatorPrefHelper().get()) { 843 if (!source.active) { 844 continue; 845 } 846 String i = source.url; 847 try { 848 if (!i.startsWith("resource:")) { 849 Logging.info(tr("Adding {0} to tag checker", i)); 850 } else if (Logging.isDebugEnabled()) { 851 Logging.debug(tr("Adding {0} to tag checker", i)); 852 } 853 addMapCSS(i); 854 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) { 855 FileWatcher.getDefaultInstance().registerSource(source); 856 } 857 } catch (IOException | IllegalStateException | IllegalArgumentException ex) { 858 Logging.warn(tr("Failed to add {0} to tag checker", i)); 859 Logging.log(Logging.LEVEL_WARN, ex); 860 } catch (ParseException | TokenMgrError ex) { 861 Logging.warn(tr("Failed to add {0} to tag checker", i)); 862 Logging.warn(ex); 863 } 864 } 865 } 866 867 private static Method getFunctionMethod(String method) { 868 try { 869 return Functions.class.getDeclaredMethod(method, Environment.class, String.class); 870 } catch (NoSuchMethodException | SecurityException e) { 871 Logging.error(e); 872 return null; 873 } 874 } 875 876 private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) { 877 return check.rule.selectors.stream() 878 .filter(s -> s instanceof GeneralSelector) 879 .flatMap(s -> ((GeneralSelector) s).getConditions().stream()) 880 .filter(c -> c instanceof ExpressionCondition) 881 .map(c -> ((ExpressionCondition) c).getExpression()) 882 .filter(c -> c instanceof ParameterFunction) 883 .map(c -> (ParameterFunction) c) 884 .filter(c -> c.getMethod().equals(insideMethod)) 885 .flatMap(c -> c.getArgs().stream()) 886 .filter(e -> e instanceof LiteralExpression) 887 .map(e -> ((LiteralExpression) e).getLiteral()) 888 .filter(l -> l instanceof String) 889 .map(l -> ((String) l).split(",")[0]) 890 .findFirst(); 891 } 892 893 private static LatLon getLocation(TagCheck check, Method insideMethod) { 894 Optional<String> inside = getFirstInsideCountry(check, insideMethod); 895 if (inside.isPresent()) { 896 GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get()); 897 if (index != null) { 898 GeoProperty<Boolean> prop = index.getGeoProperty(); 899 if (prop instanceof DefaultGeoProperty) { 900 Rectangle bounds = ((DefaultGeoProperty) prop).getArea().getBounds(); 901 return new LatLon(bounds.getCenterY(), bounds.getCenterX()); 902 } 903 } 904 } 905 return LatLon.ZERO; 906 } 907 908 /** 909 * Checks that rule assertions are met for the given set of TagChecks. 910 * @param schecks The TagChecks for which assertions have to be checked 911 * @return A set of error messages, empty if all assertions are met 912 * @since 7356 913 */ 914 public Set<String> checkAsserts(final Collection<TagCheck> schecks) { 915 Set<String> assertionErrors = new LinkedHashSet<>(); 916 final Method insideMethod = getFunctionMethod("inside"); 917 final DataSet ds = new DataSet(); 918 for (final TagCheck check : schecks) { 919 Logging.debug("Check: {0}", check); 920 for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) { 921 Logging.debug("- Assertion: {0}", i); 922 final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true); 923 // Build minimal ordered list of checks to run to test the assertion 924 List<Set<TagCheck>> checksToRun = new ArrayList<>(); 925 Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks); 926 if (!checkDependencies.isEmpty()) { 927 checksToRun.add(checkDependencies); 928 } 929 checksToRun.add(Collections.singleton(check)); 930 // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors 931 addPrimitive(ds, p); 932 final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun); 933 Logging.debug("- Errors: {0}", pErrors); 934 @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"}) 935 final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule)); 936 if (isError != i.getValue()) { 937 final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})", 938 check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys()); 939 assertionErrors.add(error); 940 } 941 ds.removePrimitive(p); 942 } 943 } 944 return assertionErrors; 945 } 946 947 private static void addPrimitive(DataSet ds, OsmPrimitive p) { 948 if (p instanceof Way) { 949 ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n)); 950 } else if (p instanceof Relation) { 951 ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember())); 952 } 953 ds.addPrimitive(p); 954 } 955 956 @Override 957 public synchronized int hashCode() { 958 return Objects.hash(super.hashCode(), checks); 959 } 960 961 @Override 962 public synchronized boolean equals(Object obj) { 963 if (this == obj) return true; 964 if (obj == null || getClass() != obj.getClass()) return false; 965 if (!super.equals(obj)) return false; 966 MapCSSTagChecker that = (MapCSSTagChecker) obj; 967 return Objects.equals(checks, that.checks); 968 } 969 970 /** 971 * Reload tagchecker rule. 972 * @param rule tagchecker rule to reload 973 * @since 12825 974 */ 975 public static void reloadRule(SourceEntry rule) { 976 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class); 977 if (tagChecker != null) { 978 try { 979 tagChecker.addMapCSS(rule.url); 980 } catch (IOException | ParseException | TokenMgrError e) { 981 Logging.warn(e); 982 } 983 } 984 } 985 986 @Override 987 public synchronized void startTest(ProgressMonitor progressMonitor) { 988 super.startTest(progressMonitor); 989 super.setShowElements(true); 990 if (indexData == null) { 991 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ALL_TESTS); 992 } 993 tested.clear(); 994 } 995 996 @Override 997 public synchronized void endTest() { 998 if (partialSelection && !tested.isEmpty()) { 999 // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15 1000 // execute tests for objects which might contain or cross previously tested elements 1001 1002 // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors 1003 // matching the previously tested elements 1004 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ONLY_SELECTED_TESTS); 1005 1006 Set<OsmPrimitive> surrounding = new HashSet<>(); 1007 for (OsmPrimitive p : tested) { 1008 if (p.getDataSet() != null) { 1009 surrounding.addAll(p.getDataSet().searchWays(p.getBBox())); 1010 surrounding.addAll(p.getDataSet().searchRelations(p.getBBox())); 1011 } 1012 } 1013 final boolean includeOtherSeverity = includeOtherSeverityChecks(); 1014 for (OsmPrimitive p : surrounding) { 1015 if (tested.contains(p)) 1016 continue; 1017 Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity); 1018 for (TestError e : additionalErrors) { 1019 if (e.getPrimitives().stream().anyMatch(tested::contains)) 1020 addIfNotSimilar(e, errors); 1021 } 1022 } 1023 tested.clear(); 1024 } 1025 super.endTest(); 1026 // no need to keep the index, it is quickly build and doubles the memory needs 1027 indexData = null; 1028 } 1029 1030 private boolean includeOtherSeverityChecks() { 1031 return isBeforeUpload ? ValidatorPrefHelper.PREF_OTHER_UPLOAD.get() : ValidatorPrefHelper.PREF_OTHER.get(); 1032 } 1033 1034}