Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 1 | # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style |
| 3 | # license that can be found in the LICENSE file or at |
| 4 | # https://developers.google.com/open-source/licenses/bsd |
| 5 | |
| 6 | """Implementation of the filter rules helper functions.""" |
| 7 | from __future__ import print_function |
| 8 | from __future__ import division |
| 9 | from __future__ import absolute_import |
| 10 | |
| 11 | import logging |
| 12 | import re |
| 13 | import six |
| 14 | |
| 15 | from six import string_types |
| 16 | |
| 17 | import settings |
| 18 | from features import features_constants |
| 19 | from framework import cloud_tasks_helpers |
| 20 | from framework import exceptions |
| 21 | from framework import framework_bizobj |
| 22 | from framework import framework_constants |
| 23 | from framework import urls |
| 24 | from framework import validate |
| 25 | from proto import ast_pb2 |
| 26 | from proto import tracker_pb2 |
| 27 | from search import query2ast |
| 28 | from search import searchpipeline |
| 29 | from tracker import component_helpers |
| 30 | from tracker import tracker_bizobj |
| 31 | from tracker import tracker_constants |
| 32 | from tracker import tracker_helpers |
| 33 | |
| 34 | |
| 35 | # Maximum number of filer rules that can be specified in a given |
| 36 | # project. This helps us bound the amount of time needed to |
| 37 | # (re)compute derived fields. |
| 38 | MAX_RULES = 200 |
| 39 | |
| 40 | BLOCK = tracker_constants.RECOMPUTE_DERIVED_FIELDS_BLOCK_SIZE |
| 41 | |
| 42 | |
| 43 | # TODO(jrobbins): implement a more efficient way to update just those |
| 44 | # issues affected by a specific component change. |
| 45 | def RecomputeAllDerivedFields(cnxn, services, project, config): |
| 46 | """Create work items to update all issues after filter rule changes. |
| 47 | |
| 48 | Args: |
| 49 | cnxn: connection to SQL database. |
| 50 | services: connections to backend services. |
| 51 | project: Project PB for the project that was edited. |
| 52 | config: ProjectIssueConfig PB for the project that was edited, |
| 53 | including the edits made. |
| 54 | """ |
| 55 | if not settings.recompute_derived_fields_in_worker: |
| 56 | # Background tasks are not enabled, just do everything in the servlet. |
| 57 | RecomputeAllDerivedFieldsNow(cnxn, services, project, config) |
| 58 | return |
| 59 | |
| 60 | highest_id = services.issue.GetHighestLocalID(cnxn, project.project_id) |
| 61 | if highest_id == 0: |
| 62 | return # No work to do. |
| 63 | |
| 64 | # Enqueue work items for blocks of issues to recompute. |
| 65 | steps = list(range(1, highest_id + 1, BLOCK)) |
| 66 | steps.reverse() # Update higher numbered issues sooner, old issues last. |
| 67 | # Cycle through shard_ids just to load-balance among the replicas. Each |
| 68 | # block includes all issues in that local_id range, not just 1/10 of them. |
| 69 | shard_id = 0 |
| 70 | for step in steps: |
| 71 | params = { |
| 72 | 'project_id': project.project_id, |
| 73 | 'lower_bound': step, |
| 74 | 'upper_bound': min(step + BLOCK, highest_id + 1), |
| 75 | 'shard_id': shard_id, |
| 76 | } |
| 77 | task = cloud_tasks_helpers.generate_simple_task( |
| 78 | urls.RECOMPUTE_DERIVED_FIELDS_TASK + '.do', params) |
| 79 | cloud_tasks_helpers.create_task( |
| 80 | task, queue=features_constants.QUEUE_RECOMPUTE_DERIVED_FIELDS) |
| 81 | |
| 82 | shard_id = (shard_id + 1) % settings.num_logical_shards |
| 83 | |
| 84 | |
| 85 | def RecomputeAllDerivedFieldsNow( |
| 86 | cnxn, services, project, config, lower_bound=None, upper_bound=None): |
| 87 | """Re-apply all filter rules to all issues in a project. |
| 88 | |
| 89 | Args: |
| 90 | cnxn: connection to SQL database. |
| 91 | services: connections to persistence layer. |
| 92 | project: Project PB for the project that was changed. |
| 93 | config: ProjectIssueConfig for that project. |
| 94 | lower_bound: optional int lowest issue ID to consider, inclusive. |
| 95 | upper_bound: optional int highest issue ID to consider, exclusive. |
| 96 | |
| 97 | SIDE-EFFECT: updates all issues in the project. Stores and re-indexes |
| 98 | all those that were changed. |
| 99 | """ |
| 100 | if lower_bound is not None and upper_bound is not None: |
| 101 | issues = services.issue.GetIssuesByLocalIDs( |
| 102 | cnxn, project.project_id, list(range(lower_bound, upper_bound)), |
| 103 | use_cache=False) |
| 104 | else: |
| 105 | issues = services.issue.GetAllIssuesInProject( |
| 106 | cnxn, project.project_id, use_cache=False) |
| 107 | |
| 108 | rules = services.features.GetFilterRules(cnxn, project.project_id) |
| 109 | predicate_asts = ParsePredicateASTs(rules, config, []) |
| 110 | modified_issues = [] |
| 111 | for issue in issues: |
| 112 | any_change, _traces = ApplyGivenRules( |
| 113 | cnxn, services, issue, config, rules, predicate_asts) |
| 114 | if any_change: |
| 115 | modified_issues.append(issue) |
| 116 | |
| 117 | services.issue.UpdateIssues(cnxn, modified_issues, just_derived=True) |
| 118 | |
| 119 | # Doing the FTS indexing can be too slow, so queue up the issues |
| 120 | # that need to be re-indexed by a cron-job later. |
| 121 | services.issue.EnqueueIssuesForIndexing( |
| 122 | cnxn, [issue.issue_id for issue in modified_issues]) |
| 123 | |
| 124 | |
| 125 | def ParsePredicateASTs(rules, config, me_user_ids): |
| 126 | """Parse the given rules in QueryAST PBs.""" |
| 127 | predicates = [rule.predicate for rule in rules] |
| 128 | if me_user_ids: |
| 129 | predicates = [ |
| 130 | searchpipeline.ReplaceKeywordsWithUserIDs(me_user_ids, pred)[0] |
| 131 | for pred in predicates] |
| 132 | predicate_asts = [ |
| 133 | query2ast.ParseUserQuery(pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config) |
| 134 | for pred in predicates] |
| 135 | return predicate_asts |
| 136 | |
| 137 | |
| 138 | def ApplyFilterRules(cnxn, services, issue, config): |
| 139 | """Apply the filter rules for this project to the given issue. |
| 140 | |
| 141 | Args: |
| 142 | cnxn: database connection, used to look up user IDs. |
| 143 | services: persistence layer for users, issues, and projects. |
| 144 | issue: An Issue PB that has just been updated with new explicit values. |
| 145 | config: The project's issue tracker config PB. |
| 146 | |
| 147 | Returns: |
| 148 | A pair (any_changes, traces) where any_changes is true if any changes |
| 149 | were made to the issue derived fields, and traces is a dictionary |
| 150 | {(field_id, new_value): explanation_str} of traces that |
| 151 | explain which rule generated each derived value. |
| 152 | |
| 153 | SIDE-EFFECT: update the derived_* fields of the Issue PB. |
| 154 | """ |
| 155 | rules = services.features.GetFilterRules(cnxn, issue.project_id) |
| 156 | predicate_asts = ParsePredicateASTs(rules, config, []) |
| 157 | return ApplyGivenRules(cnxn, services, issue, config, rules, predicate_asts) |
| 158 | |
| 159 | |
| 160 | def ApplyGivenRules(cnxn, services, issue, config, rules, predicate_asts): |
| 161 | """Apply the filter rules for this project to the given issue. |
| 162 | |
| 163 | Args: |
| 164 | cnxn: database connection, used to look up user IDs. |
| 165 | services: persistence layer for users, issues, and projects. |
| 166 | issue: An Issue PB that has just been updated with new explicit values. |
| 167 | config: The project's issue tracker config PB. |
| 168 | rules: list of FilterRule PBs. |
| 169 | |
| 170 | Returns: |
| 171 | A pair (any_changes, traces) where any_changes is true if any changes |
| 172 | were made to the issue derived fields, and traces is a dictionary |
| 173 | {(field_id, new_value): explanation_str} of traces that |
| 174 | explain which rule generated each derived value. |
| 175 | |
| 176 | SIDE-EFFECT: update the derived_* fields of the Issue PB. |
| 177 | """ |
| 178 | (derived_owner_id, derived_status, derived_cc_ids, |
| 179 | derived_labels, derived_notify_addrs, traces, |
| 180 | new_warnings, new_errors) = _ComputeDerivedFields( |
| 181 | cnxn, services, issue, config, rules, predicate_asts) |
| 182 | |
| 183 | any_change = (derived_owner_id != issue.derived_owner_id or |
| 184 | derived_status != issue.derived_status or |
| 185 | derived_cc_ids != issue.derived_cc_ids or |
| 186 | derived_labels != issue.derived_labels or |
| 187 | derived_notify_addrs != issue.derived_notify_addrs) |
| 188 | |
| 189 | # Remember any derived values. |
| 190 | issue.derived_owner_id = derived_owner_id |
| 191 | issue.derived_status = derived_status |
| 192 | issue.derived_cc_ids = derived_cc_ids |
| 193 | issue.derived_labels = derived_labels |
| 194 | issue.derived_notify_addrs = derived_notify_addrs |
| 195 | issue.derived_warnings = new_warnings |
| 196 | issue.derived_errors = new_errors |
| 197 | |
| 198 | return any_change, traces |
| 199 | |
| 200 | |
| 201 | def _ComputeDerivedFields(cnxn, services, issue, config, rules, predicate_asts): |
| 202 | """Compute derived field values for an issue based on filter rules. |
| 203 | |
| 204 | Args: |
| 205 | cnxn: database connection, used to look up user IDs. |
| 206 | services: persistence layer for users, issues, and projects. |
| 207 | issue: the issue to examine. |
| 208 | config: ProjectIssueConfig for the project containing the issue. |
| 209 | rules: list of FilterRule PBs. |
| 210 | predicate_asts: QueryAST PB for each rule. |
| 211 | |
| 212 | Returns: |
| 213 | A 8-tuple of derived values for owner_id, status, cc_ids, labels, |
| 214 | notify_addrs, traces, warnings, and errors. These values are the result |
| 215 | of applying all rules in order. Filter rules only produce derived values |
| 216 | that do not conflict with the explicit field values of the issue. |
| 217 | """ |
| 218 | excl_prefixes = [ |
| 219 | prefix.lower() for prefix in config.exclusive_label_prefixes] |
| 220 | # Examine the explicit labels and Cc's on the issue. |
| 221 | lower_labels = [lab.lower() for lab in issue.labels] |
| 222 | label_set = set(lower_labels) |
| 223 | cc_set = set(issue.cc_ids) |
| 224 | excl_prefixes_used = set() |
| 225 | for lab in lower_labels: |
| 226 | prefix = lab.split('-')[0] |
| 227 | if prefix in excl_prefixes: |
| 228 | excl_prefixes_used.add(prefix) |
| 229 | prefix_values_added = {} |
| 230 | |
| 231 | # Start with the assumption that rules don't change anything, then |
| 232 | # accumulate changes. |
| 233 | derived_owner_id = framework_constants.NO_USER_SPECIFIED |
| 234 | derived_status = '' |
| 235 | derived_cc_ids = [] |
| 236 | derived_labels = [] |
| 237 | derived_notify_addrs = [] |
| 238 | traces = {} # {(field_id, new_value): explanation_str} |
| 239 | new_warnings = [] |
| 240 | new_errors = [] |
| 241 | |
| 242 | def AddLabelConsideringExclusivePrefixes(label): |
| 243 | lab_lower = label.lower() |
| 244 | if lab_lower in label_set: |
| 245 | return False # We already have that label. |
| 246 | prefix = lab_lower.split('-')[0] |
| 247 | if '-' in lab_lower and prefix in excl_prefixes: |
| 248 | if prefix in excl_prefixes_used: |
| 249 | return False # Issue already has that prefix. |
| 250 | # Replace any earlied-added label that had the same exclusive prefix. |
| 251 | if prefix in prefix_values_added: |
| 252 | label_set.remove(prefix_values_added[prefix].lower()) |
| 253 | derived_labels.remove(prefix_values_added[prefix]) |
| 254 | prefix_values_added[prefix] = label |
| 255 | |
| 256 | derived_labels.append(label) |
| 257 | label_set.add(lab_lower) |
| 258 | return True |
| 259 | |
| 260 | # Apply component labels and auto-cc's before doing the rules. |
| 261 | components = tracker_bizobj.GetIssueComponentsAndAncestors(issue, config) |
| 262 | for cd in components: |
| 263 | for cc_id in cd.cc_ids: |
| 264 | if cc_id not in cc_set: |
| 265 | derived_cc_ids.append(cc_id) |
| 266 | cc_set.add(cc_id) |
| 267 | traces[(tracker_pb2.FieldID.CC, cc_id)] = ( |
| 268 | 'Added by component %s' % cd.path) |
| 269 | |
| 270 | for label_id in cd.label_ids: |
| 271 | lab = services.config.LookupLabel(cnxn, config.project_id, label_id) |
| 272 | if AddLabelConsideringExclusivePrefixes(lab): |
| 273 | traces[(tracker_pb2.FieldID.LABELS, lab)] = ( |
| 274 | 'Added by component %s' % cd.path) |
| 275 | |
| 276 | # Apply each rule in order. Later rules see the results of earlier rules. |
| 277 | # Later rules can overwrite or add to results of earlier rules. |
| 278 | # TODO(jrobbins): also pass in in-progress values for owner and CCs so |
| 279 | # that early rules that set those can affect later rules that check them. |
| 280 | for rule, predicate_ast in zip(rules, predicate_asts): |
| 281 | (rule_owner_id, rule_status, rule_add_cc_ids, |
| 282 | rule_add_labels, rule_add_notify, rule_add_warning, |
| 283 | rule_add_error) = _ApplyRule( |
| 284 | cnxn, services, rule, predicate_ast, issue, label_set, config) |
| 285 | |
| 286 | # logging.info( |
| 287 | # 'rule "%s" gave %r, %r, %r, %r, %r', |
| 288 | # rule.predicate, rule_owner_id, rule_status, rule_add_cc_ids, |
| 289 | # rule_add_labels, rule_add_notify) |
| 290 | |
| 291 | if rule_owner_id and not issue.owner_id: |
| 292 | derived_owner_id = rule_owner_id |
| 293 | traces[(tracker_pb2.FieldID.OWNER, rule_owner_id)] = ( |
| 294 | 'Added by rule: IF %s THEN SET DEFAULT OWNER' % rule.predicate) |
| 295 | |
| 296 | if rule_status and not issue.status: |
| 297 | derived_status = rule_status |
| 298 | traces[(tracker_pb2.FieldID.STATUS, rule_status)] = ( |
| 299 | 'Added by rule: IF %s THEN SET DEFAULT STATUS' % rule.predicate) |
| 300 | |
| 301 | for cc_id in rule_add_cc_ids: |
| 302 | if cc_id not in cc_set: |
| 303 | derived_cc_ids.append(cc_id) |
| 304 | cc_set.add(cc_id) |
| 305 | traces[(tracker_pb2.FieldID.CC, cc_id)] = ( |
| 306 | 'Added by rule: IF %s THEN ADD CC' % rule.predicate) |
| 307 | |
| 308 | for lab in rule_add_labels: |
| 309 | if AddLabelConsideringExclusivePrefixes(lab): |
| 310 | traces[(tracker_pb2.FieldID.LABELS, lab)] = ( |
| 311 | 'Added by rule: IF %s THEN ADD LABEL' % rule.predicate) |
| 312 | |
| 313 | for addr in rule_add_notify: |
| 314 | if addr not in derived_notify_addrs: |
| 315 | derived_notify_addrs.append(addr) |
| 316 | # Note: No trace because also-notify addresses are not shown in the UI. |
| 317 | |
| 318 | if rule_add_warning: |
| 319 | new_warnings.append(rule_add_warning) |
| 320 | traces[(tracker_pb2.FieldID.WARNING, rule_add_warning)] = ( |
| 321 | 'Added by rule: IF %s THEN ADD WARNING' % rule.predicate) |
| 322 | |
| 323 | if rule_add_error: |
| 324 | new_errors.append(rule_add_error) |
| 325 | traces[(tracker_pb2.FieldID.ERROR, rule_add_error)] = ( |
| 326 | 'Added by rule: IF %s THEN ADD ERROR' % rule.predicate) |
| 327 | |
| 328 | return (derived_owner_id, derived_status, derived_cc_ids, derived_labels, |
| 329 | derived_notify_addrs, traces, new_warnings, new_errors) |
| 330 | |
| 331 | |
| 332 | def EvalPredicate( |
| 333 | cnxn, services, predicate_ast, issue, label_set, config, owner_id, cc_ids, |
| 334 | status): |
| 335 | """Return True if the given issue satisfies the given predicate. |
| 336 | |
| 337 | Args: |
| 338 | cnxn: Connection to SQL database. |
| 339 | services: persistence layer for users and issues. |
| 340 | predicate_ast: QueryAST for rule or saved query string. |
| 341 | issue: Issue PB of the issue to evaluate. |
| 342 | label_set: set of lower-cased labels on the issue. |
| 343 | config: ProjectIssueConfig for the project that contains the issue. |
| 344 | owner_id: int user ID of the issue owner. |
| 345 | cc_ids: list of int user IDs of the users Cc'd on the issue. |
| 346 | status: string status value of the issue. |
| 347 | |
| 348 | Returns: |
| 349 | True if the issue satisfies the predicate. |
| 350 | |
| 351 | Note: filter rule evaluation passes in only the explicit owner_id, |
| 352 | cc_ids, and status whereas subscription evaluation passes in the |
| 353 | combination of explicit values and derived values. |
| 354 | """ |
| 355 | # TODO(jrobbins): Call ast2ast to simplify the predicate and do |
| 356 | # most lookups. Refactor to allow that to be done once. |
| 357 | project = services.project.GetProject(cnxn, config.project_id) |
| 358 | for conj in predicate_ast.conjunctions: |
| 359 | if all(_ApplyCond(cnxn, services, project, cond, issue, label_set, config, |
| 360 | owner_id, cc_ids, status) |
| 361 | for cond in conj.conds): |
| 362 | return True |
| 363 | |
| 364 | # All OR-clauses were evaluated, but none of them was matched. |
| 365 | return False |
| 366 | |
| 367 | |
| 368 | def _ApplyRule( |
| 369 | cnxn, services, rule_pb, predicate_ast, issue, label_set, config): |
| 370 | """Test if the given rule should fire and return its result. |
| 371 | |
| 372 | Args: |
| 373 | cnxn: database connection, used to look up user IDs. |
| 374 | services: persistence layer for users and issues. |
| 375 | rule_pb: FilterRule PB instance with a predicate and various actions. |
| 376 | predicate_ast: QueryAST for the rule predicate. |
| 377 | issue: The Issue PB to be considered. |
| 378 | label_set: set of lowercased labels from an issue's explicit |
| 379 | label_list plus and labels that have accumlated from previous rules. |
| 380 | config: ProjectIssueConfig for the project containing the issue. |
| 381 | |
| 382 | Returns: |
| 383 | A 6-tuple of the results from this rule: derived owner id, status, |
| 384 | cc_ids to add, labels to add, notify addresses to add, and a warning |
| 385 | string. Currently only one will be set and the others will all be |
| 386 | None or an empty list. |
| 387 | """ |
| 388 | if EvalPredicate( |
| 389 | cnxn, services, predicate_ast, issue, label_set, config, |
| 390 | issue.owner_id, issue.cc_ids, issue.status): |
| 391 | logging.info('rule adds: %r', rule_pb.add_labels) |
| 392 | return (rule_pb.default_owner_id, rule_pb.default_status, |
| 393 | rule_pb.add_cc_ids, rule_pb.add_labels, |
| 394 | rule_pb.add_notify_addrs, rule_pb.warning, rule_pb.error) |
| 395 | else: |
| 396 | return None, None, [], [], [], None, None |
| 397 | |
| 398 | |
| 399 | def _ApplyCond( |
| 400 | cnxn, services, project, term, issue, label_set, config, owner_id, cc_ids, |
| 401 | status): |
| 402 | """Return True if the given issue satisfied the given predicate term.""" |
| 403 | op = term.op |
| 404 | vals = term.str_values or term.int_values |
| 405 | # Since rules are per-project, there'll be exactly 1 field |
| 406 | fd = term.field_defs[0] |
| 407 | field = fd.field_name |
| 408 | |
| 409 | if field == 'label': |
| 410 | return _Compare(op, vals, label_set) |
| 411 | if field == 'component': |
| 412 | return _CompareComponents(config, op, vals, issue.component_ids) |
| 413 | if field == 'any_field': |
| 414 | return _Compare(op, vals, label_set) or _Compare(op, vals, [issue.summary]) |
| 415 | if field == 'attachments': |
| 416 | return _Compare(op, term.int_values, [issue.attachment_count]) |
| 417 | if field == 'blocked': |
| 418 | return _Compare(op, vals, issue.blocked_on_iids) |
| 419 | if field == 'blockedon': |
| 420 | return _CompareIssueRefs( |
| 421 | cnxn, services, project, op, term.str_values, issue.blocked_on_iids) |
| 422 | if field == 'blocking': |
| 423 | return _CompareIssueRefs( |
| 424 | cnxn, services, project, op, term.str_values, issue.blocking_iids) |
| 425 | if field == 'cc': |
| 426 | return _CompareUsers(cnxn, services.user, op, vals, cc_ids) |
| 427 | if field == 'closed': |
| 428 | return (issue.closed_timestamp and |
| 429 | _Compare(op, vals, [issue.closed_timestamp])) |
| 430 | if field == 'id': |
| 431 | return _Compare(op, vals, [issue.local_id]) |
| 432 | if field == 'mergedinto': |
| 433 | return _CompareIssueRefs( |
| 434 | cnxn, services, project, op, term.str_values, [issue.merged_into or 0]) |
| 435 | if field == 'modified': |
| 436 | return (issue.modified_timestamp and |
| 437 | _Compare(op, vals, [issue.modified_timestamp])) |
| 438 | if field == 'open': |
| 439 | # TODO(jrobbins): this just checks the explicit status, not the result |
| 440 | # of any previous rules. |
| 441 | return tracker_helpers.MeansOpenInProject(status, config) |
| 442 | if field == 'opened': |
| 443 | return (issue.opened_timestamp and |
| 444 | _Compare(op, vals, [issue.opened_timestamp])) |
| 445 | if field == 'owner': |
| 446 | return _CompareUsers(cnxn, services.user, op, vals, [owner_id]) |
| 447 | if field == 'reporter': |
| 448 | return _CompareUsers(cnxn, services.user, op, vals, [issue.reporter_id]) |
| 449 | if field == 'stars': |
| 450 | return _Compare(op, term.int_values, [issue.star_count]) |
| 451 | if field == 'status': |
| 452 | return _Compare(op, vals, [status.lower()]) |
| 453 | if field == 'summary': |
| 454 | return _Compare(op, vals, [issue.summary]) |
| 455 | |
| 456 | # Since rules are per-project, it makes no sense to support field project. |
| 457 | # We would need to load comments to support fields comment, commentby, |
| 458 | # description, attachment. |
| 459 | # Supporting starredby is probably not worth the complexity. |
| 460 | |
| 461 | logging.info('Rule with unsupported field %r was False', field) |
| 462 | return False |
| 463 | |
| 464 | |
| 465 | def _CheckTrivialCases(op, issue_values): |
| 466 | """Check has:x and -has:x terms and no values. Otherwise, return None.""" |
| 467 | # We can do these operators without looking up anything or even knowing |
| 468 | # which field is being checked. |
| 469 | issue_values_exist = bool( |
| 470 | issue_values and issue_values != [''] and issue_values != [0]) |
| 471 | if op == ast_pb2.QueryOp.IS_DEFINED: |
| 472 | return issue_values_exist |
| 473 | elif op == ast_pb2.QueryOp.IS_NOT_DEFINED: |
| 474 | return not issue_values_exist |
| 475 | elif not issue_values_exist: |
| 476 | # No other operator can match empty values. |
| 477 | return op in (ast_pb2.QueryOp.NE, ast_pb2.QueryOp.NOT_TEXT_HAS) |
| 478 | |
| 479 | return None # Caller should continue processing the term. |
| 480 | |
| 481 | def _CompareComponents(config, op, rule_values, issue_values): |
| 482 | """Compare the components specified in the rule vs those in the issue.""" |
| 483 | trivial_result = _CheckTrivialCases(op, issue_values) |
| 484 | if trivial_result is not None: |
| 485 | return trivial_result |
| 486 | |
| 487 | exact = op in (ast_pb2.QueryOp.EQ, ast_pb2.QueryOp.NE) |
| 488 | rule_component_ids = set() |
| 489 | for path in rule_values: |
| 490 | rule_component_ids.update(tracker_bizobj.FindMatchingComponentIDs( |
| 491 | path, config, exact=exact)) |
| 492 | |
| 493 | if op == ast_pb2.QueryOp.TEXT_HAS or op == ast_pb2.QueryOp.EQ: |
| 494 | return any(rv in issue_values for rv in rule_component_ids) |
| 495 | elif op == ast_pb2.QueryOp.NOT_TEXT_HAS or op == ast_pb2.QueryOp.NE: |
| 496 | return all(rv not in issue_values for rv in rule_component_ids) |
| 497 | |
| 498 | return False |
| 499 | |
| 500 | |
| 501 | def _CompareIssueRefs( |
| 502 | cnxn, services, project, op, rule_str_values, issue_values): |
| 503 | """Compare the issues specified in the rule vs referenced in the issue.""" |
| 504 | trivial_result = _CheckTrivialCases(op, issue_values) |
| 505 | if trivial_result is not None: |
| 506 | return trivial_result |
| 507 | |
| 508 | rule_refs = [] |
| 509 | for str_val in rule_str_values: |
| 510 | ref = tracker_bizobj.ParseIssueRef(str_val) |
| 511 | if ref: |
| 512 | rule_refs.append(ref) |
| 513 | rule_ref_project_names = set( |
| 514 | pn for pn, local_id in rule_refs if pn) |
| 515 | rule_ref_projects_dict = services.project.GetProjectsByName( |
| 516 | cnxn, rule_ref_project_names) |
| 517 | rule_ref_projects_dict[project.project_name] = project |
| 518 | rule_iids, _misses = services.issue.ResolveIssueRefs( |
| 519 | cnxn, rule_ref_projects_dict, project.project_name, rule_refs) |
| 520 | |
| 521 | if op == ast_pb2.QueryOp.TEXT_HAS: |
| 522 | op = ast_pb2.QueryOp.EQ |
| 523 | if op == ast_pb2.QueryOp.NOT_TEXT_HAS: |
| 524 | op = ast_pb2.QueryOp.NE |
| 525 | |
| 526 | return _Compare(op, rule_iids, issue_values) |
| 527 | |
| 528 | |
| 529 | def _CompareUsers(cnxn, user_service, op, rule_values, issue_values): |
| 530 | """Compare the user(s) specified in the rule and the issue.""" |
| 531 | # Note that all occurances of "me" in rule_values should have already |
| 532 | # been resolved to str(user_id) of the subscribing user. |
| 533 | # TODO(jrobbins): Project filter rules should not be allowed to have "me". |
| 534 | |
| 535 | trivial_result = _CheckTrivialCases(op, issue_values) |
| 536 | if trivial_result is not None: |
| 537 | return trivial_result |
| 538 | |
| 539 | try: |
| 540 | return _CompareUserIDs(op, rule_values, issue_values) |
| 541 | except ValueError: |
| 542 | return _CompareEmails(cnxn, user_service, op, rule_values, issue_values) |
| 543 | |
| 544 | |
| 545 | def _CompareUserIDs(op, rule_values, issue_values): |
| 546 | """Compare users according to specified user ID integer strings.""" |
| 547 | rule_user_ids = [int(uid_str) for uid_str in rule_values] |
| 548 | |
| 549 | if op == ast_pb2.QueryOp.TEXT_HAS or op == ast_pb2.QueryOp.EQ: |
| 550 | return any(rv in issue_values for rv in rule_user_ids) |
| 551 | elif op == ast_pb2.QueryOp.NOT_TEXT_HAS or op == ast_pb2.QueryOp.NE: |
| 552 | return all(rv not in issue_values for rv in rule_user_ids) |
| 553 | |
| 554 | logging.info('unexpected numeric user operator %r %r %r', |
| 555 | op, rule_values, issue_values) |
| 556 | return False |
| 557 | |
| 558 | |
| 559 | def _CompareEmails(cnxn, user_service, op, rule_values, issue_values): |
| 560 | """Compare users based on email addresses.""" |
| 561 | issue_emails = list( |
| 562 | user_service.LookupUserEmails(cnxn, issue_values).values()) |
| 563 | |
| 564 | if op == ast_pb2.QueryOp.TEXT_HAS: |
| 565 | return any(_HasText(rv, issue_emails) for rv in rule_values) |
| 566 | elif op == ast_pb2.QueryOp.NOT_TEXT_HAS: |
| 567 | return all(not _HasText(rv, issue_emails) for rv in rule_values) |
| 568 | elif op == ast_pb2.QueryOp.EQ: |
| 569 | return any(rv in issue_emails for rv in rule_values) |
| 570 | elif op == ast_pb2.QueryOp.NE: |
| 571 | return all(rv not in issue_emails for rv in rule_values) |
| 572 | |
| 573 | logging.info('unexpected user operator %r %r %r', |
| 574 | op, rule_values, issue_values) |
| 575 | return False |
| 576 | |
| 577 | |
| 578 | def _Compare(op, rule_values, issue_values): |
| 579 | """Compare the values specified in the rule and the issue.""" |
| 580 | trivial_result = _CheckTrivialCases(op, issue_values) |
| 581 | if trivial_result is not None: |
| 582 | return trivial_result |
| 583 | |
| 584 | if (op in [ast_pb2.QueryOp.TEXT_HAS, ast_pb2.QueryOp.NOT_TEXT_HAS] and |
| 585 | issue_values and not isinstance(min(issue_values), string_types)): |
| 586 | return False # Empty or numeric fields cannot match substrings |
| 587 | elif op == ast_pb2.QueryOp.TEXT_HAS: |
| 588 | return any(_HasText(rv, issue_values) for rv in rule_values) |
| 589 | elif op == ast_pb2.QueryOp.NOT_TEXT_HAS: |
| 590 | return all(not _HasText(rv, issue_values) for rv in rule_values) |
| 591 | |
| 592 | val_type = type(min(issue_values)) |
| 593 | if val_type in six.integer_types: |
| 594 | try: |
| 595 | rule_values = [int(rv) for rv in rule_values] |
| 596 | except ValueError: |
| 597 | logging.info('rule value conversion to int failed: %r', rule_values) |
| 598 | return False |
| 599 | |
| 600 | if op == ast_pb2.QueryOp.EQ: |
| 601 | return any(rv in issue_values for rv in rule_values) |
| 602 | elif op == ast_pb2.QueryOp.NE: |
| 603 | return all(rv not in issue_values for rv in rule_values) |
| 604 | |
| 605 | if val_type not in six.integer_types: |
| 606 | return False # Inequalities only work on numeric fields |
| 607 | |
| 608 | if op == ast_pb2.QueryOp.GT: |
| 609 | return min(issue_values) > min(rule_values) |
| 610 | elif op == ast_pb2.QueryOp.GE: |
| 611 | return min(issue_values) >= min(rule_values) |
| 612 | elif op == ast_pb2.QueryOp.LT: |
| 613 | return max(issue_values) < max(rule_values) |
| 614 | elif op == ast_pb2.QueryOp.LE: |
| 615 | return max(issue_values) <= max(rule_values) |
| 616 | |
| 617 | logging.info('unexpected operator %r %r %r', op, rule_values, issue_values) |
| 618 | return False |
| 619 | |
| 620 | |
| 621 | def _HasText(rule_text, issue_values): |
| 622 | """Return True if the issue contains the rule text, case insensitive.""" |
| 623 | rule_lower = rule_text.lower() |
| 624 | for iv in issue_values: |
| 625 | if iv is not None and rule_lower in iv.lower(): |
| 626 | return True |
| 627 | |
| 628 | return False |
| 629 | |
| 630 | |
| 631 | def MakeRule( |
| 632 | predicate, default_status=None, default_owner_id=None, add_cc_ids=None, |
| 633 | add_labels=None, add_notify=None, warning=None, error=None): |
| 634 | """Make a FilterRule PB with the supplied information. |
| 635 | |
| 636 | Args: |
| 637 | predicate: string query that will trigger the rule if satisfied. |
| 638 | default_status: optional default status to set if rule fires. |
| 639 | default_owner_id: optional default owner_id to set if rule fires. |
| 640 | add_cc_ids: optional cc ids to set if rule fires. |
| 641 | add_labels: optional label strings to set if rule fires. |
| 642 | add_notify: optional notify email addresses to set if rule fires. |
| 643 | warning: optional string for a software development process warning. |
| 644 | error: optional string for a software development process error. |
| 645 | |
| 646 | Returns: |
| 647 | A new FilterRule PB. |
| 648 | """ |
| 649 | rule_pb = tracker_pb2.FilterRule() |
| 650 | rule_pb.predicate = predicate |
| 651 | |
| 652 | if add_labels: |
| 653 | rule_pb.add_labels = add_labels |
| 654 | if default_status: |
| 655 | rule_pb.default_status = default_status |
| 656 | if default_owner_id: |
| 657 | rule_pb.default_owner_id = default_owner_id |
| 658 | if add_cc_ids: |
| 659 | rule_pb.add_cc_ids = add_cc_ids |
| 660 | if add_notify: |
| 661 | rule_pb.add_notify_addrs = add_notify |
| 662 | if warning: |
| 663 | rule_pb.warning = warning |
| 664 | if error: |
| 665 | rule_pb.error = error |
| 666 | |
| 667 | return rule_pb |
| 668 | |
| 669 | |
| 670 | def ParseRules(cnxn, post_data, user_service, errors, prefix=''): |
| 671 | """Parse rules from the user and return a list of FilterRule PBs. |
| 672 | |
| 673 | Args: |
| 674 | cnxn: connection to database. |
| 675 | post_data: dictionary of html form data. |
| 676 | user_service: connection to user backend services. |
| 677 | errors: EZTErrors message used to display field validation errors. |
| 678 | prefix: optional string prefix used to differentiate the form fields |
| 679 | for existing rules from the form fields for new rules. |
| 680 | |
| 681 | Returns: |
| 682 | A list of FilterRule PBs |
| 683 | """ |
| 684 | rules = [] |
| 685 | |
| 686 | # The best we can do for now is show all validation errors at the bottom of |
| 687 | # the filter rules section, not directly on the rule that had the error :(. |
| 688 | error_list = [] |
| 689 | |
| 690 | for i in range(1, MAX_RULES + 1): |
| 691 | if ('%spredicate%s' % (prefix, i)) not in post_data: |
| 692 | continue # skip any entries that are blank or have no predicate. |
| 693 | predicate = post_data['%spredicate%s' % (prefix, i)].strip() |
| 694 | action_type = post_data.get('%saction_type%s' % (prefix, i), |
| 695 | 'add_labels').strip() |
| 696 | action_value = post_data.get('%saction_value%s' % (prefix, i), |
| 697 | '').strip() |
| 698 | if predicate: |
| 699 | # Note: action_value may be '', meaning no-op. |
| 700 | rules.append(_ParseOneRule( |
| 701 | cnxn, predicate, action_type, action_value, user_service, i, |
| 702 | error_list)) |
| 703 | |
| 704 | if error_list: |
| 705 | errors.rules = error_list |
| 706 | |
| 707 | return rules |
| 708 | |
| 709 | |
| 710 | def _ParseOneRule( |
| 711 | cnxn, predicate, action_type, action_value, user_service, |
| 712 | rule_num, error_list): |
| 713 | """Parse one FilterRule based on the action type.""" |
| 714 | |
| 715 | if action_type == 'default_status': |
| 716 | status = framework_bizobj.CanonicalizeLabel(action_value) |
| 717 | rule = MakeRule(predicate, default_status=status) |
| 718 | |
| 719 | elif action_type == 'default_owner': |
| 720 | if action_value: |
| 721 | try: |
| 722 | user_id = user_service.LookupUserID(cnxn, action_value) |
| 723 | except exceptions.NoSuchUserException: |
| 724 | user_id = framework_constants.NO_USER_SPECIFIED |
| 725 | error_list.append( |
| 726 | 'Rule %d: No such user: %s' % (rule_num, action_value)) |
| 727 | else: |
| 728 | user_id = framework_constants.NO_USER_SPECIFIED |
| 729 | rule = MakeRule(predicate, default_owner_id=user_id) |
| 730 | |
| 731 | elif action_type == 'add_ccs': |
| 732 | cc_ids = [] |
| 733 | for email in re.split('[,;\s]+', action_value): |
| 734 | if not email.strip(): |
| 735 | continue |
| 736 | try: |
| 737 | user_id = user_service.LookupUserID( |
| 738 | cnxn, email.strip(), autocreate=True) |
| 739 | cc_ids.append(user_id) |
| 740 | except exceptions.NoSuchUserException: |
| 741 | error_list.append( |
| 742 | 'Rule %d: No such user: %s' % (rule_num, email.strip())) |
| 743 | |
| 744 | rule = MakeRule(predicate, add_cc_ids=cc_ids) |
| 745 | |
| 746 | elif action_type == 'add_labels': |
| 747 | add_labels = framework_constants.IDENTIFIER_RE.findall(action_value) |
| 748 | rule = MakeRule(predicate, add_labels=add_labels) |
| 749 | |
| 750 | elif action_type == 'also_notify': |
| 751 | add_notify = [] |
| 752 | for addr in re.split('[,;\s]+', action_value): |
| 753 | if validate.IsValidEmail(addr.strip()): |
| 754 | add_notify.append(addr.strip()) |
| 755 | else: |
| 756 | error_list.append( |
| 757 | 'Rule %d: Invalid email address: %s' % (rule_num, addr.strip())) |
| 758 | |
| 759 | rule = MakeRule(predicate, add_notify=add_notify) |
| 760 | |
| 761 | elif action_type == 'warning': |
| 762 | rule = MakeRule(predicate, warning=action_value) |
| 763 | |
| 764 | elif action_type == 'error': |
| 765 | rule = MakeRule(predicate, error=action_value) |
| 766 | |
| 767 | else: |
| 768 | logging.info('unexpected action type, probably tampering:%r', action_type) |
| 769 | raise exceptions.InputException() |
| 770 | |
| 771 | return rule |
| 772 | |
| 773 | |
| 774 | def OwnerCcsInvolvedInFilterRules(rules): |
| 775 | """Finds all user_ids in the given rules and returns them. |
| 776 | |
| 777 | Args: |
| 778 | rules: a list of FilterRule PBs. |
| 779 | |
| 780 | Returns: |
| 781 | A set of user_ids. |
| 782 | """ |
| 783 | user_ids = set() |
| 784 | for rule in rules: |
| 785 | if rule.default_owner_id: |
| 786 | user_ids.add(rule.default_owner_id) |
| 787 | user_ids.update(rule.add_cc_ids) |
| 788 | return user_ids |
| 789 | |
| 790 | |
| 791 | def BuildFilterRuleStrings(filter_rules, emails_by_id): |
| 792 | """Builds strings that represent filter rules. |
| 793 | |
| 794 | Args: |
| 795 | filter_rules: a list of FilterRule PBs. |
| 796 | emails_by_id: a dict of {user_id: email, ..} of user_ids in the FilterRules. |
| 797 | |
| 798 | Returns: |
| 799 | A list of strings each representing a FilterRule. |
| 800 | eg. "if predicate then consequence" |
| 801 | """ |
| 802 | rule_strs = [] |
| 803 | for rule in filter_rules: |
| 804 | cons = "" |
| 805 | if rule.add_labels: |
| 806 | cons = 'add label(s): %s' % ', '.join(rule.add_labels) |
| 807 | elif rule.default_status: |
| 808 | cons = 'set default status: %s' % rule.default_status |
| 809 | elif rule.default_owner_id: |
| 810 | cons = 'set default owner: %s' % emails_by_id.get( |
| 811 | rule.default_owner_id, 'user not found') |
| 812 | elif rule.add_cc_ids: |
| 813 | cons = 'add cc(s): %s' % ', '.join( |
| 814 | [emails_by_id.get(user_id, 'user not found') |
| 815 | for user_id in rule.add_cc_ids]) |
| 816 | elif rule.add_notify_addrs: |
| 817 | cons = 'notify: %s' % ', '.join(rule.add_notify_addrs) |
| 818 | |
| 819 | rule_strs.append('if %s then %s' % (rule.predicate, cons)) |
| 820 | |
| 821 | return rule_strs |
| 822 | |
| 823 | |
| 824 | def BuildRedactedFilterRuleStrings( |
| 825 | cnxn, rules_by_project, user_service, hide_emails): |
| 826 | """Converts FilterRule PBs in strings that hide references to hide_emails. |
| 827 | |
| 828 | Args: |
| 829 | rules_by_project: a dict of {project_id, [filter_rule, ...], ...} |
| 830 | with FilterRule PBs. |
| 831 | user_service: |
| 832 | hide_emails: a list of emails that should not be shown in rule strings. |
| 833 | """ |
| 834 | rule_strs_by_project = {} |
| 835 | prohibited_re = re.compile( |
| 836 | r'\b%s\b' % r'\b|\b'.join(map(re.escape, hide_emails))) |
| 837 | for project_id, rules in rules_by_project.items(): |
| 838 | user_ids_in_rules = OwnerCcsInvolvedInFilterRules(rules) |
| 839 | emails_by_id = user_service.LookupUserEmails( |
| 840 | cnxn, user_ids_in_rules, ignore_missed=True) |
| 841 | rule_strs = BuildFilterRuleStrings(rules, emails_by_id) |
| 842 | censored_strs = [ |
| 843 | prohibited_re.sub(framework_constants.DELETED_USER_NAME, rule_str) |
| 844 | for rule_str in rule_strs] |
| 845 | |
| 846 | rule_strs_by_project[project_id] = censored_strs |
| 847 | |
| 848 | return rule_strs_by_project |