blob: c9f9e5a0ba41e37990587f3bb52e41c39fe490e6 [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Helper functions and classes used by the Monorail Issue Tracker pages.
This module has functions that are reused in multiple servlets or
other modules.
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import collections
import itertools
import logging
import re
import time
import urllib
from google.appengine.api import app_identity
from six import string_types
import settings
from features import federated
from framework import authdata
from framework import exceptions
from framework import filecontent
from framework import framework_bizobj
from framework import framework_constants
from framework import framework_helpers
from framework import framework_views
from framework import permissions
from framework import sorting
from framework import template_helpers
from framework import urls
from project import project_helpers
from proto import tracker_pb2
from services import client_config_svc
from tracker import field_helpers
from tracker import tracker_bizobj
from tracker import tracker_constants
# HTML input field names for blocked on and blocking issue refs.
BLOCKED_ON = 'blocked_on'
BLOCKING = 'blocking'
# This string is used in HTML form element names to identify custom fields.
# E.g., a value for a custom field with field_id 12 would be specified in
# an HTML form element with name="custom_12".
_CUSTOM_FIELD_NAME_PREFIX = 'custom_'
# When the attachment quota gets within 1MB of the limit, stop offering
# users the option to attach files.
_SOFT_QUOTA_LEEWAY = 1024 * 1024
# Accessors for sorting built-in fields.
SORTABLE_FIELDS = {
'project': lambda issue: issue.project_name,
'id': lambda issue: issue.local_id,
'owner': tracker_bizobj.GetOwnerId, # And postprocessor
'reporter': lambda issue: issue.reporter_id, # And postprocessor
'component': lambda issue: issue.component_ids,
'cc': tracker_bizobj.GetCcIds, # And postprocessor
'summary': lambda issue: issue.summary.lower(),
'stars': lambda issue: issue.star_count,
'attachments': lambda issue: issue.attachment_count,
'opened': lambda issue: issue.opened_timestamp,
'closed': lambda issue: issue.closed_timestamp,
'modified': lambda issue: issue.modified_timestamp,
'status': tracker_bizobj.GetStatus,
'blocked': lambda issue: bool(issue.blocked_on_iids),
'blockedon': lambda issue: issue.blocked_on_iids or sorting.MAX_STRING,
'blocking': lambda issue: issue.blocking_iids or sorting.MAX_STRING,
'mergedinto': lambda issue: issue.merged_into or sorting.MAX_STRING,
'ownermodified': lambda issue: issue.owner_modified_timestamp,
'statusmodified': lambda issue: issue.status_modified_timestamp,
'componentmodified': lambda issue: issue.component_modified_timestamp,
'ownerlastvisit': tracker_bizobj.GetOwnerId, # And postprocessor
}
# Some fields take a user ID from the issue and then use that to index
# into a dictionary of user views, and then get a field of the user view
# as the value to sort key.
SORTABLE_FIELDS_POSTPROCESSORS = {
'owner': lambda user_view: user_view.email,
'reporter': lambda user_view: user_view.email,
'cc': lambda user_view: user_view.email,
'ownerlastvisit': lambda user_view: -user_view.user.last_visit_timestamp,
}
# Here are some restriction labels to help people do the most common things
# that they might want to do with restrictions.
_FREQUENT_ISSUE_RESTRICTIONS = [
(permissions.VIEW, permissions.EDIT_ISSUE,
'Only users who can edit the issue may access it'),
(permissions.ADD_ISSUE_COMMENT, permissions.EDIT_ISSUE,
'Only users who can edit the issue may add comments'),
]
# These issue restrictions should be offered as examples whenever the project
# does not have any custom permissions in use already.
_EXAMPLE_ISSUE_RESTRICTIONS = [
(permissions.VIEW, 'CoreTeam',
'Custom permission CoreTeam is needed to access'),
]
# Namedtuples that hold data parsed from post_data.
ParsedComponents = collections.namedtuple(
'ParsedComponents', 'entered_str, paths, paths_remove')
ParsedFields = collections.namedtuple(
'ParsedFields',
'vals, vals_remove, fields_clear, '
'phase_vals, phase_vals_remove')
ParsedUsers = collections.namedtuple(
'ParsedUsers', 'owner_username, owner_id, cc_usernames, '
'cc_usernames_remove, cc_ids, cc_ids_remove')
ParsedBlockers = collections.namedtuple(
'ParsedBlockers', 'entered_str, iids, dangling_refs, '
'federated_ref_strings')
ParsedHotlistRef = collections.namedtuple(
'ParsedHotlistRef', 'user_email, hotlist_name')
ParsedHotlists = collections.namedtuple(
'ParsedHotlists', 'entered_str, hotlist_refs')
ParsedIssue = collections.namedtuple(
'ParsedIssue', 'summary, comment, is_description, status, users, labels, '
'labels_remove, components, fields, template_name, attachments, '
'kept_attachments, blocked_on, blocking, hotlists')
def ParseIssueRequest(cnxn, post_data, services, errors, default_project_name):
"""Parse all the possible arguments out of the request.
Args:
cnxn: connection to SQL database.
post_data: HTML form information.
services: Connections to persistence layer.
errors: object to accumulate validation error info.
default_project_name: name of the project that contains the issue.
Returns:
A namedtuple with all parsed information. User IDs are looked up, but
also the strings are returned to allow bouncing the user back to correct
any errors.
"""
summary = post_data.get('summary', '')
comment = post_data.get('comment', '')
is_description = bool(post_data.get('description', ''))
status = post_data.get('status', '')
template_name = urllib.unquote_plus(post_data.get('template_name', ''))
component_str = post_data.get('components', '')
label_strs = post_data.getall('label')
if is_description:
tmpl_txt = post_data.get('tmpl_txt', '')
comment = MarkupDescriptionOnInput(comment, tmpl_txt)
comp_paths, comp_paths_remove = _ClassifyPlusMinusItems(
re.split('[,;\s]+', component_str))
parsed_components = ParsedComponents(
component_str, comp_paths, comp_paths_remove)
labels, labels_remove = _ClassifyPlusMinusItems(label_strs)
parsed_fields = _ParseIssueRequestFields(post_data)
# TODO(jrobbins): change from numbered fields to a multi-valued field.
attachments = _ParseIssueRequestAttachments(post_data)
kept_attachments = _ParseIssueRequestKeptAttachments(post_data)
parsed_users = _ParseIssueRequestUsers(cnxn, post_data, services)
parsed_blocked_on = _ParseBlockers(
cnxn, post_data, services, errors, default_project_name, BLOCKED_ON)
parsed_blocking = _ParseBlockers(
cnxn, post_data, services, errors, default_project_name, BLOCKING)
parsed_hotlists = _ParseHotlists(post_data)
parsed_issue = ParsedIssue(
summary, comment, is_description, status, parsed_users, labels,
labels_remove, parsed_components, parsed_fields, template_name,
attachments, kept_attachments, parsed_blocked_on, parsed_blocking,
parsed_hotlists)
return parsed_issue
def MarkupDescriptionOnInput(content, tmpl_text):
"""Return HTML for the content of an issue description or comment.
Args:
content: the text sumbitted by the user, any user-entered markup
has already been escaped.
tmpl_text: the initial text that was put into the textarea.
Returns:
The description content text with template lines highlighted.
"""
tmpl_lines = tmpl_text.split('\n')
tmpl_lines = [pl.strip() for pl in tmpl_lines if pl.strip()]
entered_lines = content.split('\n')
marked_lines = [_MarkupDescriptionLineOnInput(line, tmpl_lines)
for line in entered_lines]
return '\n'.join(marked_lines)
def _MarkupDescriptionLineOnInput(line, tmpl_lines):
"""Markup one line of an issue description that was just entered.
Args:
line: string containing one line of the user-entered comment.
tmpl_lines: list of strings for the text of the template lines.
Returns:
The same user-entered line, or that line highlighted to
indicate that it came from the issue template.
"""
for tmpl_line in tmpl_lines:
if line.startswith(tmpl_line):
return '<b>' + tmpl_line + '</b>' + line[len(tmpl_line):]
return line
def _ClassifyPlusMinusItems(add_remove_list):
"""Classify the given plus-or-minus items into add and remove lists."""
add_remove_set = {s.strip() for s in add_remove_list}
add_strs = [s for s in add_remove_set if s and not s.startswith('-')]
remove_strs = [s[1:] for s in add_remove_set if s[1:] and s.startswith('-')]
return add_strs, remove_strs
def _ParseHotlists(post_data):
entered_str = post_data.get('hotlists', '').strip()
hotlist_refs = []
for ref_str in re.split('[,;\s]+', entered_str):
if not ref_str:
continue
if ':' in ref_str:
if ref_str.split(':')[0]:
# E-mail isn't empty; full reference.
hotlist_refs.append(ParsedHotlistRef(*ref_str.split(':', 1)))
else:
# Short reference.
hotlist_refs.append(ParsedHotlistRef(None, ref_str.split(':', 1)[1]))
else:
# Short reference
hotlist_refs.append(ParsedHotlistRef(None, ref_str))
parsed_hotlists = ParsedHotlists(entered_str, hotlist_refs)
return parsed_hotlists
def _ParseIssueRequestFields(post_data):
"""Iterate over post_data and return custom field values found in it."""
field_val_strs = {}
field_val_strs_remove = {}
phase_field_val_strs = collections.defaultdict(dict)
phase_field_val_strs_remove = collections.defaultdict(dict)
for key in post_data.keys():
if key.startswith(_CUSTOM_FIELD_NAME_PREFIX):
val_strs = [v for v in post_data.getall(key) if v]
if val_strs:
try:
field_id = int(key[len(_CUSTOM_FIELD_NAME_PREFIX):])
phase_name = None
except ValueError: # key must be in format <field_id>_<phase_name>
field_id, phase_name = key[len(_CUSTOM_FIELD_NAME_PREFIX):].split(
'_', 1)
field_id = int(field_id)
if post_data.get('op_' + key) == 'remove':
if phase_name:
phase_field_val_strs_remove[field_id][phase_name] = val_strs
else:
field_val_strs_remove[field_id] = val_strs
else:
if phase_name:
phase_field_val_strs[field_id][phase_name] = val_strs
else:
field_val_strs[field_id] = val_strs
# TODO(jojwang): monorail:5154, no support for clearing phase field values.
fields_clear = []
op_prefix = 'op_' + _CUSTOM_FIELD_NAME_PREFIX
for op_key in post_data.keys():
if op_key.startswith(op_prefix):
if post_data.get(op_key) == 'clear':
field_id = int(op_key[len(op_prefix):])
fields_clear.append(field_id)
return ParsedFields(
field_val_strs, field_val_strs_remove, fields_clear,
phase_field_val_strs, phase_field_val_strs_remove)
def _ParseIssueRequestAttachments(post_data):
"""Extract and clean-up any attached files from the post data.
Args:
post_data: dict w/ values from the user's HTTP POST form data.
Returns:
[(filename, filecontents, mimetype), ...] with items for each attachment.
"""
# TODO(jrobbins): change from numbered fields to a multi-valued field.
attachments = []
for i in range(1, 16):
if 'file%s' % i in post_data:
item = post_data['file%s' % i]
if isinstance(item, string_types):
continue
if '\\' in item.filename: # IE insists on giving us the whole path.
item.filename = item.filename[item.filename.rindex('\\') + 1:]
if not item.filename:
continue # Skip any FILE fields that were not filled in.
attachments.append((
item.filename, item.value,
filecontent.GuessContentTypeFromFilename(item.filename)))
return attachments
def _ParseIssueRequestKeptAttachments(post_data):
"""Extract attachment ids for attachments kept when updating description
Args:
post_data: dict w/ values from the user's HTTP POST form data.
Returns:
a list of attachment ids for kept attachments
"""
kept_attachments = post_data.getall('keep-attachment')
return [int(aid) for aid in kept_attachments]
def _ParseIssueRequestUsers(cnxn, post_data, services):
"""Extract usernames from the POST data, categorize them, and look up IDs.
Args:
cnxn: connection to SQL database.
post_data: dict w/ data from the HTTP POST.
services: Services.
Returns:
A namedtuple (owner_username, owner_id, cc_usernames, cc_usernames_remove,
cc_ids, cc_ids_remove), containing:
- issue owner's name and user ID, if any
- the list of all cc'd usernames
- the user IDs to add or remove from the issue CC list.
Any of these user IDs may be None if the corresponding username
or email address is invalid.
"""
# Get the user-entered values from post_data.
cc_username_str = post_data.get('cc', '').lower()
owner_email = post_data.get('owner', '').strip().lower()
cc_usernames, cc_usernames_remove = _ClassifyPlusMinusItems(
re.split('[,;\s]+', cc_username_str))
# Figure out the email addresses to lookup and do the lookup.
emails_to_lookup = cc_usernames + cc_usernames_remove
if owner_email:
emails_to_lookup.append(owner_email)
all_user_ids = services.user.LookupUserIDs(
cnxn, emails_to_lookup, autocreate=True)
if owner_email:
owner_id = all_user_ids.get(owner_email)
else:
owner_id = framework_constants.NO_USER_SPECIFIED
# Lookup the user IDs of the Cc addresses to add or remove.
cc_ids = [all_user_ids.get(cc) for cc in cc_usernames if cc]
cc_ids_remove = [all_user_ids.get(cc) for cc in cc_usernames_remove if cc]
return ParsedUsers(owner_email, owner_id, cc_usernames, cc_usernames_remove,
cc_ids, cc_ids_remove)
def _ParseBlockers(cnxn, post_data, services, errors, default_project_name,
field_name):
"""Parse input for issues that the current issue is blocking/blocked on.
Args:
cnxn: connection to SQL database.
post_data: dict w/ values from the user's HTTP POST.
services: connections to backend services.
errors: object to accumulate validation error info.
default_project_name: name of the project that contains the issue.
field_name: string HTML input field name, e.g., BLOCKED_ON or BLOCKING.
Returns:
A namedtuple with the user input string, and a list of issue IDs.
"""
entered_str = post_data.get(field_name, '').strip()
blocker_iids = []
dangling_ref_tuples = []
federated_ref_strings = []
issue_ref = None
for ref_str in re.split('[,;\s]+', entered_str):
# Handle federated references.
if federated.IsShortlinkValid(ref_str):
federated_ref_strings.append(ref_str)
continue
try:
issue_ref = tracker_bizobj.ParseIssueRef(ref_str)
except ValueError:
setattr(errors, field_name, 'Invalid issue ID %s' % ref_str.strip())
break
if not issue_ref:
continue
blocker_project_name, blocker_issue_id = issue_ref
if not blocker_project_name:
blocker_project_name = default_project_name
# Detect and report if the same issue was specified.
current_issue_id = int(post_data.get('id')) if post_data.get('id') else -1
if (blocker_issue_id == current_issue_id and
blocker_project_name == default_project_name):
setattr(errors, field_name, 'Cannot be %s the same issue' % field_name)
break
ref_projects = services.project.GetProjectsByName(
cnxn, set([blocker_project_name]))
blocker_iid, _misses = services.issue.ResolveIssueRefs(
cnxn, ref_projects, default_project_name, [issue_ref])
if not blocker_iid:
if blocker_project_name in settings.recognized_codesite_projects:
# We didn't find the issue, but it had a explicitly-specified project
# which we know is on Codesite. Allow it as a dangling reference.
dangling_ref_tuples.append(issue_ref)
continue
else:
# Otherwise, it doesn't exist, so report it.
setattr(errors, field_name, 'Invalid issue ID %s' % ref_str.strip())
break
if blocker_iid[0] not in blocker_iids:
blocker_iids.extend(blocker_iid)
blocker_iids.sort()
dangling_ref_tuples.sort()
return ParsedBlockers(entered_str, blocker_iids, dangling_ref_tuples,
federated_ref_strings)
def PairDerivedValuesWithRuleExplanations(
proposed_issue, traces, derived_users_by_id):
"""Pair up values and explanations into JSON objects."""
derived_labels_and_why = [
{'value': lab,
'why': traces.get((tracker_pb2.FieldID.LABELS, lab))}
for lab in proposed_issue.derived_labels]
derived_users_by_id = {
user_id: user_view.display_name
for user_id, user_view in derived_users_by_id.items()
if user_view.display_name}
derived_owner_and_why = []
if proposed_issue.derived_owner_id:
derived_owner_and_why = [{
'value': derived_users_by_id[proposed_issue.derived_owner_id],
'why': traces.get(
(tracker_pb2.FieldID.OWNER, proposed_issue.derived_owner_id))}]
derived_cc_and_why = [
{'value': derived_users_by_id[cc_id],
'why': traces.get((tracker_pb2.FieldID.CC, cc_id))}
for cc_id in proposed_issue.derived_cc_ids
if cc_id in derived_users_by_id]
warnings_and_why = [
{'value': warning,
'why': traces.get((tracker_pb2.FieldID.WARNING, warning))}
for warning in proposed_issue.derived_warnings]
errors_and_why = [
{'value': error,
'why': traces.get((tracker_pb2.FieldID.ERROR, error))}
for error in proposed_issue.derived_errors]
return (derived_labels_and_why, derived_owner_and_why, derived_cc_and_why,
warnings_and_why, errors_and_why)
def IsValidIssueOwner(cnxn, project, owner_id, services):
"""Return True if the given user ID can be an issue owner.
Args:
cnxn: connection to SQL database.
project: the current Project PB.
owner_id: the user ID of the proposed issue owner.
services: connections to backends.
It is OK to have 0 for the owner_id, that simply means that the issue is
unassigned.
Returns:
A pair (valid, err_msg). valid is True if the given user ID can be an
issue owner. err_msg is an error message string to display to the user
if valid == False, and is None if valid == True.
"""
# An issue is always allowed to have no owner specified.
if owner_id == framework_constants.NO_USER_SPECIFIED:
return True, None
try:
auth = authdata.AuthData.FromUserID(cnxn, owner_id, services)
if not framework_bizobj.UserIsInProject(project, auth.effective_ids):
return False, 'Issue owner must be a project member.'
except exceptions.NoSuchUserException:
return False, 'Issue owner user ID not found.'
group_ids = services.usergroup.DetermineWhichUserIDsAreGroups(
cnxn, [owner_id])
if owner_id in group_ids:
return False, 'Issue owner cannot be a user group.'
return True, None
def GetAllowedOpenedAndClosedIssues(mr, issue_ids, services):
"""Get filtered lists of open and closed issues identified by issue_ids.
The function then filters the results to only the issues that the user
is allowed to view. E.g., we only auto-link to issues that the user
would be able to view if they clicked the link.
Args:
mr: commonly used info parsed from the request.
issue_ids: list of int issue IDs for the target issues.
services: connection to issue, config, and project persistence layers.
Returns:
Two lists of issues that the user is allowed to view: one for open
issues and one for closed issues.
"""
open_issues, closed_issues = services.issue.GetOpenAndClosedIssues(
mr.cnxn, issue_ids)
return GetAllowedIssues(mr, [open_issues, closed_issues], services)
def GetAllowedIssues(mr, issue_groups, services):
"""Filter lists of issues identified by issue_groups.
Args:
mr: commonly used info parsed from the request.
issue_groups: list of list of issues to filter.
services: connection to issue, config, and project persistence layers.
Returns:
List of filtered list of issues.
"""
project_dict = GetAllIssueProjects(
mr.cnxn, itertools.chain.from_iterable(issue_groups), services.project)
config_dict = services.config.GetProjectConfigs(mr.cnxn,
list(project_dict.keys()))
return [FilterOutNonViewableIssues(
mr.auth.effective_ids, mr.auth.user_pb, project_dict, config_dict,
issues)
for issues in issue_groups]
def MakeViewsForUsersInIssues(cnxn, issue_list, user_service, omit_ids=None):
"""Lookup all the users involved in any of the given issues.
Args:
cnxn: connection to SQL database.
issue_list: list of Issue PBs from a result query.
user_service: Connection to User backend storage.
omit_ids: a list of user_ids to omit, e.g., because we already have them.
Returns:
A dictionary {user_id: user_view,...} for all the users involved
in the given issues.
"""
issue_participant_id_set = tracker_bizobj.UsersInvolvedInIssues(issue_list)
if omit_ids:
issue_participant_id_set.difference_update(omit_ids)
# TODO(jrobbins): consider caching View objects as well.
users_by_id = framework_views.MakeAllUserViews(
cnxn, user_service, issue_participant_id_set)
return users_by_id
def FormatIssueListURL(
mr, config, absolute=True, project_names=None, **kwargs):
"""Format a link back to list view as configured by user."""
if project_names is None:
project_names = [mr.project_name]
if tracker_constants.JUMP_RE.match(mr.query):
kwargs['q'] = 'id=%s' % mr.query
kwargs['can'] = 1 # The specified issue might be closed.
else:
kwargs['q'] = mr.query
if mr.can and mr.can != 2:
kwargs['can'] = mr.can
def_col_spec = config.default_col_spec
if mr.col_spec and mr.col_spec != def_col_spec:
kwargs['colspec'] = mr.col_spec
if mr.sort_spec:
kwargs['sort'] = mr.sort_spec
if mr.group_by_spec:
kwargs['groupby'] = mr.group_by_spec
if mr.start:
kwargs['start'] = mr.start
if mr.num != tracker_constants.DEFAULT_RESULTS_PER_PAGE:
kwargs['num'] = mr.num
if len(project_names) == 1:
url = '/p/%s%s' % (project_names[0], urls.ISSUE_LIST)
else:
url = urls.ISSUE_LIST
kwargs['projects'] = ','.join(sorted(project_names))
param_strings = ['%s=%s' % (k, urllib.quote((u'%s' % v).encode('utf-8')))
for k, v in kwargs.items()]
if param_strings:
url += '?' + '&'.join(sorted(param_strings))
if absolute:
url = '%s://%s%s' % (mr.request.scheme, mr.request.host, url)
return url
def FormatRelativeIssueURL(project_name, path, **kwargs):
"""Format a URL to get to an issue in the named project.
Args:
project_name: string name of the project containing the issue.
path: string servlet path, e.g., from framework/urls.py.
**kwargs: additional query-string parameters to include in the URL.
Returns:
A URL string.
"""
return framework_helpers.FormatURL(
None, '/p/%s%s' % (project_name, path), **kwargs)
def FormatCrBugURL(project_name, local_id):
"""Format a short URL to get to an issue in the named project.
Args:
project_name: string name of the project containing the issue.
local_id: int local ID of the issue.
Returns:
A URL string.
"""
if app_identity.get_application_id() != 'monorail-prod':
return FormatRelativeIssueURL(
project_name, urls.ISSUE_DETAIL, id=local_id)
if project_name == 'chromium':
return 'https://crbug.com/%d' % local_id
return 'https://crbug.com/%s/%d' % (project_name, local_id)
def ComputeNewQuotaBytesUsed(project, attachments):
"""Add the given attachments to the project's attachment quota usage.
Args:
project: Project PB for the project being updated.
attachments: a list of attachments being added to an issue.
Returns:
The new number of bytes used.
Raises:
OverAttachmentQuota: If project would go over quota.
"""
total_attach_size = 0
for _filename, content, _mimetype in attachments:
total_attach_size += len(content)
new_bytes_used = project.attachment_bytes_used + total_attach_size
quota = (project.attachment_quota or
tracker_constants.ISSUE_ATTACHMENTS_QUOTA_HARD)
if new_bytes_used > quota:
raise exceptions.OverAttachmentQuota(new_bytes_used - quota)
return new_bytes_used
def IsUnderSoftAttachmentQuota(project):
"""Check the project's attachment quota against the soft quota limit.
If there is a custom quota on the project, this will check against
that instead of the system-wide default quota.
Args:
project: Project PB for the project to examine
Returns:
True if the project is under quota, false otherwise.
"""
quota = tracker_constants.ISSUE_ATTACHMENTS_QUOTA_SOFT
if project.attachment_quota:
quota = project.attachment_quota - _SOFT_QUOTA_LEEWAY
return project.attachment_bytes_used < quota
def GetAllIssueProjects(cnxn, issues, project_service):
"""Get all the projects that the given issues belong to.
Args:
cnxn: connection to SQL database.
issues: list of issues, which may come from different projects.
project_service: connection to project persistence layer.
Returns:
A dictionary {project_id: project} of all the projects that
any of the given issues belongs to.
"""
needed_project_ids = {issue.project_id for issue in issues}
project_dict = project_service.GetProjects(cnxn, needed_project_ids)
return project_dict
def GetPermissionsInAllProjects(user, effective_ids, projects):
"""Look up the permissions for the given user in each project."""
return {
project.project_id:
permissions.GetPermissions(user, effective_ids, project)
for project in projects}
def FilterOutNonViewableIssues(
effective_ids, user, project_dict, config_dict, issues):
"""Return a filtered list of issues that the user can view."""
perms_dict = GetPermissionsInAllProjects(
user, effective_ids, list(project_dict.values()))
denied_project_ids = {
pid for pid, p in project_dict.items()
if not permissions.CanView(effective_ids, perms_dict[pid], p, [])}
results = []
for issue in issues:
if issue.deleted or issue.project_id in denied_project_ids:
continue
if not permissions.HasRestrictions(issue):
may_view = True
else:
perms = perms_dict[issue.project_id]
project = project_dict[issue.project_id]
config = config_dict.get(issue.project_id, config_dict.get('harmonized'))
granted_perms = tracker_bizobj.GetGrantedPerms(
issue, effective_ids, config)
may_view = permissions.CanViewIssue(
effective_ids, perms, project, issue, granted_perms=granted_perms)
if may_view:
results.append(issue)
return results
def MeansOpenInProject(status, config):
"""Return true if this status means that the issue is still open.
This defaults to true if we could not find a matching status.
Args:
status: issue status string. E.g., 'New'.
config: the config of the current project.
Returns:
Boolean True if the status means that the issue is open.
"""
status_lower = status.lower()
# iterate over the list of known statuses for this project
# return true if we find a match that declares itself to be open
for wks in config.well_known_statuses:
if wks.status.lower() == status_lower:
return wks.means_open
return True
def IsNoisy(num_comments, num_starrers):
"""Return True if this is a "noisy" issue that would send a ton of emails.
The rule is that a very active issue with a large number of comments
and starrers will only send notification when a comment (or change)
is made by a project member.
Args:
num_comments: int number of comments on issue so far.
num_starrers: int number of users who starred the issue.
Returns:
True if we will not bother starrers with an email notification for
changes made by non-members.
"""
return (num_comments >= tracker_constants.NOISY_ISSUE_COMMENT_COUNT and
num_starrers >= tracker_constants.NOISY_ISSUE_STARRER_COUNT)
def MergeCCsAndAddComment(services, mr, issue, merge_into_issue):
"""Modify the CC field of the target issue and add a comment to it."""
return MergeCCsAndAddCommentMultipleIssues(
services, mr, [issue], merge_into_issue)
def MergeCCsAndAddCommentMultipleIssues(
services, mr, issues, merge_into_issue):
"""Modify the CC field of the target issue and add a comment to it."""
merge_comment = ''
for issue in issues:
if issue.project_name == merge_into_issue.project_name:
issue_ref_str = '%d' % issue.local_id
else:
issue_ref_str = '%s:%d' % (issue.project_name, issue.local_id)
if merge_comment:
merge_comment += '\n'
merge_comment += 'Issue %s has been merged into this issue.' % issue_ref_str
add_cc = _ComputeNewCcsFromIssueMerge(merge_into_issue, issues)
config = services.config.GetProjectConfig(
mr.cnxn, merge_into_issue.project_id)
delta = tracker_pb2.IssueDelta(cc_ids_add=add_cc)
_, merge_comment_pb = services.issue.DeltaUpdateIssue(
mr.cnxn, services, mr.auth.user_id, merge_into_issue.project_id,
config, merge_into_issue, delta, index_now=False, comment=merge_comment)
return merge_comment_pb
def GetAttachmentIfAllowed(mr, services):
"""Retrieve the requested attachment, or raise an appropriate exception.
Args:
mr: commonly used info parsed from the request.
services: connections to backend services.
Returns:
The requested Attachment PB, and the Issue that it belongs to.
Raises:
NoSuchAttachmentException: attachment was not found or was marked deleted.
NoSuchIssueException: issue that contains attachment was not found.
PermissionException: the user is not allowed to view the attachment.
"""
attachment = None
attachment, cid, issue_id = services.issue.GetAttachmentAndContext(
mr.cnxn, mr.aid)
issue = services.issue.GetIssue(mr.cnxn, issue_id)
config = services.config.GetProjectConfig(mr.cnxn, issue.project_id)
granted_perms = tracker_bizobj.GetGrantedPerms(
issue, mr.auth.effective_ids, config)
permit_view = permissions.CanViewIssue(
mr.auth.effective_ids, mr.perms, mr.project, issue,
granted_perms=granted_perms)
if not permit_view:
raise permissions.PermissionException('Cannot view attachment\'s issue')
comment = services.issue.GetComment(mr.cnxn, cid)
commenter = services.user.GetUser(mr.cnxn, comment.user_id)
issue_perms = permissions.UpdateIssuePermissions(
mr.perms, mr.project, issue, mr.auth.effective_ids,
granted_perms=granted_perms)
can_view_comment = permissions.CanViewComment(
comment, commenter, mr.auth.user_id, issue_perms)
if not can_view_comment:
raise permissions.PermissionException('Cannot view attachment\'s comment')
return attachment, issue
def LabelsMaskedByFields(config, field_names, trim_prefix=False):
"""Return a list of EZTItems for labels that would be masked by fields."""
return _LabelsMaskedOrNot(config, field_names, trim_prefix=trim_prefix)
def LabelsNotMaskedByFields(config, field_names, trim_prefix=False):
"""Return a list of EZTItems for labels that would not be masked."""
return _LabelsMaskedOrNot(
config, field_names, invert=True, trim_prefix=trim_prefix)
def _LabelsMaskedOrNot(config, field_names, invert=False, trim_prefix=False):
"""Return EZTItems for labels that'd be masked. Or not, when invert=True."""
field_names = [fn.lower() for fn in field_names]
result = []
for wkl in config.well_known_labels:
masked_by = tracker_bizobj.LabelIsMaskedByField(wkl.label, field_names)
if (masked_by and not invert) or (not masked_by and invert):
display_name = wkl.label
if trim_prefix:
display_name = display_name[len(masked_by) + 1:]
result.append(template_helpers.EZTItem(
name=display_name,
name_padded=display_name.ljust(20),
commented='#' if wkl.deprecated else '',
docstring=wkl.label_docstring,
docstring_short=template_helpers.FitUnsafeText(
wkl.label_docstring, 40),
idx=len(result)))
return result
def LookupComponentIDs(component_paths, config, errors=None):
"""Look up the IDs of the specified components in the given config."""
component_ids = []
for path in component_paths:
if not path:
continue
cd = tracker_bizobj.FindComponentDef(path, config)
if cd:
component_ids.append(cd.component_id)
else:
error_text = 'Unknown component %s' % path
if errors:
errors.components = error_text
else:
logging.info(error_text)
return component_ids
def ParsePostDataUsers(cnxn, pd_users_str, user_service):
"""Parse all the usernames from a users string found in a post data."""
emails, _remove = _ClassifyPlusMinusItems(re.split('[,;\s]+', pd_users_str))
users_ids_by_email = user_service.LookupUserIDs(cnxn, emails, autocreate=True)
user_ids = [users_ids_by_email[username] for username in emails if username]
return user_ids, pd_users_str
def FilterIssueTypes(config):
"""Return a list of well-known issue types."""
well_known_issue_types = []
for wk_label in config.well_known_labels:
if wk_label.label.lower().startswith('type-'):
_, type_name = wk_label.label.split('-', 1)
well_known_issue_types.append(type_name)
return well_known_issue_types
def ParseMergeFields(
cnxn, services, project_name, post_data, status, config, issue, errors):
"""Parse info that identifies the issue to merge into, if any."""
merge_into_text = ''
merge_into_ref = None
merge_into_issue = None
if status not in config.statuses_offer_merge:
return '', None
merge_into_text = post_data.get('merge_into', '')
if merge_into_text:
try:
merge_into_ref = tracker_bizobj.ParseIssueRef(merge_into_text)
except ValueError:
logging.info('merge_into not an int: %r', merge_into_text)
errors.merge_into_id = 'Please enter a valid issue ID'
if not merge_into_ref:
errors.merge_into_id = 'Please enter an issue ID'
return merge_into_text, None
merge_into_project_name, merge_into_id = merge_into_ref
if (merge_into_id == issue.local_id and
(merge_into_project_name == project_name or
not merge_into_project_name)):
logging.info('user tried to merge issue into itself: %r', merge_into_ref)
errors.merge_into_id = 'Cannot merge issue into itself'
return merge_into_text, None
project = services.project.GetProjectByName(
cnxn, merge_into_project_name or project_name)
try:
# Because we will modify this issue, load from DB rather than cache.
merge_into_issue = services.issue.GetIssueByLocalID(
cnxn, project.project_id, merge_into_id, use_cache=False)
except Exception:
logging.info('merge_into issue not found: %r', merge_into_ref)
errors.merge_into_id = 'No such issue'
return merge_into_text, None
return merge_into_text, merge_into_issue
def GetNewIssueStarrers(cnxn, services, issue_ids, merge_into_iid):
# type: (MonorailConnection, Services, Sequence[int], int) ->
# Collection[int]
"""Get starrers of current issue who have not starred the target issue."""
source_starrers_dict = services.issue_star.LookupItemsStarrers(
cnxn, issue_ids)
source_starrers = list(
itertools.chain.from_iterable(source_starrers_dict.values()))
target_starrers = services.issue_star.LookupItemStarrers(
cnxn, merge_into_iid)
return set(source_starrers) - set(target_starrers)
def AddIssueStarrers(
cnxn, services, mr, merge_into_iid, merge_into_project, new_starrers):
"""Merge all the starrers for the current issue into the target issue."""
project = merge_into_project or mr.project
config = services.config.GetProjectConfig(mr.cnxn, project.project_id)
services.issue_star.SetStarsBatch(
cnxn, services, config, merge_into_iid, new_starrers, True)
def IsMergeAllowed(merge_into_issue, mr, services):
"""Check to see if user has permission to merge with specified issue."""
merge_into_project = services.project.GetProjectByName(
mr.cnxn, merge_into_issue.project_name)
merge_into_config = services.config.GetProjectConfig(
mr.cnxn, merge_into_project.project_id)
merge_granted_perms = tracker_bizobj.GetGrantedPerms(
merge_into_issue, mr.auth.effective_ids, merge_into_config)
merge_view_allowed = mr.perms.CanUsePerm(
permissions.VIEW, mr.auth.effective_ids,
merge_into_project, permissions.GetRestrictions(merge_into_issue),
granted_perms=merge_granted_perms)
merge_edit_allowed = mr.perms.CanUsePerm(
permissions.EDIT_ISSUE, mr.auth.effective_ids,
merge_into_project, permissions.GetRestrictions(merge_into_issue),
granted_perms=merge_granted_perms)
return merge_view_allowed and merge_edit_allowed
def GetVisibleMembers(mr, project, services):
all_member_ids = project_helpers.AllProjectMembers(project)
all_group_ids = services.usergroup.DetermineWhichUserIDsAreGroups(
mr.cnxn, all_member_ids)
(ac_exclusion_ids, no_expand_ids
) = services.project.GetProjectAutocompleteExclusion(
mr.cnxn, project.project_id)
group_ids_to_expand = [
gid for gid in all_group_ids if gid not in no_expand_ids]
# TODO(jrobbins): Normally, users will be allowed view the members
# of any user group if the project From: email address is listed
# as a group member, as well as any group that they are personally
# members of.
member_ids, owner_ids = services.usergroup.LookupVisibleMembers(
mr.cnxn, group_ids_to_expand, mr.perms, mr.auth.effective_ids, services)
indirect_user_ids = set()
for gids in member_ids.values():
indirect_user_ids.update(gids)
for gids in owner_ids.values():
indirect_user_ids.update(gids)
visible_member_ids = _FilterMemberData(
mr, project.owner_ids, project.committer_ids, project.contributor_ids,
indirect_user_ids, project)
visible_member_ids = _MergeLinkedMembers(
mr.cnxn, services.user, visible_member_ids)
visible_member_views = framework_views.MakeAllUserViews(
mr.cnxn, services.user, visible_member_ids, group_ids=all_group_ids)
framework_views.RevealAllEmailsToMembers(
mr.cnxn, services, mr.auth, visible_member_views, project)
# Filter out service accounts
service_acct_emails = set(
client_config_svc.GetClientConfigSvc().GetClientIDEmails()[1])
visible_member_views = {
m.user_id: m
for m in visible_member_views.values()
# Hide service accounts from autocomplete.
if not framework_helpers.IsServiceAccount(
m.email, client_emails=service_acct_emails)
# Hide users who opted out of autocomplete.
and not m.user_id in ac_exclusion_ids
# Hide users who have obscured email addresses.
and not m.obscure_email
}
return visible_member_views
def _MergeLinkedMembers(cnxn, user_service, user_ids):
"""Remove any linked child accounts if the parent would also be shown."""
all_ids = set(user_ids)
users_by_id = user_service.GetUsersByIDs(cnxn, user_ids)
result = [uid for uid in user_ids
if users_by_id[uid].linked_parent_id not in all_ids]
return result
def _FilterMemberData(
mr, owner_ids, committer_ids, contributor_ids, indirect_member_ids,
project):
"""Return a filtered list of members that the user can view.
In most projects, everyone can view the entire member list. But,
some projects are configured to only allow project owners to see
all members. In those projects, committers and contributors do not
see any contributors. Regardless of how the project is configured
or the role that the user plays in the current project, we include
any indirect members through user groups that the user has access
to view.
Args:
mr: Commonly used info parsed from the HTTP request.
owner_views: list of user IDs for project owners.
committer_views: list of user IDs for project committers.
contributor_views: list of user IDs for project contributors.
indirect_member_views: list of user IDs for users who have
an indirect role in the project via a user group, and that the
logged in user is allowed to see.
project: the Project we're interested in.
Returns:
A list of owners, committer and visible indirect members if the user is not
signed in. If the project is set to display contributors to non-owners or
the signed in user has necessary permissions then additionally a list of
contributors.
"""
visible_members_ids = set()
# Everyone can view owners and committers
visible_members_ids.update(owner_ids)
visible_members_ids.update(committer_ids)
# The list of indirect members is already limited to ones that the user
# is allowed to see according to user group settings.
visible_members_ids.update(indirect_member_ids)
# If the user is allowed to view the list of contributors, add those too.
if permissions.CanViewContributorList(mr, project):
visible_members_ids.update(contributor_ids)
return sorted(visible_members_ids)
def GetLabelOptions(config, custom_permissions):
"""Prepares label options for autocomplete."""
labels = []
field_names = [
fd.field_name
for fd in config.field_defs
if not fd.is_deleted
and fd.field_type is tracker_pb2.FieldTypes.ENUM_TYPE
]
non_masked_labels = LabelsNotMaskedByFields(config, field_names)
for wkl in non_masked_labels:
if not wkl.commented:
item = {'name': wkl.name, 'doc': wkl.docstring}
labels.append(item)
frequent_restrictions = _FREQUENT_ISSUE_RESTRICTIONS[:]
if not custom_permissions:
frequent_restrictions.extend(_EXAMPLE_ISSUE_RESTRICTIONS)
labels.extend(_BuildRestrictionChoices(
frequent_restrictions, permissions.STANDARD_ISSUE_PERMISSIONS,
custom_permissions))
return labels
def _BuildRestrictionChoices(freq_restrictions, actions, custom_permissions):
"""Return a list of autocompletion choices for restriction labels.
Args:
freq_restrictions: list of (action, perm, doc) tuples for restrictions
that are frequently used.
actions: list of strings for actions that are relevant to the current
artifact.
custom_permissions: list of strings with custom permissions for the project.
Returns:
A list of dictionaries [{'name': 'perm name', 'doc': 'docstring'}, ...]
suitable for use in a JSON feed to our JS autocompletion functions.
"""
choices = []
for action, perm, doc in freq_restrictions:
choices.append({
'name': 'Restrict-%s-%s' % (action, perm),
'doc': doc,
})
for action in actions:
for perm in custom_permissions:
choices.append({
'name': 'Restrict-%s-%s' % (action, perm),
'doc': 'Permission %s needed to use %s' % (perm, action),
})
return choices
def FilterKeptAttachments(
is_description, kept_attachments, comments, approval_id):
"""Filter kept attachments to be a subset of last description's attachments.
Args:
is_description: bool, if the comment is a change to the issue description.
kept_attachments: list of ints with the attachment ids for attachments
kept from previous descriptions, if the comment is a change to the
issue description.
comments: list of IssueComment PBs for the issue we want to edit.
approval_id: int id of the APPROVAL_TYPE fielddef, if we're editing an
approval description, or None otherwise.
Returns:
A list of kept_attachment ids that are a subset of the last description.
"""
if not is_description:
return None
attachment_ids = set()
for comment in reversed(comments):
if comment.is_description and comment.approval_id == approval_id:
attachment_ids = set([a.attachment_id for a in comment.attachments])
break
kept_attachments = [
aid for aid in kept_attachments if aid in attachment_ids]
return kept_attachments
def _GetEnumFieldValuesAndDocstrings(field_def, config):
# type: (proto.tracker_pb2.LabelDef, proto.tracker_pb2.ProjectIssueConfig) ->
# Sequence[tuple(string, string)]
"""Get sequence of value, docstring tuples for an enum field"""
label_defs = config.well_known_labels
lower_field_name = field_def.field_name.lower()
tuples = []
for ld in label_defs:
if (ld.label.lower().startswith(lower_field_name + '-') and
not ld.deprecated):
label_value = ld.label[len(lower_field_name) + 1:]
tuples.append((label_value, ld.label_docstring))
else:
continue
return tuples
# _IssueChangesTuple is returned by ApplyAllIssueChanges() and is used to bundle
# the updated issues. resulting amendments, and other information needed by the
# called to process the changes in the DB and send notifications.
_IssueChangesTuple = collections.namedtuple(
'_IssueChangesTuple', [
'issues_to_update_dict', 'merged_from_add_by_iid', 'amendments_by_iid',
'imp_amendments_by_iid', 'old_owners_by_iid', 'old_statuses_by_iid',
'old_components_by_iid', 'new_starrers_by_iid'
])
# type: (Mapping[int, Issue], DefaultDict[int, Sequence[int]],
# Mapping[int, Amendment], Mapping[int, Amendment], Mapping[int, int],
# Mapping[int, str], Mapping[int, Sequence[int]],
# Mapping[int, Sequence[int]])-> None
def ApplyAllIssueChanges(cnxn, issue_delta_pairs, services):
# type: (MonorailConnection, Sequence[Tuple[Issue, IssueDelta]], Services) ->
# IssueChangesTuple
"""Modify the given issues with the given deltas and impacted issues in RAM.
Filter rules are not applied in this method.
This method implements phases 3 and 4 of the process for modifying issues.
See WorkEnv.ModifyIssues() for other phases and overall process.
Args:
cnxn: MonorailConnection object.
issue_delta_pairs: List of tuples that couple Issues with the IssueDeltas
that represent the updates we want to make to each Issue.
services: Services object for connection to backend services.
Returns:
An _IssueChangesTuple named tuple.
"""
impacted_tracker = _IssueChangeImpactedIssues()
project_ids = {issue.project_id for issue, _delta in issue_delta_pairs}
configs_by_pid = services.config.GetProjectConfigs(cnxn, list(project_ids))
# Track issues which have been modified in RAM and will need to
# be updated in the DB.
issues_to_update_dict = {}
amendments_by_iid = {}
old_owners_by_iid = {}
old_statuses_by_iid = {}
old_components_by_iid = {}
# PHASE 3: Update the main issues in RAM (not indirectly, impacted issues).
for issue, delta in issue_delta_pairs:
# Cache old data that will be used by future computations.
old_owner = tracker_bizobj.GetOwnerId(issue)
old_status = tracker_bizobj.GetStatus(issue)
if delta.owner_id is not None and delta.owner_id != old_owner:
old_owners_by_iid[issue.issue_id] = old_owner
if delta.status is not None and delta.status != old_status:
old_statuses_by_iid[issue.issue_id] = old_status
new_components = set(issue.component_ids)
new_components.update(delta.comp_ids_add or [])
new_components.difference_update(delta.comp_ids_remove or [])
if set(issue.component_ids) != new_components:
old_components_by_iid[issue.issue_id] = issue.component_ids
impacted_tracker.TrackImpactedIssues(issue, delta)
config = configs_by_pid.get(issue.project_id)
amendments, _impacted_iids = tracker_bizobj.ApplyIssueDelta(
cnxn, services.issue, issue, delta, config)
if amendments:
issues_to_update_dict[issue.issue_id] = issue
amendments_by_iid[issue.issue_id] = amendments
# PHASE 4: Update impacted issues in RAM.
logging.info('Applying impacted issue changes: %r', impacted_tracker.__dict__)
imp_amendments_by_iid = {}
impacted_iids = impacted_tracker.ComputeAllImpactedIIDs()
new_starrers_by_iid = {}
for issue_id in impacted_iids:
# Changes made to an impacted issue should be on top of changes
# made to it in PHASE 3 where it might have been a 'main' issue.
issue = issues_to_update_dict.get(
issue_id, services.issue.GetIssue(cnxn, issue_id, use_cache=False))
# Apply impacted changes.
amendments, new_starrers = impacted_tracker.ApplyImpactedIssueChanges(
cnxn, issue, services)
if amendments:
imp_amendments_by_iid[issue.issue_id] = amendments
issues_to_update_dict[issue.issue_id] = issue
if new_starrers:
new_starrers_by_iid[issue.issue_id] = new_starrers
return _IssueChangesTuple(
issues_to_update_dict, impacted_tracker.merged_from_add,
amendments_by_iid, imp_amendments_by_iid, old_owners_by_iid,
old_statuses_by_iid, old_components_by_iid, new_starrers_by_iid)
def UpdateClosedTimestamp(config, issue, old_effective_status):
# type: (proto.tracker_pb2.ProjectIssueConfig, proto.tracker_pb2.Issue, str)
# -> None
"""Sets or unsets the closed_timestamp based based on status changes.
If the status is changing from open to closed, the closed_timestamp is set to
the current time.
If the status is changing form closed to open, the close_timestamp is unset.
If the status is changing from one closed to another closed, or from one
open to another open, no operations are performed.
Args:
config: the project configuration
issue: the issue being updated (a protocol buffer)
old_effective_status: the old issue status string. E.g., 'New'
SIDE EFFECTS:
Updated issue in place with new closed timestamp.
"""
old_effective_status = old_effective_status or ''
# open -> closed
if (MeansOpenInProject(old_effective_status, config) and
not MeansOpenInProject(tracker_bizobj.GetStatus(issue), config)):
issue.closed_timestamp = int(time.time())
return
# closed -> open
if (not MeansOpenInProject(old_effective_status, config) and
MeansOpenInProject(tracker_bizobj.GetStatus(issue), config)):
issue.reset('closed_timestamp')
return
def GroupUniqueDeltaIssues(issue_delta_pairs):
# type: (Tuple[Issue, IssueDelta]) -> (
# Sequence[IssueDelta], Sequence[Sequence[Issue]])
"""Identifies unique IssueDeltas and groups Issues with identical IssueDeltas.
Args:
issue_delta_pairs: List of tuples that couple Issues with the IssueDeltas
that represent the updates we want to make to each Issue.
Returns:
(unique_deltas, issues_for_unique_deltas):
unique_deltas: List of unique IssueDeltas found in issue_delta_pairs.
issues_for_unique_deltas: List of Issue lists. Each Issue list
contains all the Issues that had identical IssueDeltas.
Each issues_for_unique_deltas[i] is the list of Issues
that had unique_deltas[i] as their IssueDeltas.
"""
unique_deltas = []
issues_for_unique_deltas = []
for issue, delta in issue_delta_pairs:
try:
delta_index = unique_deltas.index(delta)
issues_for_unique_deltas[delta_index].append(issue)
except ValueError:
# delta is not in unique_deltas yet.
# Add delta to unique_deltas and add a new list of issues
# to issues_for_unique_deltas at the same index.
unique_deltas.append(delta)
issues_for_unique_deltas.append([issue])
return unique_deltas, issues_for_unique_deltas
def _AssertNoConflictingDeltas(issue_delta_pairs, refs_dict, err_agg):
# type: (Sequence[Tuple[Issue, IssueDelta]], Mapping[int, str],
# exceptions.ErrorAggregator) -> None
"""Checks if any issue deltas conflict with each other or themselves.
Note: refs_dict should contain issue ref strings for all issues found
in issue_delta_pairs, including all issues found in
{blocked_on|blocking}_{add|remove}.
"""
err_message = 'Changes for {} conflict for {}'
# Track all delta blocked_on_add and blocking_add in terms of
# 'blocking_add' so we can track when a {blocked_on|blocking}_remove
# is in conflict with some {blocked_on|blocking}_add.
blocking_add = collections.defaultdict(list)
for issue, delta in issue_delta_pairs:
blocking_add[issue.issue_id].extend(delta.blocking_add)
for imp_iid in delta.blocked_on_add:
blocking_add[imp_iid].append(issue.issue_id)
# Check *_remove for conflicts with tracking blocking_add.
for issue, delta in issue_delta_pairs:
added_iids = blocking_add[issue.issue_id]
# Get intersection of iids that are in `blocking_remove` and
# the tracked `blocking_add`.
conflict_iids = set(delta.blocking_remove) & set(added_iids)
# Get iids of `blocked_on_remove` that conflict with the
# tracked `blocking_add`.
for possible_conflict_iid in delta.blocked_on_remove:
if issue.issue_id in blocking_add[possible_conflict_iid]:
conflict_iids.add(possible_conflict_iid)
if conflict_iids:
refs_str = ', '.join([refs_dict[iid] for iid in conflict_iids])
err_agg.AddErrorMessage(err_message, refs_dict[issue.issue_id], refs_str)
def PrepareIssueChanges(
cnxn,
issue_delta_pairs,
services,
attachment_uploads=None,
comment_content=None):
# type: (MonorailConnection, Sequence[Tuple[Issue, IssueDelta]], Services,
# Optional[Sequence[framework_helpers.AttachmentUpload]], Optional[str])
# -> Mapping[int, int]
"""Clean the deltas and assert they are valid for each paired issue."""
_EnforceNonMergeStatusDeltas(cnxn, issue_delta_pairs, services)
_AssertIssueChangesValid(
cnxn, issue_delta_pairs, services, comment_content=comment_content)
if attachment_uploads:
return _EnforceAttachmentQuotaLimits(
cnxn, issue_delta_pairs, services, attachment_uploads)
return {}
def _EnforceAttachmentQuotaLimits(
cnxn, issue_delta_pairs, services, attachment_uploads):
# type: (MonorailConnection, Sequence[Tuple[Issue, IssueDelta]], Services
# Optional[Sequence[framework_helpers.AttachmentUpload]]
# -> Mapping[int, int]
"""Assert that the attachments don't exceed project quotas."""
issue_count_by_pid = collections.defaultdict(int)
for issue, _delta in issue_delta_pairs:
issue_count_by_pid[issue.project_id] += 1
projects_by_id = services.project.GetProjects(cnxn, issue_count_by_pid.keys())
new_bytes_by_pid = {}
with exceptions.ErrorAggregator(exceptions.OverAttachmentQuota) as err_agg:
for pid, count in issue_count_by_pid.items():
project = projects_by_id[pid]
try:
new_bytes_used = ComputeNewQuotaBytesUsed(
project, attachment_uploads * count)
new_bytes_by_pid[pid] = new_bytes_used
except exceptions.OverAttachmentQuota:
err_agg.AddErrorMessage(
'Attachment quota exceeded for project {}', project.project_name)
return new_bytes_by_pid
def _AssertIssueChangesValid(
cnxn, issue_delta_pairs, services, comment_content=None):
# type: (MonorailConnection, Sequence[Tuple[Issue, IssueDelta]], Services,
# Optional[str]) -> None
"""Assert that the delta changes are valid for each paired issue.
Note: this method does not check if the changes trigger any FilterRule
`warnings` or `errors`.
"""
project_ids = list(
{issue.project_id for (issue, _delta) in issue_delta_pairs})
projects_by_id = services.project.GetProjects(cnxn, project_ids)
configs_by_id = services.config.GetProjectConfigs(cnxn, project_ids)
refs_dict = {
iss.issue_id: '%s:%d' % (iss.project_name, iss.local_id)
for iss, _delta in issue_delta_pairs
}
# Add refs of deltas' blocking/blocked_on issues needed by
# _AssertNoConflictingDeltas.
relation_iids = set()
for _iss, delta in issue_delta_pairs:
relation_iids.update(
delta.blocked_on_remove + delta.blocking_remove + delta.blocked_on_add +
delta.blocking_add)
relation_issues_dict, misses = services.issue.GetIssuesDict(
cnxn, relation_iids)
if misses:
raise exceptions.NoSuchIssueException(
'Could not find issues with ids: %r' % misses)
for iid, iss in relation_issues_dict.items():
if iid not in refs_dict:
refs_dict[iid] = '%s:%d' % (iss.project_name, iss.local_id)
with exceptions.ErrorAggregator(exceptions.InputException) as err_agg:
if (comment_content and
len(comment_content.strip()) > tracker_constants.MAX_COMMENT_CHARS):
err_agg.AddErrorMessage('Comment is too long.')
_AssertNoConflictingDeltas(issue_delta_pairs, refs_dict, err_agg)
for issue, delta in issue_delta_pairs:
project = projects_by_id.get(issue.project_id)
config = configs_by_id.get(issue.project_id)
issue_ref = refs_dict[issue.issue_id]
if (delta.merged_into is not None or
delta.merged_into_external is not None or delta.status is not None):
end_status = delta.status or issue.status
merged_options = [
delta.merged_into, delta.merged_into_external, issue.merged_into,
issue.merged_into_external
]
end_merged_into = next(
(merge for merge in merged_options if merge is not None), None)
is_merge_status = end_status.lower() in [
status.lower() for status in config.statuses_offer_merge
]
if ((is_merge_status and not end_merged_into) or
(not is_merge_status and end_merged_into)):
err_agg.AddErrorMessage(
'{}: MERGED type statuses must accompany mergedInto values.',
issue_ref)
if delta.merged_into and issue.issue_id == delta.merged_into:
err_agg.AddErrorMessage(
'{}: Cannot merge an issue into itself.', issue_ref)
if (issue.issue_id in set(
delta.blocked_on_add)) or (issue.issue_id in set(delta.blocking_add)):
err_agg.AddErrorMessage(
'{}: Cannot block an issue on itself.', issue_ref)
if (delta.owner_id is not None) and (delta.owner_id != issue.owner_id):
parsed_owner_valid, msg = IsValidIssueOwner(
cnxn, project, delta.owner_id, services)
if not parsed_owner_valid:
err_agg.AddErrorMessage('{}: {}', issue_ref, msg)
# Owner already check by IsValidIssueOwner
all_users = [uid for uid in delta.cc_ids_add]
field_users = [fv.user_id for fv in delta.field_vals_add if fv.user_id]
all_users.extend(field_users)
AssertUsersExist(cnxn, services, all_users, err_agg)
if (delta.summary and
len(delta.summary.strip()) > tracker_constants.MAX_SUMMARY_CHARS):
err_agg.AddErrorMessage('{}: Summary is too long.', issue_ref)
if delta.summary == '':
err_agg.AddErrorMessage('{}: Summary required.', issue_ref)
if delta.status == '':
err_agg.AddErrorMessage('{}: Status is required.', issue_ref)
# Do not pass in issue for validation, as issue is pre-update, and would
# result in being unable to edit issues in invalid states.
fvs_err_msgs = field_helpers.ValidateCustomFields(
cnxn, services, delta.field_vals_add, config, project)
if fvs_err_msgs:
err_agg.AddErrorMessage('{}: {}', issue_ref, '\n'.join(fvs_err_msgs))
# TODO(crbug.com/monorail/9156): Validate that we do not remove fields
# such that a required field becomes unset.
def AssertUsersExist(cnxn, services, user_ids, err_agg):
# type: (MonorailConnection, Services, Sequence[int], ErrorAggregator) -> None
"""Assert that all users exist.
Has the side-effect of adding error messages to the input ErrorAggregator.
"""
users_dict = services.user.GetUsersByIDs(cnxn, user_ids, skip_missed=True)
found_ids = set(users_dict.keys())
missing = [user_id for user_id in user_ids if user_id not in found_ids]
for missing_user_id in missing:
err_agg.AddErrorMessage(
'users/{}: User does not exist.'.format(missing_user_id))
def AssertValidIssueForCreate(cnxn, services, issue, description):
# type: (MonorailConnection, Services, Issue, str) -> None
"""Assert that issue proto is valid for issue creation.
Args:
cnxn: A connection object to use services with.
services: An object containing services to use to look up relevant data.
issues: A PB containing the issue to validate.
description: The description for the issue.
Raises:
InputException if the issue is not valid.
"""
project = services.project.GetProject(cnxn, issue.project_id)
config = services.config.GetProjectConfig(cnxn, issue.project_id)
with exceptions.ErrorAggregator(exceptions.InputException) as err_agg:
owner_is_valid, owner_err_msg = IsValidIssueOwner(
cnxn, project, issue.owner_id, services)
if not owner_is_valid:
err_agg.AddErrorMessage(owner_err_msg)
if not issue.summary.strip():
err_agg.AddErrorMessage('Summary is required')
if not description.strip():
err_agg.AddErrorMessage('Description is required')
if len(issue.summary) > tracker_constants.MAX_SUMMARY_CHARS:
err_agg.AddErrorMessage('Summary is too long')
if len(description) > tracker_constants.MAX_COMMENT_CHARS:
err_agg.AddErrorMessage('Description is too long')
# Check all users exist. Owner already check by IsValidIssueOwner.
all_users = [uid for uid in issue.cc_ids]
for av in issue.approval_values:
all_users.extend(av.approver_ids)
field_users = [fv.user_id for fv in issue.field_values if fv.user_id]
all_users.extend(field_users)
AssertUsersExist(cnxn, services, all_users, err_agg)
field_validity_errors = field_helpers.ValidateCustomFields(
cnxn, services, issue.field_values, config, project, issue=issue)
if field_validity_errors:
err_agg.AddErrorMessage("\n".join(field_validity_errors))
if not services.config.LookupStatusID(cnxn, issue.project_id, issue.status,
autocreate=False):
err_agg.AddErrorMessage('Undefined status: %s' % issue.status)
all_comp_ids = {
cd.component_id for cd in config.component_defs if not cd.deprecated
}
for comp_id in issue.component_ids:
if comp_id not in all_comp_ids:
err_agg.AddErrorMessage(
'Undefined or deprecated component with id: %d' % comp_id)
def _ComputeNewCcsFromIssueMerge(merge_into_issue, source_issues):
# type: (Issue, Collection[Issue]) -> Collection[int]
"""Compute ccs that should be added from source_issues to merge_into_issue."""
merge_into_restrictions = permissions.GetRestrictions(merge_into_issue)
new_cc_ids = set()
for issue in source_issues:
# We don't want to leak metadata like ccs of restricted issues.
# So we don't merge ccs from restricted source issues, unless their
# restrictions match the restrictions of the target.
if permissions.HasRestrictions(issue, perm='View'):
source_restrictions = permissions.GetRestrictions(issue)
if (issue.project_id != merge_into_issue.project_id or
set(source_restrictions) != set(merge_into_restrictions)):
continue
new_cc_ids.update(issue.cc_ids)
if issue.owner_id:
new_cc_ids.add(issue.owner_id)
return [cc_id for cc_id in new_cc_ids if cc_id not in merge_into_issue.cc_ids]
def _EnforceNonMergeStatusDeltas(cnxn, issue_delta_pairs, services):
# type: (MonorailConnection, Sequence[Tuple[Issue, IssueDelta]], Services)
"""Update deltas in RAM to remove merged if a MERGED status is removed."""
project_ids = list(
{issue.project_id for (issue, _delta) in issue_delta_pairs})
configs_by_id = services.config.GetProjectConfigs(cnxn, project_ids)
statuses_offer_merge_by_pid = {
pid:
[status.lower() for status in configs_by_id[pid].statuses_offer_merge]
for pid in project_ids
}
for issue, delta in issue_delta_pairs:
statuses_offer_merge = statuses_offer_merge_by_pid[issue.project_id]
# Remove merged_into and merged_into_external when a status is moved
# to a non-MERGED status ONLY if the delta does not have merged_into values
# If delta does change merged_into values, the request will fail from
# AssertIssueChangesValue().
if (delta.status and delta.status.lower() not in statuses_offer_merge and
delta.merged_into is None and delta.merged_into_external is None):
if issue.merged_into:
delta.merged_into = 0
elif issue.merged_into_external:
delta.merged_into_external = ''
class _IssueChangeImpactedIssues():
"""Class to track changes of issues impacted by updates to other issues."""
def __init__(self):
# Each of the dicts below should be used to track
# {impacted_issue_id: [issues being modified that impact the keyed issue]}.
# e.g. `blocking_remove` with {iid_1: [iid_2, iid_3]} means that
# `TrackImpactedIssues` has been called with a delta of
# IssueDelta(blocked_on_remove=[iid_1]) for both issue 2 and issue 3.
self.blocking_add = collections.defaultdict(list)
self.blocking_remove = collections.defaultdict(list)
self.blocked_on_add = collections.defaultdict(list)
self.blocked_on_remove = collections.defaultdict(list)
self.merged_from_add = collections.defaultdict(list)
self.merged_from_remove = collections.defaultdict(list)
def ComputeAllImpactedIIDs(self):
# type: () -> Collection[int]
"""Computes the unique set of all impacted issue ids."""
return set(self.blocking_add.keys() + self.blocking_remove.keys() +
self.blocked_on_add.keys() + self.blocked_on_remove.keys() +
self.merged_from_add.keys() + self.merged_from_remove.keys())
def TrackImpactedIssues(self, issue, delta):
# type: (Issue, IssueDelta) -> None
"""Track impacted issues from when `delta` is applied to `issue`.
Args:
issue: Issue that the delta will be applied to, but has not yet.
delta: IssueDelta representing the changes that will be made to
the issue.
"""
for impacted_iid in delta.blocked_on_add:
self.blocking_add[impacted_iid].append(issue.issue_id)
for impacted_iid in delta.blocked_on_remove:
self.blocking_remove[impacted_iid].append(issue.issue_id)
for impacted_iid in delta.blocking_add:
self.blocked_on_add[impacted_iid].append(issue.issue_id)
for impacted_iid in delta.blocking_remove:
self.blocked_on_remove[impacted_iid].append(issue.issue_id)
if (delta.merged_into == framework_constants.NO_ISSUE_SPECIFIED and
issue.merged_into):
self.merged_from_remove[issue.merged_into].append(issue.issue_id)
elif delta.merged_into and issue.merged_into != delta.merged_into:
self.merged_from_add[delta.merged_into].append(issue.issue_id)
if issue.merged_into:
self.merged_from_remove[issue.merged_into].append(issue.issue_id)
def ApplyImpactedIssueChanges(self, cnxn, impacted_issue, services):
# type: (MonorailConnection, Issue, Services) ->
# Tuple[Collection[Amendment], Sequence[int]]
"""Apply the tracked changes in RAM for the given impacted issue.
Args:
cnxn: connection to SQL database.
impacted_issue: Issue PB that we are applying the changes to.
services: Services used to fetch info from DB or cache.
Returns:
All the amendments that represent the changes applied to the issue
and a list of the new issue starrers.
Side-effect:
The given impacted_issue will be updated in RAM.
"""
issue_id = impacted_issue.issue_id
# Process changes for blocking/blocked_on issue changes.
amendments, _impacted_iids = tracker_bizobj.ApplyIssueBlockRelationChanges(
cnxn, impacted_issue, self.blocked_on_add[issue_id],
self.blocked_on_remove[issue_id], self.blocking_add[issue_id],
self.blocking_remove[issue_id], services.issue)
# Process changes in merged issues.
merged_from_add = self.merged_from_add.get(issue_id, [])
merged_from_remove = self.merged_from_remove.get(issue_id, [])
# Merge ccs into impacted_issue from all merged issues,
# compute new starrers, and set star_count.
new_starrers = []
if merged_from_add:
issues_dict, _misses = services.issue.GetIssuesDict(cnxn, merged_from_add)
merged_from_add_issues = issues_dict.values()
new_cc_ids = _ComputeNewCcsFromIssueMerge(
impacted_issue, merged_from_add_issues)
if new_cc_ids:
impacted_issue.cc_ids.extend(new_cc_ids)
amendments.append(
tracker_bizobj.MakeCcAmendment(new_cc_ids, []))
new_starrers = list(
GetNewIssueStarrers(cnxn, services, merged_from_add, issue_id))
if new_starrers:
impacted_issue.star_count += len(new_starrers)
if merged_from_add or merged_from_remove:
merged_from_add_refs = services.issue.LookupIssueRefs(
cnxn, merged_from_add).values()
merged_from_remove_refs = services.issue.LookupIssueRefs(
cnxn, merged_from_remove).values()
amendments.append(
tracker_bizobj.MakeMergedIntoAmendment(
merged_from_add_refs, merged_from_remove_refs,
default_project_name=impacted_issue.project_name))
return amendments, new_starrers