tracker/issueentry.py

# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd

"""Servlet that implements the entry of new issues."""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import collections
import difflib
import logging
import string
import time

from businesslogic import work_env
from features import hotlist_helpers
from features import send_notifications
from framework import exceptions
from framework import framework_bizobj
from framework import framework_constants
from framework import framework_helpers
from framework import framework_views
from framework import permissions
from framework import servlet
from framework import template_helpers
from framework import urls
import ezt
from tracker import field_helpers
from tracker import template_helpers as issue_tmpl_helpers
from tracker import tracker_bizobj
from tracker import tracker_constants
from tracker import tracker_helpers
from tracker import tracker_views
from proto import tracker_pb2

PLACEHOLDER_SUMMARY = 'Enter one-line summary'
PHASES_WITH_MILESTONES = ['Beta', 'Stable', 'Stable-Exp', 'Stable-Full']
CORP_RESTRICTION_LABEL = 'Restrict-View-Google'
RESTRICTED_FLT_FIELDS = ['notice', 'whitepaper', 'm-approved']


class IssueEntry(servlet.Servlet):
  """IssueEntry shows a page with a simple form to enter a new issue."""

  _PAGE_TEMPLATE = 'tracker/issue-entry-page.ezt'
  _MAIN_TAB_MODE = servlet.Servlet.MAIN_TAB_ISSUES

  # The issue filing wizard is a separate app that posted back to Monorail's
  # issue entry page. To make this possible for the wizard, we need to allow
  # XHR-scoped XSRF tokens.
  ALLOW_XHR = True

  def AssertBasePermission(self, mr):
    """Check whether the user has any permission to visit this page.

    Args:
      mr: commonly used info parsed from the request.
    """
    super(IssueEntry, self).AssertBasePermission(mr)
    if not self.CheckPerm(mr, permissions.CREATE_ISSUE):
      raise permissions.PermissionException(
          'User is not allowed to enter an issue')

  def GatherPageData(self, mr):
    """Build up a dictionary of data values to use when rendering the page.

    Args:
      mr: commonly used info parsed from the request.

    Returns:
      Dict of values used by EZT for rendering the page.
    """
    with mr.profiler.Phase('getting config'):
      config = self.services.config.GetProjectConfig(mr.cnxn, mr.project_id)

    # In addition to checking perms, we adjust some default field values for
    # project members.
    is_member = framework_bizobj.UserIsInProject(
        mr.project, mr.auth.effective_ids)
    page_perms = self.MakePagePerms(
        mr, None,
        permissions.CREATE_ISSUE,
        permissions.SET_STAR,
        permissions.EDIT_ISSUE,
        permissions.EDIT_ISSUE_SUMMARY,
        permissions.EDIT_ISSUE_STATUS,
        permissions.EDIT_ISSUE_OWNER,
        permissions.EDIT_ISSUE_CC)


    with work_env.WorkEnv(mr, self.services) as we:
      userprefs = we.GetUserPrefs(mr.auth.user_id)
      code_font = any(pref for pref in userprefs.prefs
                      if pref.name == 'code_font' and pref.value == 'true')

    template = self._GetTemplate(mr.cnxn, config, mr.template_name, is_member)

    if template.summary:
      initial_summary = template.summary
      initial_summary_must_be_edited = template.summary_must_be_edited
    else:
      initial_summary = PLACEHOLDER_SUMMARY
      initial_summary_must_be_edited = True

    if template.status:
      initial_status = template.status
    elif is_member:
      initial_status = 'Accepted'
    else:
      initial_status = 'New'  # not offering meta, only used in hidden field.

    component_paths = []
    for component_id in template.component_ids:
      component_paths.append(
          tracker_bizobj.FindComponentDefByID(component_id, config).path)
    initial_components = ', '.join(component_paths)

    if template.owner_id:
      initial_owner = framework_views.MakeUserView(
          mr.cnxn, self.services.user, template.owner_id)
    elif template.owner_defaults_to_member and page_perms.EditIssue:
      initial_owner = mr.auth.user_view
    else:
      initial_owner = None

    if initial_owner:
      initial_owner_name = initial_owner.email
      owner_avail_state = initial_owner.avail_state
      owner_avail_message_short = initial_owner.avail_message_short
    else:
      initial_owner_name = ''
      owner_avail_state = None
      owner_avail_message_short = None

    # Check whether to allow attachments from the entry page
    allow_attachments = tracker_helpers.IsUnderSoftAttachmentQuota(mr.project)

    config_view = tracker_views.ConfigView(mr, self.services, config, template)
    # If the user followed a link that specified the template name, make sure
    # that it is also in the menu as the current choice.
    # TODO(jeffcarp): Unit test this.
    config_view.template_view.can_view = ezt.boolean(True)

    # TODO(jeffcarp): Unit test this.
    offer_templates = len(config_view.template_names) > 1
    restrict_to_known = config.restrict_to_known
    link_or_template_labels = mr.GetListParam('labels', template.labels)
    labels, _derived_labels = tracker_bizobj.ExplicitAndDerivedNonMaskedLabels(
        link_or_template_labels, [], config)

    # Users with restrict_new_issues user pref automatically add R-V-G.
    with work_env.WorkEnv(mr, self.services) as we:
      userprefs = we.GetUserPrefs(mr.auth.user_id)
      restrict_new_issues = any(
          up.name == 'restrict_new_issues' and up.value == 'true'
          for up in userprefs.prefs)
      if restrict_new_issues:
        if not any(lab.lower().startswith('restrict-view-') for lab in labels):
          labels.append(CORP_RESTRICTION_LABEL)

    field_user_views = tracker_views.MakeFieldUserViews(
        mr.cnxn, template, self.services.user)
    approval_ids = [av.approval_id for av in template.approval_values]
    field_views = tracker_views.MakeAllFieldValueViews(
        config, link_or_template_labels, [], template.field_values,
        field_user_views, parent_approval_ids=approval_ids,
        phases=template.phases)
    # TODO(jojwang): monorail:6305, remove this hack when Edit perms for field
    # values are implemented.
    field_views = [view for view in field_views
                   if view.field_name.lower() not in RESTRICTED_FLT_FIELDS]
    uneditable_fields = ezt.boolean(False)
    for fv in field_views:
      if permissions.CanEditValueForFieldDef(
          mr.auth.effective_ids, mr.perms, mr.project, fv.field_def.field_def):
        fv.is_editable = ezt.boolean(True)
      else:
        fv.is_editable = ezt.boolean(False)
        uneditable_fields = ezt.boolean(True)

    # TODO(jrobbins): remove "or []" after next release.
    (prechecked_approvals, required_approval_ids,
     phases) = issue_tmpl_helpers.GatherApprovalsPageData(
         template.approval_values or [], template.phases, config)
    approvals = [view for view in field_views if view.field_id in
                 approval_ids]

    page_data = {
        'issue_tab_mode':
            'issueEntry',
        'initial_summary':
            initial_summary,
        'template_summary':
            initial_summary,
        'clear_summary_on_click':
            ezt.boolean(
                initial_summary_must_be_edited and
                'initial_summary' not in mr.form_overrides),
        'must_edit_summary':
            ezt.boolean(initial_summary_must_be_edited),
        'initial_description':
            template.content,
        'template_name':
            template.name,
        'component_required':
            ezt.boolean(template.component_required),
        'initial_status':
            initial_status,
        'initial_owner':
            initial_owner_name,
        'owner_avail_state':
            owner_avail_state,
        'owner_avail_message_short':
            owner_avail_message_short,
        'initial_components':
            initial_components,
        'initial_cc':
            '',
        'initial_blocked_on':
            '',
        'initial_blocking':
            '',
        'initial_hotlists':
            '',
        'labels':
            labels,
        'fields':
            field_views,
        'any_errors':
            ezt.boolean(mr.errors.AnyErrors()),
        'page_perms':
            page_perms,
        'allow_attachments':
            ezt.boolean(allow_attachments),
        'max_attach_size':
            template_helpers.BytesKbOrMb(
                framework_constants.MAX_POST_BODY_SIZE),
        'offer_templates':
            ezt.boolean(offer_templates),
        'config':
            config_view,
        'restrict_to_known':
            ezt.boolean(restrict_to_known),
        'is_member':
            ezt.boolean(is_member),
        'code_font':
            ezt.boolean(code_font),
        # The following are necessary for displaying phases that come with
        # this template. These are read-only.
        'allow_edit':
            ezt.boolean(False),
        'uneditable_fields':
            uneditable_fields,
        'initial_phases':
            phases,
        'approvals':
            approvals,
        'prechecked_approvals':
            prechecked_approvals,
        'required_approval_ids':
            required_approval_ids,
        # See monorail:4692 and the use of PHASES_WITH_MILESTONES
        # in elements/flt/mr-launch-overview/mr-phase.js
        'issue_phase_names':
            list(
                {
                    phase.name.lower()
                    for phase in phases
                    if phase.name in PHASES_WITH_MILESTONES
                }),
    }

    return page_data

  def GatherHelpData(self, mr, page_data):
    """Return a dict of values to drive on-page user help.

    Args:
      mr: commonly used info parsed from the request.
      page_data: Dictionary of base and page template data.

    Returns:
      A dict of values to drive on-page user help, to be added to page_data.
    """
    help_data = super(IssueEntry, self).GatherHelpData(mr, page_data)
    dismissed = []
    if mr.auth.user_pb:
      with work_env.WorkEnv(mr, self.services) as we:
        userprefs = we.GetUserPrefs(mr.auth.user_id)
      dismissed = [
          pv.name for pv in userprefs.prefs if pv.value == 'true']
    is_privileged_domain_user = framework_bizobj.IsPriviledgedDomainUser(
        mr.auth.user_pb.email)
    if (mr.auth.user_id and
        'privacy_click_through' not in dismissed):
      help_data['cue'] = 'privacy_click_through'
    elif (mr.auth.user_id and
        'code_of_conduct' not in dismissed):
      help_data['cue'] = 'code_of_conduct'

    help_data.update({
        'is_privileged_domain_user': ezt.boolean(is_privileged_domain_user),
        })
    return help_data

  def ProcessFormData(self, mr, post_data):
    """Process the issue entry form.

    Args:
      mr: commonly used info parsed from the request.
      post_data: The post_data dict for the current request.

    Returns:
      String URL to redirect the user to after processing.
    """
    config = self.services.config.GetProjectConfig(mr.cnxn, mr.project_id)

    parsed = tracker_helpers.ParseIssueRequest(
        mr.cnxn, post_data, self.services, mr.errors, mr.project_name)

    # Updates parsed.labels and parsed.fields in place.
    field_helpers.ShiftEnumFieldsIntoLabels(
        parsed.labels, parsed.labels_remove, parsed.fields.vals,
        parsed.fields.vals_remove, config)

    labels = _DiscardUnusedTemplateLabelPrefixes(parsed.labels)

    is_member = framework_bizobj.UserIsInProject(
        mr.project, mr.auth.effective_ids)
    template = self._GetTemplate(
        mr.cnxn, config, parsed.template_name, is_member)

    (approval_values,
     phases) = issue_tmpl_helpers.FilterApprovalsAndPhases(
         template.approval_values or [], template.phases, config)

    # Issue PB with only approval_values and labels filled out, for the purpose
    # of computing applicable fields.
    partial_issue = tracker_pb2.Issue(
        approval_values=approval_values, labels=labels)
    applicable_fields = field_helpers.ListApplicableFieldDefs(
        [partial_issue], config)

    bounce_labels = parsed.labels[:]
    bounce_fields = tracker_views.MakeBounceFieldValueViews(
        parsed.fields.vals,
        parsed.fields.phase_vals,
        config,
        applicable_fields=applicable_fields)

    phase_ids_by_name = {
        phase.name.lower(): [phase.phase_id] for phase in template.phases}
    field_values = field_helpers.ParseFieldValues(
        mr.cnxn, self.services.user, parsed.fields.vals,
        parsed.fields.phase_vals, config,
        phase_ids_by_name=phase_ids_by_name)

    component_ids = tracker_helpers.LookupComponentIDs(
        parsed.components.paths, config, mr.errors)

    if not parsed.summary.strip() or parsed.summary == PLACEHOLDER_SUMMARY:
      mr.errors.summary = 'Summary is required'

    if not parsed.comment.strip():
      mr.errors.comment = 'A description is required'

    if len(parsed.comment) > tracker_constants.MAX_COMMENT_CHARS:
      mr.errors.comment = 'Comment is too long'
    if len(parsed.summary) > tracker_constants.MAX_SUMMARY_CHARS:
      mr.errors.summary = 'Summary is too long'

    if _MatchesTemplate(parsed.comment, template):
      mr.errors.comment = 'Template must be filled out.'

    if parsed.users.owner_id is None:
      mr.errors.owner = 'Invalid owner username'
    else:
      valid, msg = tracker_helpers.IsValidIssueOwner(
          mr.cnxn, mr.project, parsed.users.owner_id, self.services)
      if not valid:
        mr.errors.owner = msg

    if None in parsed.users.cc_ids:
      mr.errors.cc = 'Invalid Cc username'

    field_helpers.AssertCustomFieldsEditPerms(
        mr, config, field_values, [], [], labels, [])
    field_helpers.ApplyRestrictedDefaultValues(
        mr, config, field_values, labels, template.field_values,
        template.labels)

    # This ValidateCustomFields call is redundant with work already done
    # in CreateIssue. However, this instance passes in an ezt_errors object
    # to allow showing related errors next to the fields they happen on.
    field_helpers.ValidateCustomFields(
        mr.cnxn,
        self.services,
        field_values,
        config,
        mr.project,
        ezt_errors=mr.errors,
        issue=partial_issue)

    hotlist_pbs = ProcessParsedHotlistRefs(
        mr, self.services, parsed.hotlists.hotlist_refs)

    if not mr.errors.AnyErrors():
      with work_env.WorkEnv(mr, self.services) as we:
        try:
          if parsed.attachments:
            new_bytes_used = tracker_helpers.ComputeNewQuotaBytesUsed(
                mr.project, parsed.attachments)
            # TODO(jrobbins): Make quota be calculated and stored as
            # part of applying the comment.
            self.services.project.UpdateProject(
                mr.cnxn, mr.project.project_id,
                attachment_bytes_used=new_bytes_used)

          marked_description = tracker_helpers.MarkupDescriptionOnInput(
              parsed.comment, template.content)
          has_star = 'star' in post_data and post_data['star'] == '1'

          if approval_values:
            _AttachDefaultApprovers(config, approval_values)

          # To preserve previous behavior, do not raise filter rule errors.
          issue, _ = we.CreateIssue(
              mr.project_id,
              parsed.summary,
              parsed.status,
              parsed.users.owner_id,
              parsed.users.cc_ids,
              labels,
              field_values,
              component_ids,
              marked_description,
              blocked_on=parsed.blocked_on.iids,
              blocking=parsed.blocking.iids,
              dangling_blocked_on=[
                  tracker_pb2.DanglingIssueRef(ext_issue_identifier=ref_string)
                  for ref_string in parsed.blocked_on.federated_ref_strings
              ],
              dangling_blocking=[
                  tracker_pb2.DanglingIssueRef(ext_issue_identifier=ref_string)
                  for ref_string in parsed.blocking.federated_ref_strings
              ],
              attachments=parsed.attachments,
              approval_values=approval_values,
              phases=phases,
              raise_filter_errors=False)

          if has_star:
            we.StarIssue(issue, True)

          if hotlist_pbs:
            hotlist_ids = {hotlist.hotlist_id for hotlist in hotlist_pbs}
            issue_tuple = (issue.issue_id, mr.auth.user_id, int(time.time()),
                           '')
            self.services.features.AddIssueToHotlists(
                mr.cnxn, hotlist_ids, issue_tuple, self.services.issue,
                self.services.chart)

        except exceptions.OverAttachmentQuota:
          mr.errors.attachments = 'Project attachment quota exceeded.'
        except exceptions.InputException as e:
          if 'Undefined or deprecated component with id' in e.message:
            mr.errors.components = 'Undefined or deprecated component'

    mr.template_name = parsed.template_name
    if mr.errors.AnyErrors():
      self.PleaseCorrect(
          mr, initial_summary=parsed.summary, initial_status=parsed.status,
          initial_owner=parsed.users.owner_username,
          initial_cc=', '.join(parsed.users.cc_usernames),
          initial_components=', '.join(parsed.components.paths),
          initial_comment=parsed.comment, labels=bounce_labels,
          fields=bounce_fields, template_name=parsed.template_name,
          initial_blocked_on=parsed.blocked_on.entered_str,
          initial_blocking=parsed.blocking.entered_str,
          initial_hotlists=parsed.hotlists.entered_str,
          component_required=ezt.boolean(template.component_required))
      return

    # format a redirect url
    return framework_helpers.FormatAbsoluteURL(
        mr, urls.ISSUE_DETAIL, id=issue.local_id)

  def _GetTemplate(self, cnxn, config, template_name, is_member):
    """Tries to fetch template by name and implements default template logic
    if not found."""
    template = None
    if template_name:
      template_name = template_name.replace('+', ' ')
      template = self.services.template.GetTemplateByName(cnxn,
          template_name, config.project_id)

    if not template:
      if is_member:
        template_id = config.default_template_for_developers
      else:
        template_id = config.default_template_for_users
      template = self.services.template.GetTemplateById(cnxn, template_id)
      # If the default templates were deleted, load all and pick the first one.
      if not template:
        templates = self.services.template.GetProjectTemplates(cnxn,
            config.project_id)
        assert len(templates) > 0, 'Project has no templates!'
        template = templates[0]

    return template


def _AttachDefaultApprovers(config, approval_values):
  approval_defs_by_id = {ad.approval_id: ad for ad in config.approval_defs}
  for av in approval_values:
    ad = approval_defs_by_id.get(av.approval_id)
    if ad:
      av.approver_ids = ad.approver_ids[:]
    else:
      logging.info('ApprovalDef with approval_id %r could not be found',
          av.approval_id)


def _MatchesTemplate(content, template):
  content = content.strip(string.whitespace)
  template_content = template.content.strip(string.whitespace)
  diff = difflib.unified_diff(content.splitlines(),
      template_content.splitlines())
  return len('\n'.join(diff)) == 0


def _DiscardUnusedTemplateLabelPrefixes(labels):
  """Drop any labels that end in '-?'.

  Args:
    labels: a list of label strings.

  Returns:
    A list of the same labels, but without any that end with '-?'.
    Those label prefixes in the new issue templates are intended to
    prompt the user to enter some label with that prefix, but if
    nothing is entered there, we do not store anything.
  """
  return [lab for lab in labels
          if not lab.endswith('-?')]


def ProcessParsedHotlistRefs(mr, services, parsed_hotlist_refs):
  """Process a list of ParsedHotlistRefs, returning referenced hotlists.

  This function validates the given ParsedHotlistRefs using four checks; if all
  of them succeed, then it returns the corresponding hotlist protobuf objects.
  If any of them fail, it sets the appropriate error string in mr.errors, and
  returns an empty list.

  Args:
    mr: the MonorailRequest object
    services: the service manager
    parsed_hotlist_refs: a list of ParsedHotlistRef objects

  Returns:
    on valid input, a list of hotlist protobuf objects
    if a check fails (and the input is thus considered invalid), an empty list

  Side-effects:
    if any of the checks fails, set mr.errors.hotlists to a descriptive error
  """
  # Pre-processing; common pieces used by functions later.
  user_hotlist_pbs = services.features.GetHotlistsByUserID(
      mr.cnxn, mr.auth.user_id)
  user_hotlist_owners_ids = {hotlist.owner_ids[0]
      for hotlist in user_hotlist_pbs}
  user_hotlist_owners_to_emails = services.user.LookupUserEmails(
      mr.cnxn, user_hotlist_owners_ids)
  user_hotlist_emails_to_owners = {v: k
      for k, v in user_hotlist_owners_to_emails.items()}
  user_hotlist_refs_to_pbs = {
      hotlist_helpers.HotlistRef(hotlist.owner_ids[0], hotlist.name): hotlist
      for hotlist in user_hotlist_pbs }
  short_refs = list()
  full_refs = list()
  for parsed_ref in parsed_hotlist_refs:
    if parsed_ref.user_email is None:
      short_refs.append(parsed_ref)
    else:
      full_refs.append(parsed_ref)

  invalid_names = hotlist_helpers.InvalidParsedHotlistRefsNames(
      parsed_hotlist_refs, user_hotlist_pbs)
  if invalid_names:
    mr.errors.hotlists = (
        'You have no hotlist(s) named: %s' % ', '.join(invalid_names))
    return []

  ambiguous_names = hotlist_helpers.AmbiguousShortrefHotlistNames(
      short_refs, user_hotlist_pbs)
  if ambiguous_names:
    mr.errors.hotlists = (
        'Ambiguous hotlist(s) specified: %s' % ', '.join(ambiguous_names))
    return []

  # At this point, all refs' named hotlists are guaranteed to exist, and
  # short refs are guaranteed to be unambiguous;
  # therefore, short refs are also valid.
  short_refs_hotlist_names = {sref.hotlist_name for sref in short_refs}
  shortref_valid_pbs = [hotlist for hotlist in user_hotlist_pbs
      if hotlist.name in short_refs_hotlist_names]

  invalid_emails = hotlist_helpers.InvalidParsedHotlistRefsEmails(
      full_refs, user_hotlist_emails_to_owners)
  if invalid_emails:
    mr.errors.hotlists = (
        'You have no hotlist(s) owned by: %s' % ', '.join(invalid_emails))
    return []

  fullref_valid_pbs, invalid_refs = (
      hotlist_helpers.GetHotlistsOfParsedHotlistFullRefs(
        full_refs, user_hotlist_emails_to_owners, user_hotlist_refs_to_pbs))
  if invalid_refs:
    invalid_refs_readable = [':'.join(parsed_ref)
        for parsed_ref in invalid_refs]
    mr.errors.hotlists = (
        'Not in your hotlist(s): %s' % ', '.join(invalid_refs_readable))
    return []

  hotlist_pbs = shortref_valid_pbs + fullref_valid_pbs

  return hotlist_pbs