Project import generated by Copybara.
GitOrigin-RevId: d9e9e3fb4e31372ec1fb43b178994ca78fa8fe70
diff --git a/features/notify_helpers.py b/features/notify_helpers.py
new file mode 100644
index 0000000..f22ed38
--- /dev/null
+++ b/features/notify_helpers.py
@@ -0,0 +1,440 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+"""Helper functions for email notifications of issue changes."""
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+
+import json
+import logging
+
+import ezt
+import six
+
+from features import autolink
+from features import autolink_constants
+from features import features_constants
+from features import filterrules_helpers
+from features import savedqueries_helpers
+from features import notify_reasons
+from framework import cloud_tasks_helpers
+from framework import emailfmt
+from framework import framework_bizobj
+from framework import framework_constants
+from framework import framework_helpers
+from framework import framework_views
+from framework import jsonfeed
+from framework import monorailrequest
+from framework import permissions
+from framework import template_helpers
+from framework import urls
+from proto import tracker_pb2
+from search import query2ast
+from search import searchpipeline
+from tracker import tracker_bizobj
+
+
+# Email tasks can get too large for AppEngine to handle. In order to prevent
+# that, we set a maximum body size, and may truncate messages to that length.
+# We set this value to 35k so that the total of 35k body + 35k html_body +
+# metadata does not exceed AppEngine's limit of 100k.
+MAX_EMAIL_BODY_SIZE = 35 * 1024
+
+# This HTML template adds mark up which enables Gmail/Inbox to display a
+# convenient link that takes users to the CL directly from the inbox without
+# having to click on the email.
+# Documentation for this schema.org markup is here:
+# https://developers.google.com/gmail/markup/reference/go-to-action
+HTML_BODY_WITH_GMAIL_ACTION_TEMPLATE = """
+<html>
+<body>
+<script type="application/ld+json">
+{
+ "@context": "http://schema.org",
+ "@type": "EmailMessage",
+ "potentialAction": {
+ "@type": "ViewAction",
+ "name": "View Issue",
+ "url": "%(url)s"
+ },
+ "description": ""
+}
+</script>
+
+<div style="font-family: arial, sans-serif; white-space:pre">%(body)s</div>
+</body>
+</html>
+"""
+
+HTML_BODY_WITHOUT_GMAIL_ACTION_TEMPLATE = """
+<html>
+<body>
+<div style="font-family: arial, sans-serif; white-space:pre">%(body)s</div>
+</body>
+</html>
+"""
+
+
+NOTIFY_RESTRICTED_ISSUES_PREF_NAME = 'notify_restricted_issues'
+NOTIFY_WITH_DETAILS = 'notify with details'
+NOTIFY_WITH_DETAILS_GOOGLE = 'notify with details: Google'
+NOTIFY_WITH_LINK_ONLY = 'notify with link only'
+
+
+def _EnqueueOutboundEmail(message_dict):
+ """Create a task to send one email message, all fields are in the dict.
+
+ We use a separate task for each outbound email to isolate errors.
+
+ Args:
+ message_dict: dict with all needed info for the task.
+ """
+ # We use a JSON-encoded payload because it ensures that the task size is
+ # effectively the same as the sum of the email bodies. Using params results
+ # in the dict being urlencoded, which can (worst case) triple the size of
+ # an email body containing many characters which need to be escaped.
+ payload = json.dumps(message_dict)
+ task = {
+ 'app_engine_http_request':
+ {
+ 'relative_uri': urls.OUTBOUND_EMAIL_TASK + '.do',
+ # Cloud Tasks expects body to be in bytes.
+ 'body': payload.encode(),
+ # Cloud tasks default body content type is octet-stream.
+ 'headers': {
+ 'Content-type': 'application/json'
+ }
+ }
+ }
+ cloud_tasks_helpers.create_task(
+ task, queue=features_constants.QUEUE_OUTBOUND_EMAIL)
+
+
+def AddAllEmailTasks(tasks):
+ """Add one GAE task for each email to be sent."""
+ notified = []
+ for task in tasks:
+ _EnqueueOutboundEmail(task)
+ notified.append(task['to'])
+
+ return notified
+
+
+class NotifyTaskBase(jsonfeed.InternalTask):
+ """Abstract base class for notification task handler."""
+
+ _EMAIL_TEMPLATE = None # Subclasses must override this.
+ _LINK_ONLY_EMAIL_TEMPLATE = None # Subclasses may override this.
+
+ CHECK_SECURITY_TOKEN = False
+
+ def __init__(self, *args, **kwargs):
+ super(NotifyTaskBase, self).__init__(*args, **kwargs)
+
+ if not self._EMAIL_TEMPLATE:
+ raise Exception('Subclasses must override _EMAIL_TEMPLATE.'
+ ' This class must not be called directly.')
+ # We use FORMAT_RAW for emails because they are plain text, not HTML.
+ # TODO(jrobbins): consider sending HTML formatted emails someday.
+ self.email_template = template_helpers.MonorailTemplate(
+ framework_constants.TEMPLATE_PATH + self._EMAIL_TEMPLATE,
+ compress_whitespace=False, base_format=ezt.FORMAT_RAW)
+
+ if self._LINK_ONLY_EMAIL_TEMPLATE:
+ self.link_only_email_template = template_helpers.MonorailTemplate(
+ framework_constants.TEMPLATE_PATH + self._LINK_ONLY_EMAIL_TEMPLATE,
+ compress_whitespace=False, base_format=ezt.FORMAT_RAW)
+
+
+def _MergeLinkedAccountReasons(addr_to_addrperm, addr_to_reasons):
+ """Return an addr_reasons_dict where parents omit child accounts."""
+ all_ids = set(addr_perm.user.user_id
+ for addr_perm in addr_to_addrperm.values()
+ if addr_perm.user)
+ merged_ids = set()
+
+ result = {}
+ for addr, reasons in addr_to_reasons.items():
+ addr_perm = addr_to_addrperm[addr]
+ parent_id = addr_perm.user.linked_parent_id if addr_perm.user else None
+ if parent_id and parent_id in all_ids:
+ # The current user is a child account and the parent would be notified,
+ # so only notify the parent.
+ merged_ids.add(parent_id)
+ else:
+ result[addr] = reasons
+
+ for addr, reasons in result.items():
+ addr_perm = addr_to_addrperm[addr]
+ if addr_perm.user and addr_perm.user.user_id in merged_ids:
+ reasons.append(notify_reasons.REASON_LINKED_ACCOUNT)
+
+ return result
+
+
+def MakeBulletedEmailWorkItems(
+ group_reason_list, issue, body_link_only, body_for_non_members,
+ body_for_members, project, hostport, commenter_view, detail_url,
+ seq_num=None, subject_prefix=None, compact_subject_prefix=None):
+ """Make a list of dicts describing email-sending tasks to notify users.
+
+ Args:
+ group_reason_list: list of (addr_perm_list, reason) tuples.
+ issue: Issue that was updated.
+ body_link_only: string body of email with minimal information.
+ body_for_non_members: string body of email to send to non-members.
+ body_for_members: string body of email to send to members.
+ project: Project that contains the issue.
+ hostport: string hostname and port number for links to the site.
+ commenter_view: UserView for the user who made the comment.
+ detail_url: str direct link to the issue.
+ seq_num: optional int sequence number of the comment.
+ subject_prefix: optional string to customize the email subject line.
+ compact_subject_prefix: optional string to customize the email subject line.
+
+ Returns:
+ A list of dictionaries, each with all needed info to send an individual
+ email to one user. Each email contains a footer that lists all the
+ reasons why that user received the email.
+ """
+ logging.info('group_reason_list is %r', group_reason_list)
+ addr_to_addrperm = {} # {email_address: AddrPerm object}
+ addr_to_reasons = {} # {email_address: [reason, ...]}
+ for group, reason in group_reason_list:
+ for memb_addr_perm in group:
+ addr = memb_addr_perm.address
+ addr_to_addrperm[addr] = memb_addr_perm
+ addr_to_reasons.setdefault(addr, []).append(reason)
+
+ addr_to_reasons = _MergeLinkedAccountReasons(
+ addr_to_addrperm, addr_to_reasons)
+ logging.info('addr_to_reasons is %r', addr_to_reasons)
+
+ email_tasks = []
+ for addr, reasons in addr_to_reasons.items():
+ memb_addr_perm = addr_to_addrperm[addr]
+ email_tasks.append(_MakeEmailWorkItem(
+ memb_addr_perm, reasons, issue, body_link_only, body_for_non_members,
+ body_for_members, project, hostport, commenter_view, detail_url,
+ seq_num=seq_num, subject_prefix=subject_prefix,
+ compact_subject_prefix=compact_subject_prefix))
+
+ return email_tasks
+
+
+def _TruncateBody(body):
+ """Truncate body string if it exceeds size limit."""
+ if len(body) > MAX_EMAIL_BODY_SIZE:
+ logging.info('Truncate body since its size %d exceeds limit', len(body))
+ return body[:MAX_EMAIL_BODY_SIZE] + '...'
+ return body
+
+
+def _GetNotifyRestrictedIssues(user_prefs, email, user):
+ """Return the notify_restricted_issues pref or a calculated default value."""
+ # If we explicitly set a pref for this address, use it.
+ if user_prefs:
+ for pref in user_prefs.prefs:
+ if pref.name == NOTIFY_RESTRICTED_ISSUES_PREF_NAME:
+ return pref.value
+
+ # Mailing lists cannot visit the site, so if it visited, it is a person.
+ if user and user.last_visit_timestamp:
+ return NOTIFY_WITH_DETAILS
+
+ # If it is a google.com mailing list, allow details for R-V-G issues.
+ if email.endswith('@google.com'):
+ return NOTIFY_WITH_DETAILS_GOOGLE
+
+ # It might be a public mailing list, so don't risk leaking any details.
+ return NOTIFY_WITH_LINK_ONLY
+
+
+def ShouldUseLinkOnly(addr_perm, issue, always_detailed=False):
+ """Return true when there is a risk of leaking a restricted issue.
+
+ We send notifications that contain only a link to the issue with no other
+ details about the change when:
+ - The issue is R-V-G and the address may be a non-google.com mailing list, or
+ - The issue is restricted with something other than R-V-G, and the user
+ may be a mailing list, or
+ - The user has a preference set.
+ """
+ if always_detailed:
+ return False
+
+ restrictions = permissions.GetRestrictions(issue, perm=permissions.VIEW)
+ if not restrictions:
+ return False
+
+ pref = _GetNotifyRestrictedIssues(
+ addr_perm.user_prefs, addr_perm.address, addr_perm.user)
+ if pref == NOTIFY_WITH_DETAILS:
+ return False
+ if (pref == NOTIFY_WITH_DETAILS_GOOGLE and
+ restrictions == ['restrict-view-google']):
+ return False
+
+ # If NOTIFY_WITH_LINK_ONLY or any unexpected value:
+ return True
+
+
+def _MakeEmailWorkItem(
+ addr_perm, reasons, issue, body_link_only,
+ body_for_non_members, body_for_members, project, hostport, commenter_view,
+ detail_url, seq_num=None, subject_prefix=None, compact_subject_prefix=None):
+ """Make one email task dict for one user, includes a detailed reason."""
+ should_use_link_only = ShouldUseLinkOnly(
+ addr_perm, issue, always_detailed=project.issue_notify_always_detailed)
+ subject_format = (
+ (subject_prefix or 'Issue ') +
+ '%(local_id)d in %(project_name)s')
+ if addr_perm.user and addr_perm.user.email_compact_subject:
+ subject_format = (
+ (compact_subject_prefix or '') +
+ '%(project_name)s:%(local_id)d')
+
+ subject = subject_format % {
+ 'local_id': issue.local_id,
+ 'project_name': issue.project_name,
+ }
+ if not should_use_link_only:
+ subject += ': ' + issue.summary
+
+ footer = _MakeNotificationFooter(reasons, addr_perm.reply_perm, hostport)
+ if isinstance(footer, six.text_type):
+ footer = footer.encode('utf-8')
+ if should_use_link_only:
+ body = _TruncateBody(body_link_only) + footer
+ elif addr_perm.is_member:
+ logging.info('got member %r, sending body for members', addr_perm.address)
+ body = _TruncateBody(body_for_members) + footer
+ else:
+ logging.info(
+ 'got non-member %r, sending body for non-members', addr_perm.address)
+ body = _TruncateBody(body_for_non_members) + footer
+ logging.info('sending message footer:\n%r', footer)
+
+ can_reply_to = (
+ addr_perm.reply_perm != notify_reasons.REPLY_NOT_ALLOWED and
+ project.process_inbound_email)
+ from_addr = emailfmt.FormatFromAddr(
+ project, commenter_view=commenter_view, reveal_addr=addr_perm.is_member,
+ can_reply_to=can_reply_to)
+ if can_reply_to:
+ reply_to = '%s@%s' % (project.project_name, emailfmt.MailDomain())
+ else:
+ reply_to = emailfmt.NoReplyAddress()
+ refs = emailfmt.GetReferences(
+ addr_perm.address, subject, seq_num,
+ '%s@%s' % (project.project_name, emailfmt.MailDomain()))
+ # We use markup to display a convenient link that takes users directly to the
+ # issue without clicking on the email.
+ html_body = None
+ template = HTML_BODY_WITH_GMAIL_ACTION_TEMPLATE
+ if addr_perm.user and not addr_perm.user.email_view_widget:
+ template = HTML_BODY_WITHOUT_GMAIL_ACTION_TEMPLATE
+ body_with_tags = _AddHTMLTags(body.decode('utf-8'))
+ # Escape single quotes which are occasionally used to contain HTML
+ # attributes and event handler definitions.
+ body_with_tags = body_with_tags.replace("'", ''')
+ html_body = template % {
+ 'url': detail_url,
+ 'body': body_with_tags,
+ }
+ return dict(
+ to=addr_perm.address, subject=subject, body=body, html_body=html_body,
+ from_addr=from_addr, reply_to=reply_to, references=refs)
+
+
+def _AddHTMLTags(body):
+ """Adds HMTL tags in the specified email body.
+
+ Specifically does the following:
+ * Detects links and adds <a href>s around the links.
+ * Substitutes <br/> for all occurrences of "\n".
+
+ See crbug.com/582463 for context.
+ """
+ # Convert all URLs into clickable links.
+ body = _AutolinkBody(body)
+
+ # Convert all "\n"s into "<br/>"s.
+ body = body.replace('\r\n', '<br/>')
+ body = body.replace('\n', '<br/>')
+ return body
+
+
+def _AutolinkBody(body):
+ """Convert text that looks like URLs into <a href=...>.
+
+ This uses autolink.py, but it does not register all the autolink components
+ because some of them depend on the current user's permissions which would
+ not make sense for an email body that will be sent to several different users.
+ """
+ email_autolink = autolink.Autolink()
+ email_autolink.RegisterComponent(
+ '01-linkify-user-profiles-or-mailto',
+ lambda request, mr: None,
+ lambda _mr, match: [match.group(0)],
+ {autolink_constants.IS_IMPLIED_EMAIL_RE: autolink.LinkifyEmail})
+ email_autolink.RegisterComponent(
+ '02-linkify-full-urls',
+ lambda request, mr: None,
+ lambda mr, match: None,
+ {autolink_constants.IS_A_LINK_RE: autolink.Linkify})
+ email_autolink.RegisterComponent(
+ '03-linkify-shorthand',
+ lambda request, mr: None,
+ lambda mr, match: None,
+ {autolink_constants.IS_A_SHORT_LINK_RE: autolink.Linkify,
+ autolink_constants.IS_A_NUMERIC_SHORT_LINK_RE: autolink.Linkify,
+ autolink_constants.IS_IMPLIED_LINK_RE: autolink.Linkify,
+ })
+
+ input_run = template_helpers.TextRun(body)
+ output_runs = email_autolink.MarkupAutolinks(
+ None, [input_run], autolink.SKIP_LOOKUPS)
+ output_strings = [run.FormatForHTMLEmail() for run in output_runs]
+ return ''.join(output_strings)
+
+
+def _MakeNotificationFooter(reasons, reply_perm, hostport):
+ """Make an informative footer for a notification email.
+
+ Args:
+ reasons: a list of strings to be used as the explanation. Empty if no
+ reason is to be given.
+ reply_perm: string which is one of REPLY_NOT_ALLOWED, REPLY_MAY_COMMENT,
+ REPLY_MAY_UPDATE.
+ hostport: string with domain_name:port_number to be used in linking to
+ the user preferences page.
+
+ Returns:
+ A string to be used as the email footer.
+ """
+ if not reasons:
+ return ''
+
+ domain_port = hostport.split(':')
+ domain_port[0] = framework_helpers.GetPreferredDomain(domain_port[0])
+ hostport = ':'.join(domain_port)
+
+ prefs_url = 'https://%s%s' % (hostport, urls.USER_SETTINGS)
+ lines = ['-- ']
+ lines.append('You received this message because:')
+ lines.extend(' %d. %s' % (idx + 1, reason)
+ for idx, reason in enumerate(reasons))
+
+ lines.extend(['', 'You may adjust your notification preferences at:',
+ prefs_url])
+
+ if reply_perm == notify_reasons.REPLY_MAY_COMMENT:
+ lines.extend(['', 'Reply to this email to add a comment.'])
+ elif reply_perm == notify_reasons.REPLY_MAY_UPDATE:
+ lines.extend(['', 'Reply to this email to add a comment or make updates.'])
+
+ return '\n'.join(lines)