Gitiles
Code Review Sign In
gerrit.avm99963.com / monorail-avm99963 / 2356fffd0aa91c404de82815f34699f781b0ffef / . / features / notify_helpers.py
blob: f22ed3813cdd2d58db8ccd440b172227833c929b [file] [log] [blame]
Copybara854996b2021-09-07 19:36:02 +0000[diff] [blame]1# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style
3# license that can be found in the LICENSE file or at
4# https://developers.google.com/open-source/licenses/bsd
5
6"""Helper functions for email notifications of issue changes."""
7from __future__ import print_function
8from __future__ import division
9from __future__ import absolute_import
10
11import json
12import logging
13
14import ezt
15import six
16
17from features import autolink
18from features import autolink_constants
19from features import features_constants
20from features import filterrules_helpers
21from features import savedqueries_helpers
22from features import notify_reasons
23from framework import cloud_tasks_helpers
24from framework import emailfmt
25from framework import framework_bizobj
26from framework import framework_constants
27from framework import framework_helpers
28from framework import framework_views
29from framework import jsonfeed
30from framework import monorailrequest
31from framework import permissions
32from framework import template_helpers
33from framework import urls
34from proto import tracker_pb2
35from search import query2ast
36from search import searchpipeline
37from tracker import tracker_bizobj
38
39
40# Email tasks can get too large for AppEngine to handle. In order to prevent
41# that, we set a maximum body size, and may truncate messages to that length.
42# We set this value to 35k so that the total of 35k body + 35k html_body +
43# metadata does not exceed AppEngine's limit of 100k.
44MAX_EMAIL_BODY_SIZE = 35 * 1024
45
46# This HTML template adds mark up which enables Gmail/Inbox to display a
47# convenient link that takes users to the CL directly from the inbox without
48# having to click on the email.
49# Documentation for this schema.org markup is here:
50# https://developers.google.com/gmail/markup/reference/go-to-action
51HTML_BODY_WITH_GMAIL_ACTION_TEMPLATE = """
52<html>
53<body>
54<script type="application/ld+json">
55{
56 "@context": "http://schema.org",
57 "@type": "EmailMessage",
58 "potentialAction": {
59 "@type": "ViewAction",
60 "name": "View Issue",
61 "url": "%(url)s"
62 },
63 "description": ""
64}
65</script>
66
67<div style="font-family: arial, sans-serif; white-space:pre">%(body)s</div>
68</body>
69</html>
70"""
71
72HTML_BODY_WITHOUT_GMAIL_ACTION_TEMPLATE = """
73<html>
74<body>
75<div style="font-family: arial, sans-serif; white-space:pre">%(body)s</div>
76</body>
77</html>
78"""
79
80
81NOTIFY_RESTRICTED_ISSUES_PREF_NAME = 'notify_restricted_issues'
82NOTIFY_WITH_DETAILS = 'notify with details'
83NOTIFY_WITH_DETAILS_GOOGLE = 'notify with details: Google'
84NOTIFY_WITH_LINK_ONLY = 'notify with link only'
85
86
87def _EnqueueOutboundEmail(message_dict):
88 """Create a task to send one email message, all fields are in the dict.
89
90 We use a separate task for each outbound email to isolate errors.
91
92 Args:
93 message_dict: dict with all needed info for the task.
94 """
95 # We use a JSON-encoded payload because it ensures that the task size is
96 # effectively the same as the sum of the email bodies. Using params results
97 # in the dict being urlencoded, which can (worst case) triple the size of
98 # an email body containing many characters which need to be escaped.
99 payload = json.dumps(message_dict)
100 task = {
101 'app_engine_http_request':
102 {
103 'relative_uri': urls.OUTBOUND_EMAIL_TASK + '.do',
104 # Cloud Tasks expects body to be in bytes.
105 'body': payload.encode(),
106 # Cloud tasks default body content type is octet-stream.
107 'headers': {
108 'Content-type': 'application/json'
109 }
110 }
111 }
112 cloud_tasks_helpers.create_task(
113 task, queue=features_constants.QUEUE_OUTBOUND_EMAIL)
114
115
116def AddAllEmailTasks(tasks):
117 """Add one GAE task for each email to be sent."""
118 notified = []
119 for task in tasks:
120 _EnqueueOutboundEmail(task)
121 notified.append(task['to'])
122
123 return notified
124
125
126class NotifyTaskBase(jsonfeed.InternalTask):
127 """Abstract base class for notification task handler."""
128
129 _EMAIL_TEMPLATE = None # Subclasses must override this.
130 _LINK_ONLY_EMAIL_TEMPLATE = None # Subclasses may override this.
131
132 CHECK_SECURITY_TOKEN = False
133
134 def __init__(self, *args, **kwargs):
135 super(NotifyTaskBase, self).__init__(*args, **kwargs)
136
137 if not self._EMAIL_TEMPLATE:
138 raise Exception('Subclasses must override _EMAIL_TEMPLATE.'
139 ' This class must not be called directly.')
140 # We use FORMAT_RAW for emails because they are plain text, not HTML.
141 # TODO(jrobbins): consider sending HTML formatted emails someday.
142 self.email_template = template_helpers.MonorailTemplate(
143 framework_constants.TEMPLATE_PATH + self._EMAIL_TEMPLATE,
144 compress_whitespace=False, base_format=ezt.FORMAT_RAW)
145
146 if self._LINK_ONLY_EMAIL_TEMPLATE:
147 self.link_only_email_template = template_helpers.MonorailTemplate(
148 framework_constants.TEMPLATE_PATH + self._LINK_ONLY_EMAIL_TEMPLATE,
149 compress_whitespace=False, base_format=ezt.FORMAT_RAW)
150
151
152def _MergeLinkedAccountReasons(addr_to_addrperm, addr_to_reasons):
153 """Return an addr_reasons_dict where parents omit child accounts."""
154 all_ids = set(addr_perm.user.user_id
155 for addr_perm in addr_to_addrperm.values()
156 if addr_perm.user)
157 merged_ids = set()
158
159 result = {}
160 for addr, reasons in addr_to_reasons.items():
161 addr_perm = addr_to_addrperm[addr]
162 parent_id = addr_perm.user.linked_parent_id if addr_perm.user else None
163 if parent_id and parent_id in all_ids:
164 # The current user is a child account and the parent would be notified,
165 # so only notify the parent.
166 merged_ids.add(parent_id)
167 else:
168 result[addr] = reasons
169
170 for addr, reasons in result.items():
171 addr_perm = addr_to_addrperm[addr]
172 if addr_perm.user and addr_perm.user.user_id in merged_ids:
173 reasons.append(notify_reasons.REASON_LINKED_ACCOUNT)
174
175 return result
176
177
178def MakeBulletedEmailWorkItems(
179 group_reason_list, issue, body_link_only, body_for_non_members,
180 body_for_members, project, hostport, commenter_view, detail_url,
181 seq_num=None, subject_prefix=None, compact_subject_prefix=None):
182 """Make a list of dicts describing email-sending tasks to notify users.
183
184 Args:
185 group_reason_list: list of (addr_perm_list, reason) tuples.
186 issue: Issue that was updated.
187 body_link_only: string body of email with minimal information.
188 body_for_non_members: string body of email to send to non-members.
189 body_for_members: string body of email to send to members.
190 project: Project that contains the issue.
191 hostport: string hostname and port number for links to the site.
192 commenter_view: UserView for the user who made the comment.
193 detail_url: str direct link to the issue.
194 seq_num: optional int sequence number of the comment.
195 subject_prefix: optional string to customize the email subject line.
196 compact_subject_prefix: optional string to customize the email subject line.
197
198 Returns:
199 A list of dictionaries, each with all needed info to send an individual
200 email to one user. Each email contains a footer that lists all the
201 reasons why that user received the email.
202 """
203 logging.info('group_reason_list is %r', group_reason_list)
204 addr_to_addrperm = {} # {email_address: AddrPerm object}
205 addr_to_reasons = {} # {email_address: [reason, ...]}
206 for group, reason in group_reason_list:
207 for memb_addr_perm in group:
208 addr = memb_addr_perm.address
209 addr_to_addrperm[addr] = memb_addr_perm
210 addr_to_reasons.setdefault(addr, []).append(reason)
211
212 addr_to_reasons = _MergeLinkedAccountReasons(
213 addr_to_addrperm, addr_to_reasons)
214 logging.info('addr_to_reasons is %r', addr_to_reasons)
215
216 email_tasks = []
217 for addr, reasons in addr_to_reasons.items():
218 memb_addr_perm = addr_to_addrperm[addr]
219 email_tasks.append(_MakeEmailWorkItem(
220 memb_addr_perm, reasons, issue, body_link_only, body_for_non_members,
221 body_for_members, project, hostport, commenter_view, detail_url,
222 seq_num=seq_num, subject_prefix=subject_prefix,
223 compact_subject_prefix=compact_subject_prefix))
224
225 return email_tasks
226
227
228def _TruncateBody(body):
229 """Truncate body string if it exceeds size limit."""
230 if len(body) > MAX_EMAIL_BODY_SIZE:
231 logging.info('Truncate body since its size %d exceeds limit', len(body))
232 return body[:MAX_EMAIL_BODY_SIZE] + '...'
233 return body
234
235
236def _GetNotifyRestrictedIssues(user_prefs, email, user):
237 """Return the notify_restricted_issues pref or a calculated default value."""
238 # If we explicitly set a pref for this address, use it.
239 if user_prefs:
240 for pref in user_prefs.prefs:
241 if pref.name == NOTIFY_RESTRICTED_ISSUES_PREF_NAME:
242 return pref.value
243
244 # Mailing lists cannot visit the site, so if it visited, it is a person.
245 if user and user.last_visit_timestamp:
246 return NOTIFY_WITH_DETAILS
247
248 # If it is a google.com mailing list, allow details for R-V-G issues.
249 if email.endswith('@google.com'):
250 return NOTIFY_WITH_DETAILS_GOOGLE
251
252 # It might be a public mailing list, so don't risk leaking any details.
253 return NOTIFY_WITH_LINK_ONLY
254
255
256def ShouldUseLinkOnly(addr_perm, issue, always_detailed=False):
257 """Return true when there is a risk of leaking a restricted issue.
258
259 We send notifications that contain only a link to the issue with no other
260 details about the change when:
261 - The issue is R-V-G and the address may be a non-google.com mailing list, or
262 - The issue is restricted with something other than R-V-G, and the user
263 may be a mailing list, or
264 - The user has a preference set.
265 """
266 if always_detailed:
267 return False
268
269 restrictions = permissions.GetRestrictions(issue, perm=permissions.VIEW)
270 if not restrictions:
271 return False
272
273 pref = _GetNotifyRestrictedIssues(
274 addr_perm.user_prefs, addr_perm.address, addr_perm.user)
275 if pref == NOTIFY_WITH_DETAILS:
276 return False
277 if (pref == NOTIFY_WITH_DETAILS_GOOGLE and
278 restrictions == ['restrict-view-google']):
279 return False
280
281 # If NOTIFY_WITH_LINK_ONLY or any unexpected value:
282 return True
283
284
285def _MakeEmailWorkItem(
286 addr_perm, reasons, issue, body_link_only,
287 body_for_non_members, body_for_members, project, hostport, commenter_view,
288 detail_url, seq_num=None, subject_prefix=None, compact_subject_prefix=None):
289 """Make one email task dict for one user, includes a detailed reason."""
290 should_use_link_only = ShouldUseLinkOnly(
291 addr_perm, issue, always_detailed=project.issue_notify_always_detailed)
292 subject_format = (
293 (subject_prefix or 'Issue ') +
294 '%(local_id)d in %(project_name)s')
295 if addr_perm.user and addr_perm.user.email_compact_subject:
296 subject_format = (
297 (compact_subject_prefix or '') +
298 '%(project_name)s:%(local_id)d')
299
300 subject = subject_format % {
301 'local_id': issue.local_id,
302 'project_name': issue.project_name,
303 }
304 if not should_use_link_only:
305 subject += ': ' + issue.summary
306
307 footer = _MakeNotificationFooter(reasons, addr_perm.reply_perm, hostport)
308 if isinstance(footer, six.text_type):
309 footer = footer.encode('utf-8')
310 if should_use_link_only:
311 body = _TruncateBody(body_link_only) + footer
312 elif addr_perm.is_member:
313 logging.info('got member %r, sending body for members', addr_perm.address)
314 body = _TruncateBody(body_for_members) + footer
315 else:
316 logging.info(
317 'got non-member %r, sending body for non-members', addr_perm.address)
318 body = _TruncateBody(body_for_non_members) + footer
319 logging.info('sending message footer:\n%r', footer)
320
321 can_reply_to = (
322 addr_perm.reply_perm != notify_reasons.REPLY_NOT_ALLOWED and
323 project.process_inbound_email)
324 from_addr = emailfmt.FormatFromAddr(
325 project, commenter_view=commenter_view, reveal_addr=addr_perm.is_member,
326 can_reply_to=can_reply_to)
327 if can_reply_to:
328 reply_to = '%s@%s' % (project.project_name, emailfmt.MailDomain())
329 else:
330 reply_to = emailfmt.NoReplyAddress()
331 refs = emailfmt.GetReferences(
332 addr_perm.address, subject, seq_num,
333 '%s@%s' % (project.project_name, emailfmt.MailDomain()))
334 # We use markup to display a convenient link that takes users directly to the
335 # issue without clicking on the email.
336 html_body = None
337 template = HTML_BODY_WITH_GMAIL_ACTION_TEMPLATE
338 if addr_perm.user and not addr_perm.user.email_view_widget:
339 template = HTML_BODY_WITHOUT_GMAIL_ACTION_TEMPLATE
340 body_with_tags = _AddHTMLTags(body.decode('utf-8'))
341 # Escape single quotes which are occasionally used to contain HTML
342 # attributes and event handler definitions.
343 body_with_tags = body_with_tags.replace("'", '&#39;')
344 html_body = template % {
345 'url': detail_url,
346 'body': body_with_tags,
347 }
348 return dict(
349 to=addr_perm.address, subject=subject, body=body, html_body=html_body,
350 from_addr=from_addr, reply_to=reply_to, references=refs)
351
352
353def _AddHTMLTags(body):
354 """Adds HMTL tags in the specified email body.
355
356 Specifically does the following:
357 * Detects links and adds <a href>s around the links.
358 * Substitutes <br/> for all occurrences of "\n".
359
360 See crbug.com/582463 for context.
361 """
362 # Convert all URLs into clickable links.
363 body = _AutolinkBody(body)
364
365 # Convert all "\n"s into "<br/>"s.
366 body = body.replace('\r\n', '<br/>')
367 body = body.replace('\n', '<br/>')
368 return body
369
370
371def _AutolinkBody(body):
372 """Convert text that looks like URLs into <a href=...>.
373
374 This uses autolink.py, but it does not register all the autolink components
375 because some of them depend on the current user's permissions which would
376 not make sense for an email body that will be sent to several different users.
377 """
378 email_autolink = autolink.Autolink()
379 email_autolink.RegisterComponent(
380 '01-linkify-user-profiles-or-mailto',
381 lambda request, mr: None,
382 lambda _mr, match: [match.group(0)],
383 {autolink_constants.IS_IMPLIED_EMAIL_RE: autolink.LinkifyEmail})
384 email_autolink.RegisterComponent(
385 '02-linkify-full-urls',
386 lambda request, mr: None,
387 lambda mr, match: None,
388 {autolink_constants.IS_A_LINK_RE: autolink.Linkify})
389 email_autolink.RegisterComponent(
390 '03-linkify-shorthand',
391 lambda request, mr: None,
392 lambda mr, match: None,
393 {autolink_constants.IS_A_SHORT_LINK_RE: autolink.Linkify,
394 autolink_constants.IS_A_NUMERIC_SHORT_LINK_RE: autolink.Linkify,
395 autolink_constants.IS_IMPLIED_LINK_RE: autolink.Linkify,
396 })
397
398 input_run = template_helpers.TextRun(body)
399 output_runs = email_autolink.MarkupAutolinks(
400 None, [input_run], autolink.SKIP_LOOKUPS)
401 output_strings = [run.FormatForHTMLEmail() for run in output_runs]
402 return ''.join(output_strings)
403
404
405def _MakeNotificationFooter(reasons, reply_perm, hostport):
406 """Make an informative footer for a notification email.
407
408 Args:
409 reasons: a list of strings to be used as the explanation. Empty if no
410 reason is to be given.
411 reply_perm: string which is one of REPLY_NOT_ALLOWED, REPLY_MAY_COMMENT,
412 REPLY_MAY_UPDATE.
413 hostport: string with domain_name:port_number to be used in linking to
414 the user preferences page.
415
416 Returns:
417 A string to be used as the email footer.
418 """
419 if not reasons:
420 return ''
421
422 domain_port = hostport.split(':')
423 domain_port[0] = framework_helpers.GetPreferredDomain(domain_port[0])
424 hostport = ':'.join(domain_port)
425
426 prefs_url = 'https://%s%s' % (hostport, urls.USER_SETTINGS)
427 lines = ['-- ']
428 lines.append('You received this message because:')
429 lines.extend(' %d. %s' % (idx + 1, reason)
430 for idx, reason in enumerate(reasons))
431
432 lines.extend(['', 'You may adjust your notification preferences at:',
433 prefs_url])
434
435 if reply_perm == notify_reasons.REPLY_MAY_COMMENT:
436 lines.extend(['', 'Reply to this email to add a comment.'])
437 elif reply_perm == notify_reasons.REPLY_MAY_UPDATE:
438 lines.extend(['', 'Reply to this email to add a comment or make updates.'])
439
440 return '\n'.join(lines)