Gitiles
Code Review Sign In
gerrit.avm99963.com / monorail-avm99963 / 1663d14d8132d1d2bacc6302122f4ce2c8af201a / . / features / notify_helpers.py
blob: 5f77307f437858f6bed1c7254f3964f523d1a600 [file] [log] [blame]
Copybara854996b2021-09-07 19:36:02 +0000[diff] [blame]1# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style
3# license that can be found in the LICENSE file or at
4# https://developers.google.com/open-source/licenses/bsd
5
6"""Helper functions for email notifications of issue changes."""
7from __future__ import print_function
8from __future__ import division
9from __future__ import absolute_import
10
11import json
12import logging
13
14import ezt
15import six
16
17from features import autolink
18from features import autolink_constants
19from features import features_constants
20from features import filterrules_helpers
21from features import savedqueries_helpers
22from features import notify_reasons
23from framework import cloud_tasks_helpers
24from framework import emailfmt
25from framework import framework_bizobj
26from framework import framework_constants
27from framework import framework_helpers
28from framework import framework_views
29from framework import jsonfeed
30from framework import monorailrequest
31from framework import permissions
32from framework import template_helpers
33from framework import urls
34from proto import tracker_pb2
35from search import query2ast
36from search import searchpipeline
37from tracker import tracker_bizobj
38
39
40# Email tasks can get too large for AppEngine to handle. In order to prevent
41# that, we set a maximum body size, and may truncate messages to that length.
42# We set this value to 35k so that the total of 35k body + 35k html_body +
43# metadata does not exceed AppEngine's limit of 100k.
44MAX_EMAIL_BODY_SIZE = 35 * 1024
45
46# This HTML template adds mark up which enables Gmail/Inbox to display a
47# convenient link that takes users to the CL directly from the inbox without
48# having to click on the email.
49# Documentation for this schema.org markup is here:
50# https://developers.google.com/gmail/markup/reference/go-to-action
51HTML_BODY_WITH_GMAIL_ACTION_TEMPLATE = """
52<html>
53<body>
54<script type="application/ld+json">
55{
56 "@context": "http://schema.org",
57 "@type": "EmailMessage",
58 "potentialAction": {
59 "@type": "ViewAction",
60 "name": "View Issue",
61 "url": "%(url)s"
62 },
63 "description": ""
64}
65</script>
66
67<div style="font-family: arial, sans-serif; white-space:pre">%(body)s</div>
68</body>
69</html>
70"""
71
72HTML_BODY_WITHOUT_GMAIL_ACTION_TEMPLATE = """
73<html>
74<body>
75<div style="font-family: arial, sans-serif; white-space:pre">%(body)s</div>
76</body>
77</html>
78"""
79
80
81NOTIFY_RESTRICTED_ISSUES_PREF_NAME = 'notify_restricted_issues'
82NOTIFY_WITH_DETAILS = 'notify with details'
83NOTIFY_WITH_DETAILS_GOOGLE = 'notify with details: Google'
84NOTIFY_WITH_LINK_ONLY = 'notify with link only'
85
86
87def _EnqueueOutboundEmail(message_dict):
88 """Create a task to send one email message, all fields are in the dict.
89
90 We use a separate task for each outbound email to isolate errors.
91
92 Args:
93 message_dict: dict with all needed info for the task.
94 """
95 # We use a JSON-encoded payload because it ensures that the task size is
96 # effectively the same as the sum of the email bodies. Using params results
97 # in the dict being urlencoded, which can (worst case) triple the size of
98 # an email body containing many characters which need to be escaped.
99 payload = json.dumps(message_dict)
100 task = {
101 'app_engine_http_request':
102 {
103 'relative_uri': urls.OUTBOUND_EMAIL_TASK + '.do',
104 # Cloud Tasks expects body to be in bytes.
105 'body': payload.encode(),
106 # Cloud tasks default body content type is octet-stream.
107 'headers': {
108 'Content-type': 'application/json'
109 }
110 }
111 }
112 cloud_tasks_helpers.create_task(
113 task, queue=features_constants.QUEUE_OUTBOUND_EMAIL)
114
115
116def AddAllEmailTasks(tasks):
117 """Add one GAE task for each email to be sent."""
118 notified = []
119 for task in tasks:
120 _EnqueueOutboundEmail(task)
121 notified.append(task['to'])
122
123 return notified
124
125
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +0200[diff] [blame]126# TODO: change to FlaskInternalTask when convert to flask
Copybara854996b2021-09-07 19:36:02 +0000[diff] [blame]127class NotifyTaskBase(jsonfeed.InternalTask):
128 """Abstract base class for notification task handler."""
129
130 _EMAIL_TEMPLATE = None # Subclasses must override this.
131 _LINK_ONLY_EMAIL_TEMPLATE = None # Subclasses may override this.
132
133 CHECK_SECURITY_TOKEN = False
134
135 def __init__(self, *args, **kwargs):
136 super(NotifyTaskBase, self).__init__(*args, **kwargs)
137
138 if not self._EMAIL_TEMPLATE:
139 raise Exception('Subclasses must override _EMAIL_TEMPLATE.'
140 ' This class must not be called directly.')
141 # We use FORMAT_RAW for emails because they are plain text, not HTML.
142 # TODO(jrobbins): consider sending HTML formatted emails someday.
143 self.email_template = template_helpers.MonorailTemplate(
144 framework_constants.TEMPLATE_PATH + self._EMAIL_TEMPLATE,
145 compress_whitespace=False, base_format=ezt.FORMAT_RAW)
146
147 if self._LINK_ONLY_EMAIL_TEMPLATE:
148 self.link_only_email_template = template_helpers.MonorailTemplate(
149 framework_constants.TEMPLATE_PATH + self._LINK_ONLY_EMAIL_TEMPLATE,
150 compress_whitespace=False, base_format=ezt.FORMAT_RAW)
151
152
153def _MergeLinkedAccountReasons(addr_to_addrperm, addr_to_reasons):
154 """Return an addr_reasons_dict where parents omit child accounts."""
155 all_ids = set(addr_perm.user.user_id
156 for addr_perm in addr_to_addrperm.values()
157 if addr_perm.user)
158 merged_ids = set()
159
160 result = {}
161 for addr, reasons in addr_to_reasons.items():
162 addr_perm = addr_to_addrperm[addr]
163 parent_id = addr_perm.user.linked_parent_id if addr_perm.user else None
164 if parent_id and parent_id in all_ids:
165 # The current user is a child account and the parent would be notified,
166 # so only notify the parent.
167 merged_ids.add(parent_id)
168 else:
169 result[addr] = reasons
170
171 for addr, reasons in result.items():
172 addr_perm = addr_to_addrperm[addr]
173 if addr_perm.user and addr_perm.user.user_id in merged_ids:
174 reasons.append(notify_reasons.REASON_LINKED_ACCOUNT)
175
176 return result
177
178
179def MakeBulletedEmailWorkItems(
180 group_reason_list, issue, body_link_only, body_for_non_members,
181 body_for_members, project, hostport, commenter_view, detail_url,
182 seq_num=None, subject_prefix=None, compact_subject_prefix=None):
183 """Make a list of dicts describing email-sending tasks to notify users.
184
185 Args:
186 group_reason_list: list of (addr_perm_list, reason) tuples.
187 issue: Issue that was updated.
188 body_link_only: string body of email with minimal information.
189 body_for_non_members: string body of email to send to non-members.
190 body_for_members: string body of email to send to members.
191 project: Project that contains the issue.
192 hostport: string hostname and port number for links to the site.
193 commenter_view: UserView for the user who made the comment.
194 detail_url: str direct link to the issue.
195 seq_num: optional int sequence number of the comment.
196 subject_prefix: optional string to customize the email subject line.
197 compact_subject_prefix: optional string to customize the email subject line.
198
199 Returns:
200 A list of dictionaries, each with all needed info to send an individual
201 email to one user. Each email contains a footer that lists all the
202 reasons why that user received the email.
203 """
204 logging.info('group_reason_list is %r', group_reason_list)
205 addr_to_addrperm = {} # {email_address: AddrPerm object}
206 addr_to_reasons = {} # {email_address: [reason, ...]}
207 for group, reason in group_reason_list:
208 for memb_addr_perm in group:
209 addr = memb_addr_perm.address
210 addr_to_addrperm[addr] = memb_addr_perm
211 addr_to_reasons.setdefault(addr, []).append(reason)
212
213 addr_to_reasons = _MergeLinkedAccountReasons(
214 addr_to_addrperm, addr_to_reasons)
215 logging.info('addr_to_reasons is %r', addr_to_reasons)
216
217 email_tasks = []
218 for addr, reasons in addr_to_reasons.items():
219 memb_addr_perm = addr_to_addrperm[addr]
220 email_tasks.append(_MakeEmailWorkItem(
221 memb_addr_perm, reasons, issue, body_link_only, body_for_non_members,
222 body_for_members, project, hostport, commenter_view, detail_url,
223 seq_num=seq_num, subject_prefix=subject_prefix,
224 compact_subject_prefix=compact_subject_prefix))
225
226 return email_tasks
227
228
229def _TruncateBody(body):
230 """Truncate body string if it exceeds size limit."""
231 if len(body) > MAX_EMAIL_BODY_SIZE:
232 logging.info('Truncate body since its size %d exceeds limit', len(body))
233 return body[:MAX_EMAIL_BODY_SIZE] + '...'
234 return body
235
236
237def _GetNotifyRestrictedIssues(user_prefs, email, user):
238 """Return the notify_restricted_issues pref or a calculated default value."""
239 # If we explicitly set a pref for this address, use it.
240 if user_prefs:
241 for pref in user_prefs.prefs:
242 if pref.name == NOTIFY_RESTRICTED_ISSUES_PREF_NAME:
243 return pref.value
244
245 # Mailing lists cannot visit the site, so if it visited, it is a person.
246 if user and user.last_visit_timestamp:
247 return NOTIFY_WITH_DETAILS
248
249 # If it is a google.com mailing list, allow details for R-V-G issues.
250 if email.endswith('@google.com'):
251 return NOTIFY_WITH_DETAILS_GOOGLE
252
253 # It might be a public mailing list, so don't risk leaking any details.
254 return NOTIFY_WITH_LINK_ONLY
255
256
257def ShouldUseLinkOnly(addr_perm, issue, always_detailed=False):
258 """Return true when there is a risk of leaking a restricted issue.
259
260 We send notifications that contain only a link to the issue with no other
261 details about the change when:
262 - The issue is R-V-G and the address may be a non-google.com mailing list, or
263 - The issue is restricted with something other than R-V-G, and the user
264 may be a mailing list, or
265 - The user has a preference set.
266 """
267 if always_detailed:
268 return False
269
270 restrictions = permissions.GetRestrictions(issue, perm=permissions.VIEW)
271 if not restrictions:
272 return False
273
274 pref = _GetNotifyRestrictedIssues(
275 addr_perm.user_prefs, addr_perm.address, addr_perm.user)
276 if pref == NOTIFY_WITH_DETAILS:
277 return False
278 if (pref == NOTIFY_WITH_DETAILS_GOOGLE and
279 restrictions == ['restrict-view-google']):
280 return False
281
282 # If NOTIFY_WITH_LINK_ONLY or any unexpected value:
283 return True
284
285
286def _MakeEmailWorkItem(
287 addr_perm, reasons, issue, body_link_only,
288 body_for_non_members, body_for_members, project, hostport, commenter_view,
289 detail_url, seq_num=None, subject_prefix=None, compact_subject_prefix=None):
290 """Make one email task dict for one user, includes a detailed reason."""
291 should_use_link_only = ShouldUseLinkOnly(
292 addr_perm, issue, always_detailed=project.issue_notify_always_detailed)
293 subject_format = (
294 (subject_prefix or 'Issue ') +
295 '%(local_id)d in %(project_name)s')
296 if addr_perm.user and addr_perm.user.email_compact_subject:
297 subject_format = (
298 (compact_subject_prefix or '') +
299 '%(project_name)s:%(local_id)d')
300
301 subject = subject_format % {
302 'local_id': issue.local_id,
303 'project_name': issue.project_name,
304 }
305 if not should_use_link_only:
306 subject += ': ' + issue.summary
307
308 footer = _MakeNotificationFooter(reasons, addr_perm.reply_perm, hostport)
309 if isinstance(footer, six.text_type):
310 footer = footer.encode('utf-8')
311 if should_use_link_only:
312 body = _TruncateBody(body_link_only) + footer
313 elif addr_perm.is_member:
314 logging.info('got member %r, sending body for members', addr_perm.address)
315 body = _TruncateBody(body_for_members) + footer
316 else:
317 logging.info(
318 'got non-member %r, sending body for non-members', addr_perm.address)
319 body = _TruncateBody(body_for_non_members) + footer
320 logging.info('sending message footer:\n%r', footer)
321
322 can_reply_to = (
323 addr_perm.reply_perm != notify_reasons.REPLY_NOT_ALLOWED and
324 project.process_inbound_email)
325 from_addr = emailfmt.FormatFromAddr(
326 project, commenter_view=commenter_view, reveal_addr=addr_perm.is_member,
327 can_reply_to=can_reply_to)
328 if can_reply_to:
329 reply_to = '%s@%s' % (project.project_name, emailfmt.MailDomain())
330 else:
331 reply_to = emailfmt.NoReplyAddress()
332 refs = emailfmt.GetReferences(
333 addr_perm.address, subject, seq_num,
334 '%s@%s' % (project.project_name, emailfmt.MailDomain()))
335 # We use markup to display a convenient link that takes users directly to the
336 # issue without clicking on the email.
337 html_body = None
338 template = HTML_BODY_WITH_GMAIL_ACTION_TEMPLATE
339 if addr_perm.user and not addr_perm.user.email_view_widget:
340 template = HTML_BODY_WITHOUT_GMAIL_ACTION_TEMPLATE
341 body_with_tags = _AddHTMLTags(body.decode('utf-8'))
342 # Escape single quotes which are occasionally used to contain HTML
343 # attributes and event handler definitions.
344 body_with_tags = body_with_tags.replace("'", '&#39;')
345 html_body = template % {
346 'url': detail_url,
347 'body': body_with_tags,
348 }
349 return dict(
350 to=addr_perm.address, subject=subject, body=body, html_body=html_body,
351 from_addr=from_addr, reply_to=reply_to, references=refs)
352
353
354def _AddHTMLTags(body):
355 """Adds HMTL tags in the specified email body.
356
357 Specifically does the following:
358 * Detects links and adds <a href>s around the links.
359 * Substitutes <br/> for all occurrences of "\n".
360
361 See crbug.com/582463 for context.
362 """
363 # Convert all URLs into clickable links.
364 body = _AutolinkBody(body)
365
366 # Convert all "\n"s into "<br/>"s.
367 body = body.replace('\r\n', '<br/>')
368 body = body.replace('\n', '<br/>')
369 return body
370
371
372def _AutolinkBody(body):
373 """Convert text that looks like URLs into <a href=...>.
374
375 This uses autolink.py, but it does not register all the autolink components
376 because some of them depend on the current user's permissions which would
377 not make sense for an email body that will be sent to several different users.
378 """
379 email_autolink = autolink.Autolink()
380 email_autolink.RegisterComponent(
381 '01-linkify-user-profiles-or-mailto',
382 lambda request, mr: None,
383 lambda _mr, match: [match.group(0)],
384 {autolink_constants.IS_IMPLIED_EMAIL_RE: autolink.LinkifyEmail})
385 email_autolink.RegisterComponent(
386 '02-linkify-full-urls',
387 lambda request, mr: None,
388 lambda mr, match: None,
389 {autolink_constants.IS_A_LINK_RE: autolink.Linkify})
390 email_autolink.RegisterComponent(
391 '03-linkify-shorthand',
392 lambda request, mr: None,
393 lambda mr, match: None,
394 {autolink_constants.IS_A_SHORT_LINK_RE: autolink.Linkify,
395 autolink_constants.IS_A_NUMERIC_SHORT_LINK_RE: autolink.Linkify,
396 autolink_constants.IS_IMPLIED_LINK_RE: autolink.Linkify,
397 })
398
399 input_run = template_helpers.TextRun(body)
400 output_runs = email_autolink.MarkupAutolinks(
401 None, [input_run], autolink.SKIP_LOOKUPS)
402 output_strings = [run.FormatForHTMLEmail() for run in output_runs]
403 return ''.join(output_strings)
404
405
406def _MakeNotificationFooter(reasons, reply_perm, hostport):
407 """Make an informative footer for a notification email.
408
409 Args:
410 reasons: a list of strings to be used as the explanation. Empty if no
411 reason is to be given.
412 reply_perm: string which is one of REPLY_NOT_ALLOWED, REPLY_MAY_COMMENT,
413 REPLY_MAY_UPDATE.
414 hostport: string with domain_name:port_number to be used in linking to
415 the user preferences page.
416
417 Returns:
418 A string to be used as the email footer.
419 """
420 if not reasons:
421 return ''
422
423 domain_port = hostport.split(':')
424 domain_port[0] = framework_helpers.GetPreferredDomain(domain_port[0])
425 hostport = ':'.join(domain_port)
426
427 prefs_url = 'https://%s%s' % (hostport, urls.USER_SETTINGS)
428 lines = ['-- ']
429 lines.append('You received this message because:')
430 lines.extend(' %d. %s' % (idx + 1, reason)
431 for idx, reason in enumerate(reasons))
432
433 lines.extend(['', 'You may adjust your notification preferences at:',
434 prefs_url])
435
436 if reply_perm == notify_reasons.REPLY_MAY_COMMENT:
437 lines.extend(['', 'Reply to this email to add a comment.'])
438 elif reply_perm == notify_reasons.REPLY_MAY_UPDATE:
439 lines.extend(['', 'Reply to this email to add a comment or make updates.'])
440
441 return '\n'.join(lines)