blob: df392a8fbf0a2ca517f100ff6215f341b6fd7eff [file] [log] [blame]
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +01001# Copyright 2016 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
Copybara854996b2021-09-07 19:36:02 +00004
5"""Handler to process inbound email with issue comments and commands."""
6from __future__ import print_function
7from __future__ import division
8from __future__ import absolute_import
9
10import email
11import logging
12import os
13import re
14import time
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020015import six
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +020016from six.moves import urllib
Copybara854996b2021-09-07 19:36:02 +000017
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020018import flask
Copybara854996b2021-09-07 19:36:02 +000019import ezt
20
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020021from google.appengine.api import mail
22if six.PY2:
23 from google.appengine.ext.webapp.mail_handlers import BounceNotification
24else:
25 from google.appengine.api.mail import BounceNotification
Copybara854996b2021-09-07 19:36:02 +000026
Copybara854996b2021-09-07 19:36:02 +000027
28import settings
Copybara854996b2021-09-07 19:36:02 +000029from features import alert2issue
30from features import commitlogcommands
31from features import notify_helpers
32from framework import authdata
33from framework import emailfmt
34from framework import exceptions
35from framework import framework_constants
36from framework import monorailcontext
37from framework import permissions
38from framework import sql
39from framework import template_helpers
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +010040from mrproto import project_pb2
Copybara854996b2021-09-07 19:36:02 +000041
42
43TEMPLATE_PATH_BASE = framework_constants.TEMPLATE_PATH
44
45MSG_TEMPLATES = {
46 'banned': 'features/inboundemail-banned.ezt',
47 'body_too_long': 'features/inboundemail-body-too-long.ezt',
48 'project_not_found': 'features/inboundemail-project-not-found.ezt',
49 'not_a_reply': 'features/inboundemail-not-a-reply.ezt',
50 'no_account': 'features/inboundemail-no-account.ezt',
51 'no_artifact': 'features/inboundemail-no-artifact.ezt',
52 'no_perms': 'features/inboundemail-no-perms.ezt',
53 'replies_disabled': 'features/inboundemail-replies-disabled.ezt',
54 }
55
56
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020057class InboundEmail(object):
Copybara854996b2021-09-07 19:36:02 +000058 """Servlet to handle inbound email messages."""
59
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020060 def __init__(self, services=None):
61 self.services = services or flask.current_app.config['services']
Copybara854996b2021-09-07 19:36:02 +000062 self._templates = {}
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020063 self.request = flask.request
Copybara854996b2021-09-07 19:36:02 +000064 for name, template_path in MSG_TEMPLATES.items():
65 self._templates[name] = template_helpers.MonorailTemplate(
66 TEMPLATE_PATH_BASE + template_path,
67 compress_whitespace=False, base_format=ezt.FORMAT_RAW)
68
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020069 def HandleInboundEmail(self, project_addr=None):
70 if self.request.method == 'POST':
71 self.post(project_addr)
72 elif self.request.method == 'GET':
73 self.get(project_addr)
74 return ''
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +020075
Copybara854996b2021-09-07 19:36:02 +000076 def get(self, project_addr=None):
77 logging.info('\n\n\nGET for InboundEmail and project_addr is %r',
78 project_addr)
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +020079 self.Handler(
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020080 mail.InboundEmailMessage(self.request.get_data()),
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +020081 urllib.parse.unquote(project_addr))
Copybara854996b2021-09-07 19:36:02 +000082
83 def post(self, project_addr=None):
84 logging.info('\n\n\nPOST for InboundEmail and project_addr is %r',
85 project_addr)
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +020086 self.Handler(
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020087 mail.InboundEmailMessage(self.request.get_data()),
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +020088 urllib.parse.unquote(project_addr))
Copybara854996b2021-09-07 19:36:02 +000089
90 def Handler(self, inbound_email_message, project_addr):
91 """Process an inbound email message."""
92 msg = inbound_email_message.original
93 email_tasks = self.ProcessMail(msg, project_addr)
94
95 if email_tasks:
96 notify_helpers.AddAllEmailTasks(email_tasks)
97
98 def ProcessMail(self, msg, project_addr):
99 """Process an inbound email message."""
100 # TODO(jrobbins): If the message is HUGE, don't even try to parse
101 # it. Silently give up.
102
103 (from_addr, to_addrs, cc_addrs, references, incident_id, subject,
104 body) = emailfmt.ParseEmailMessage(msg)
105
106 logging.info('Proj addr: %r', project_addr)
107 logging.info('From addr: %r', from_addr)
108 logging.info('Subject: %r', subject)
109 logging.info('To: %r', to_addrs)
110 logging.info('Cc: %r', cc_addrs)
111 logging.info('References: %r', references)
112 logging.info('Incident Id: %r', incident_id)
113 logging.info('Body: %r', body)
114
115 # If message body is very large, reject it and send an error email.
116 if emailfmt.IsBodyTooBigToParse(body):
117 return _MakeErrorMessageReplyTask(
118 project_addr, from_addr, self._templates['body_too_long'])
119
120 # Make sure that the project reply-to address is in the To: line.
121 if not emailfmt.IsProjectAddressOnToLine(project_addr, to_addrs):
122 return None
123
124 project_name, verb, trooper_queue = emailfmt.IdentifyProjectVerbAndLabel(
125 project_addr)
126
127 is_alert = bool(verb and verb.lower() == 'alert')
128 error_addr = from_addr
129 local_id = None
130 author_addr = from_addr
131
132 if is_alert:
133 error_addr = settings.alert_escalation_email
134 author_addr = settings.alert_service_account
135 else:
136 local_id = emailfmt.IdentifyIssue(project_name, subject)
137 if not local_id:
138 logging.info('Could not identify issue: %s %s', project_addr, subject)
139 # No error message, because message was probably not intended for us.
140 return None
141
142 cnxn = sql.MonorailConnection()
143 if self.services.cache_manager:
144 self.services.cache_manager.DoDistributedInvalidation(cnxn)
145
146 project = self.services.project.GetProjectByName(cnxn, project_name)
147 # Authenticate the author_addr and perm check.
148 try:
149 mc = monorailcontext.MonorailContext(
150 self.services, cnxn=cnxn, requester=author_addr, autocreate=is_alert)
151 mc.LookupLoggedInUserPerms(project)
152 except exceptions.NoSuchUserException:
153 return _MakeErrorMessageReplyTask(
154 project_addr, error_addr, self._templates['no_account'])
155
156 # TODO(zhangtiff): Add separate email templates for alert error cases.
157 if not project or project.state != project_pb2.ProjectState.LIVE:
158 return _MakeErrorMessageReplyTask(
159 project_addr, error_addr, self._templates['project_not_found'])
160
161 if not project.process_inbound_email:
162 return _MakeErrorMessageReplyTask(
163 project_addr, error_addr, self._templates['replies_disabled'],
164 project_name=project_name)
165
166 # Verify that this is a reply to a notification that we could have sent.
167 is_development = os.environ['SERVER_SOFTWARE'].startswith('Development')
168 if not (is_alert or is_development):
169 for ref in references:
170 if emailfmt.ValidateReferencesHeader(ref, project, from_addr, subject):
171 break # Found a message ID that we could have sent.
172 if emailfmt.ValidateReferencesHeader(
173 ref, project, from_addr.lower(), subject):
174 break # Also match all-lowercase from-address.
175 else: # for-else: if loop completes with no valid reference found.
176 return _MakeErrorMessageReplyTask(
177 project_addr, from_addr, self._templates['not_a_reply'])
178
179 # Note: If the issue summary line is changed, a new thread is created,
180 # and replies to the old thread will no longer work because the subject
181 # line hash will not match, which seems reasonable.
182
183 if mc.auth.user_pb.banned:
184 logging.info('Banned user %s tried to post to %s',
185 from_addr, project_addr)
186 return _MakeErrorMessageReplyTask(
187 project_addr, error_addr, self._templates['banned'])
188
189 # If the email is an alert, switch to the alert handling path.
190 if is_alert:
191 alert2issue.ProcessEmailNotification(
192 self.services, cnxn, project, project_addr, from_addr,
193 mc.auth, subject, body, incident_id, msg, trooper_queue)
194 return None
195
196 # This email is a response to an email about a comment.
197 self.ProcessIssueReply(
198 mc, project, local_id, project_addr, body)
199
200 return None
201
202 def ProcessIssueReply(
203 self, mc, project, local_id, project_addr, body):
204 """Examine an issue reply email body and add a comment to the issue.
205
206 Args:
207 mc: MonorailContext with cnxn and the requester email, user_id, perms.
208 project: Project PB for the project containing the issue.
209 local_id: int ID of the issue being replied to.
210 project_addr: string email address used for outbound emails from
211 that project.
212 body: string email body text of the reply email.
213
214 Returns:
215 A list of follow-up work items, e.g., to notify other users of
216 the new comment, or to notify the user that their reply was not
217 processed.
218
219 Side-effect:
220 Adds a new comment to the issue, if no error is reported.
221 """
222 try:
223 issue = self.services.issue.GetIssueByLocalID(
224 mc.cnxn, project.project_id, local_id)
225 except exceptions.NoSuchIssueException:
226 issue = None
227
228 if not issue or issue.deleted:
229 # The referenced issue was not found, e.g., it might have been
230 # deleted, or someone messed with the subject line. Reject it.
231 return _MakeErrorMessageReplyTask(
232 project_addr, mc.auth.email, self._templates['no_artifact'],
233 artifact_phrase='issue %d' % local_id,
234 project_name=project.project_name)
235
236 can_view = mc.perms.CanUsePerm(
237 permissions.VIEW, mc.auth.effective_ids, project,
238 permissions.GetRestrictions(issue))
239 can_comment = mc.perms.CanUsePerm(
240 permissions.ADD_ISSUE_COMMENT, mc.auth.effective_ids, project,
241 permissions.GetRestrictions(issue))
242 if not can_view or not can_comment:
243 return _MakeErrorMessageReplyTask(
244 project_addr, mc.auth.email, self._templates['no_perms'],
245 artifact_phrase='issue %d' % local_id,
246 project_name=project.project_name)
247
248 # TODO(jrobbins): if the user does not have EDIT_ISSUE and the inbound
249 # email tries to make an edit, send back an error message.
250
251 lines = body.strip().split('\n')
252 uia = commitlogcommands.UpdateIssueAction(local_id)
253 uia.Parse(mc.cnxn, project.project_name, mc.auth.user_id, lines,
254 self.services, strip_quoted_lines=True)
255 uia.Run(mc, self.services)
256
257
258def _MakeErrorMessageReplyTask(
259 project_addr, sender_addr, template, **callers_page_data):
260 """Return a new task to send an error message email.
261
262 Args:
263 project_addr: string email address that the inbound email was delivered to.
264 sender_addr: string email address of user who sent the email that we could
265 not process.
266 template: EZT template used to generate the email error message. The
267 first line of this generated text will be used as the subject line.
268 callers_page_data: template data dict for body of the message.
269
270 Returns:
271 A list with a single Email task that can be enqueued to
272 actually send the email.
273
274 Raises:
275 ValueError: if the template does begin with a "Subject:" line.
276 """
277 email_data = {
278 'project_addr': project_addr,
279 'sender_addr': sender_addr
280 }
281 email_data.update(callers_page_data)
282
283 generated_lines = template.GetResponse(email_data)
284 subject, body = generated_lines.split('\n', 1)
285 if subject.startswith('Subject: '):
286 subject = subject[len('Subject: '):]
287 else:
288 raise ValueError('Email template does not begin with "Subject:" line.')
289
290 email_task = dict(to=sender_addr, subject=subject, body=body,
291 from_addr=emailfmt.NoReplyAddress())
292 logging.info('sending email error reply: %r', email_task)
293
294 return [email_task]
295
296
297BAD_WRAP_RE = re.compile('=\r\n')
298BAD_EQ_RE = re.compile('=3D')
299
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +0200300
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +0200301class BouncedEmail(object):
Copybara854996b2021-09-07 19:36:02 +0000302 """Handler to notice when email to given user is bouncing."""
303
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +0200304 # For docs on AppEngine's bounce email see:
305 # https://cloud.google.com/appengine/docs/standard/python3/reference
306 # /services/bundled/google/appengine/api/mail/BounceNotification
Copybara854996b2021-09-07 19:36:02 +0000307
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +0200308 def __init__(self, services=None):
309 self.services = services or flask.current_app.config['services']
310
311 def postBouncedEmail(self):
Copybara854996b2021-09-07 19:36:02 +0000312 try:
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +0200313 # Context: https://crbug.com/monorail/11083
314 bounce_message = BounceNotification(flask.request.form)
315 self.receive(bounce_message)
Copybara854996b2021-09-07 19:36:02 +0000316 except AttributeError:
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +0200317 # Context: https://crbug.com/monorail/2105
318 raw_message = flask.request.form.get('raw-message')
Copybara854996b2021-09-07 19:36:02 +0000319 logging.info('raw_message %r', raw_message)
320 raw_message = BAD_WRAP_RE.sub('', raw_message)
321 raw_message = BAD_EQ_RE.sub('=', raw_message)
322 logging.info('fixed raw_message %r', raw_message)
323 mime_message = email.message_from_string(raw_message)
324 logging.info('get_payload gives %r', mime_message.get_payload())
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +0200325 new_form_dict = flask.request.form.copy()
326 new_form_dict['raw-message'] = mime_message
327 # Retry with mime_message
328 bounce_message = BounceNotification(new_form_dict)
329 self.receive(bounce_message)
330 return ''
Copybara854996b2021-09-07 19:36:02 +0000331
332
333 def receive(self, bounce_message):
334 email_addr = bounce_message.original.get('to')
335 logging.info('Bounce was sent to: %r', email_addr)
336
337 # TODO(crbug.com/monorail/8727): The problem is likely no longer happening.
338 # but we are adding permanent logging so we don't have to keep adding
339 # expriring logpoints.
340 if '@intel' in email_addr: # both intel.com and intel-partner.
Adrià Vilanova Martínezac4a6442022-05-15 19:05:13 +0200341 logging.info('bounce notification: %r', bounce_message.notification)
342 logging.info('bounce message original: %r', bounce_message.original)
343 # The original message's headers are the closest we get to the
344 # servers involved in the failed communication.
345 original_message = bounce_message.original_raw_message.original
346 if original_message is not None:
347 logging.info(
348 'bounce message original headers: %r', original_message.items())
Copybara854996b2021-09-07 19:36:02 +0000349
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +0200350 services = self.services
Copybara854996b2021-09-07 19:36:02 +0000351 cnxn = sql.MonorailConnection()
352
353 try:
354 user_id = services.user.LookupUserID(cnxn, email_addr)
355 user = services.user.GetUser(cnxn, user_id)
356 user.email_bounce_timestamp = int(time.time())
357 services.user.UpdateUser(cnxn, user_id, user)
358 except exceptions.NoSuchUserException:
359 logging.info('User %r not found, ignoring', email_addr)
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +0200360 logging.info('Received bounce post ... [%s]', flask.request)
Copybara854996b2021-09-07 19:36:02 +0000361 logging.info('Bounce original: %s', bounce_message.original)
362 logging.info('Bounce notification: %s', bounce_message.notification)