Adrià Vilanova Martínez | f19ea43 | 2024-01-23 20:20:52 +0100 | [diff] [blame] | 1 | # Copyright 2016 The Chromium Authors |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 4 | |
| 5 | """Handler to process inbound email with issue comments and commands.""" |
| 6 | from __future__ import print_function |
| 7 | from __future__ import division |
| 8 | from __future__ import absolute_import |
| 9 | |
| 10 | import email |
| 11 | import logging |
| 12 | import os |
| 13 | import re |
| 14 | import time |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 15 | import six |
Adrià Vilanova Martínez | de94280 | 2022-07-15 14:06:55 +0200 | [diff] [blame] | 16 | from six.moves import urllib |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 17 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 18 | import flask |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 19 | import ezt |
| 20 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 21 | from google.appengine.api import mail |
| 22 | if six.PY2: |
| 23 | from google.appengine.ext.webapp.mail_handlers import BounceNotification |
| 24 | else: |
| 25 | from google.appengine.api.mail import BounceNotification |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 26 | |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 27 | |
| 28 | import settings |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 29 | from features import alert2issue |
| 30 | from features import commitlogcommands |
| 31 | from features import notify_helpers |
| 32 | from framework import authdata |
| 33 | from framework import emailfmt |
| 34 | from framework import exceptions |
| 35 | from framework import framework_constants |
| 36 | from framework import monorailcontext |
| 37 | from framework import permissions |
| 38 | from framework import sql |
| 39 | from framework import template_helpers |
Adrià Vilanova Martínez | f19ea43 | 2024-01-23 20:20:52 +0100 | [diff] [blame] | 40 | from mrproto import project_pb2 |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 41 | |
| 42 | |
| 43 | TEMPLATE_PATH_BASE = framework_constants.TEMPLATE_PATH |
| 44 | |
| 45 | MSG_TEMPLATES = { |
| 46 | 'banned': 'features/inboundemail-banned.ezt', |
| 47 | 'body_too_long': 'features/inboundemail-body-too-long.ezt', |
| 48 | 'project_not_found': 'features/inboundemail-project-not-found.ezt', |
| 49 | 'not_a_reply': 'features/inboundemail-not-a-reply.ezt', |
| 50 | 'no_account': 'features/inboundemail-no-account.ezt', |
| 51 | 'no_artifact': 'features/inboundemail-no-artifact.ezt', |
| 52 | 'no_perms': 'features/inboundemail-no-perms.ezt', |
| 53 | 'replies_disabled': 'features/inboundemail-replies-disabled.ezt', |
| 54 | } |
| 55 | |
| 56 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 57 | class InboundEmail(object): |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 58 | """Servlet to handle inbound email messages.""" |
| 59 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 60 | def __init__(self, services=None): |
| 61 | self.services = services or flask.current_app.config['services'] |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 62 | self._templates = {} |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 63 | self.request = flask.request |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 64 | for name, template_path in MSG_TEMPLATES.items(): |
| 65 | self._templates[name] = template_helpers.MonorailTemplate( |
| 66 | TEMPLATE_PATH_BASE + template_path, |
| 67 | compress_whitespace=False, base_format=ezt.FORMAT_RAW) |
| 68 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 69 | def HandleInboundEmail(self, project_addr=None): |
| 70 | if self.request.method == 'POST': |
| 71 | self.post(project_addr) |
| 72 | elif self.request.method == 'GET': |
| 73 | self.get(project_addr) |
| 74 | return '' |
Adrià Vilanova Martínez | de94280 | 2022-07-15 14:06:55 +0200 | [diff] [blame] | 75 | |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 76 | def get(self, project_addr=None): |
| 77 | logging.info('\n\n\nGET for InboundEmail and project_addr is %r', |
| 78 | project_addr) |
Adrià Vilanova Martínez | de94280 | 2022-07-15 14:06:55 +0200 | [diff] [blame] | 79 | self.Handler( |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 80 | mail.InboundEmailMessage(self.request.get_data()), |
Adrià Vilanova Martínez | de94280 | 2022-07-15 14:06:55 +0200 | [diff] [blame] | 81 | urllib.parse.unquote(project_addr)) |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 82 | |
| 83 | def post(self, project_addr=None): |
| 84 | logging.info('\n\n\nPOST for InboundEmail and project_addr is %r', |
| 85 | project_addr) |
Adrià Vilanova Martínez | de94280 | 2022-07-15 14:06:55 +0200 | [diff] [blame] | 86 | self.Handler( |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 87 | mail.InboundEmailMessage(self.request.get_data()), |
Adrià Vilanova Martínez | de94280 | 2022-07-15 14:06:55 +0200 | [diff] [blame] | 88 | urllib.parse.unquote(project_addr)) |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 89 | |
| 90 | def Handler(self, inbound_email_message, project_addr): |
| 91 | """Process an inbound email message.""" |
| 92 | msg = inbound_email_message.original |
| 93 | email_tasks = self.ProcessMail(msg, project_addr) |
| 94 | |
| 95 | if email_tasks: |
| 96 | notify_helpers.AddAllEmailTasks(email_tasks) |
| 97 | |
| 98 | def ProcessMail(self, msg, project_addr): |
| 99 | """Process an inbound email message.""" |
| 100 | # TODO(jrobbins): If the message is HUGE, don't even try to parse |
| 101 | # it. Silently give up. |
| 102 | |
| 103 | (from_addr, to_addrs, cc_addrs, references, incident_id, subject, |
| 104 | body) = emailfmt.ParseEmailMessage(msg) |
| 105 | |
| 106 | logging.info('Proj addr: %r', project_addr) |
| 107 | logging.info('From addr: %r', from_addr) |
| 108 | logging.info('Subject: %r', subject) |
| 109 | logging.info('To: %r', to_addrs) |
| 110 | logging.info('Cc: %r', cc_addrs) |
| 111 | logging.info('References: %r', references) |
| 112 | logging.info('Incident Id: %r', incident_id) |
| 113 | logging.info('Body: %r', body) |
| 114 | |
| 115 | # If message body is very large, reject it and send an error email. |
| 116 | if emailfmt.IsBodyTooBigToParse(body): |
| 117 | return _MakeErrorMessageReplyTask( |
| 118 | project_addr, from_addr, self._templates['body_too_long']) |
| 119 | |
| 120 | # Make sure that the project reply-to address is in the To: line. |
| 121 | if not emailfmt.IsProjectAddressOnToLine(project_addr, to_addrs): |
| 122 | return None |
| 123 | |
| 124 | project_name, verb, trooper_queue = emailfmt.IdentifyProjectVerbAndLabel( |
| 125 | project_addr) |
| 126 | |
| 127 | is_alert = bool(verb and verb.lower() == 'alert') |
| 128 | error_addr = from_addr |
| 129 | local_id = None |
| 130 | author_addr = from_addr |
| 131 | |
| 132 | if is_alert: |
| 133 | error_addr = settings.alert_escalation_email |
| 134 | author_addr = settings.alert_service_account |
| 135 | else: |
| 136 | local_id = emailfmt.IdentifyIssue(project_name, subject) |
| 137 | if not local_id: |
| 138 | logging.info('Could not identify issue: %s %s', project_addr, subject) |
| 139 | # No error message, because message was probably not intended for us. |
| 140 | return None |
| 141 | |
| 142 | cnxn = sql.MonorailConnection() |
| 143 | if self.services.cache_manager: |
| 144 | self.services.cache_manager.DoDistributedInvalidation(cnxn) |
| 145 | |
| 146 | project = self.services.project.GetProjectByName(cnxn, project_name) |
| 147 | # Authenticate the author_addr and perm check. |
| 148 | try: |
| 149 | mc = monorailcontext.MonorailContext( |
| 150 | self.services, cnxn=cnxn, requester=author_addr, autocreate=is_alert) |
| 151 | mc.LookupLoggedInUserPerms(project) |
| 152 | except exceptions.NoSuchUserException: |
| 153 | return _MakeErrorMessageReplyTask( |
| 154 | project_addr, error_addr, self._templates['no_account']) |
| 155 | |
| 156 | # TODO(zhangtiff): Add separate email templates for alert error cases. |
| 157 | if not project or project.state != project_pb2.ProjectState.LIVE: |
| 158 | return _MakeErrorMessageReplyTask( |
| 159 | project_addr, error_addr, self._templates['project_not_found']) |
| 160 | |
| 161 | if not project.process_inbound_email: |
| 162 | return _MakeErrorMessageReplyTask( |
| 163 | project_addr, error_addr, self._templates['replies_disabled'], |
| 164 | project_name=project_name) |
| 165 | |
| 166 | # Verify that this is a reply to a notification that we could have sent. |
| 167 | is_development = os.environ['SERVER_SOFTWARE'].startswith('Development') |
| 168 | if not (is_alert or is_development): |
| 169 | for ref in references: |
| 170 | if emailfmt.ValidateReferencesHeader(ref, project, from_addr, subject): |
| 171 | break # Found a message ID that we could have sent. |
| 172 | if emailfmt.ValidateReferencesHeader( |
| 173 | ref, project, from_addr.lower(), subject): |
| 174 | break # Also match all-lowercase from-address. |
| 175 | else: # for-else: if loop completes with no valid reference found. |
| 176 | return _MakeErrorMessageReplyTask( |
| 177 | project_addr, from_addr, self._templates['not_a_reply']) |
| 178 | |
| 179 | # Note: If the issue summary line is changed, a new thread is created, |
| 180 | # and replies to the old thread will no longer work because the subject |
| 181 | # line hash will not match, which seems reasonable. |
| 182 | |
| 183 | if mc.auth.user_pb.banned: |
| 184 | logging.info('Banned user %s tried to post to %s', |
| 185 | from_addr, project_addr) |
| 186 | return _MakeErrorMessageReplyTask( |
| 187 | project_addr, error_addr, self._templates['banned']) |
| 188 | |
| 189 | # If the email is an alert, switch to the alert handling path. |
| 190 | if is_alert: |
| 191 | alert2issue.ProcessEmailNotification( |
| 192 | self.services, cnxn, project, project_addr, from_addr, |
| 193 | mc.auth, subject, body, incident_id, msg, trooper_queue) |
| 194 | return None |
| 195 | |
| 196 | # This email is a response to an email about a comment. |
| 197 | self.ProcessIssueReply( |
| 198 | mc, project, local_id, project_addr, body) |
| 199 | |
| 200 | return None |
| 201 | |
| 202 | def ProcessIssueReply( |
| 203 | self, mc, project, local_id, project_addr, body): |
| 204 | """Examine an issue reply email body and add a comment to the issue. |
| 205 | |
| 206 | Args: |
| 207 | mc: MonorailContext with cnxn and the requester email, user_id, perms. |
| 208 | project: Project PB for the project containing the issue. |
| 209 | local_id: int ID of the issue being replied to. |
| 210 | project_addr: string email address used for outbound emails from |
| 211 | that project. |
| 212 | body: string email body text of the reply email. |
| 213 | |
| 214 | Returns: |
| 215 | A list of follow-up work items, e.g., to notify other users of |
| 216 | the new comment, or to notify the user that their reply was not |
| 217 | processed. |
| 218 | |
| 219 | Side-effect: |
| 220 | Adds a new comment to the issue, if no error is reported. |
| 221 | """ |
| 222 | try: |
| 223 | issue = self.services.issue.GetIssueByLocalID( |
| 224 | mc.cnxn, project.project_id, local_id) |
| 225 | except exceptions.NoSuchIssueException: |
| 226 | issue = None |
| 227 | |
| 228 | if not issue or issue.deleted: |
| 229 | # The referenced issue was not found, e.g., it might have been |
| 230 | # deleted, or someone messed with the subject line. Reject it. |
| 231 | return _MakeErrorMessageReplyTask( |
| 232 | project_addr, mc.auth.email, self._templates['no_artifact'], |
| 233 | artifact_phrase='issue %d' % local_id, |
| 234 | project_name=project.project_name) |
| 235 | |
| 236 | can_view = mc.perms.CanUsePerm( |
| 237 | permissions.VIEW, mc.auth.effective_ids, project, |
| 238 | permissions.GetRestrictions(issue)) |
| 239 | can_comment = mc.perms.CanUsePerm( |
| 240 | permissions.ADD_ISSUE_COMMENT, mc.auth.effective_ids, project, |
| 241 | permissions.GetRestrictions(issue)) |
| 242 | if not can_view or not can_comment: |
| 243 | return _MakeErrorMessageReplyTask( |
| 244 | project_addr, mc.auth.email, self._templates['no_perms'], |
| 245 | artifact_phrase='issue %d' % local_id, |
| 246 | project_name=project.project_name) |
| 247 | |
| 248 | # TODO(jrobbins): if the user does not have EDIT_ISSUE and the inbound |
| 249 | # email tries to make an edit, send back an error message. |
| 250 | |
| 251 | lines = body.strip().split('\n') |
| 252 | uia = commitlogcommands.UpdateIssueAction(local_id) |
| 253 | uia.Parse(mc.cnxn, project.project_name, mc.auth.user_id, lines, |
| 254 | self.services, strip_quoted_lines=True) |
| 255 | uia.Run(mc, self.services) |
| 256 | |
| 257 | |
| 258 | def _MakeErrorMessageReplyTask( |
| 259 | project_addr, sender_addr, template, **callers_page_data): |
| 260 | """Return a new task to send an error message email. |
| 261 | |
| 262 | Args: |
| 263 | project_addr: string email address that the inbound email was delivered to. |
| 264 | sender_addr: string email address of user who sent the email that we could |
| 265 | not process. |
| 266 | template: EZT template used to generate the email error message. The |
| 267 | first line of this generated text will be used as the subject line. |
| 268 | callers_page_data: template data dict for body of the message. |
| 269 | |
| 270 | Returns: |
| 271 | A list with a single Email task that can be enqueued to |
| 272 | actually send the email. |
| 273 | |
| 274 | Raises: |
| 275 | ValueError: if the template does begin with a "Subject:" line. |
| 276 | """ |
| 277 | email_data = { |
| 278 | 'project_addr': project_addr, |
| 279 | 'sender_addr': sender_addr |
| 280 | } |
| 281 | email_data.update(callers_page_data) |
| 282 | |
| 283 | generated_lines = template.GetResponse(email_data) |
| 284 | subject, body = generated_lines.split('\n', 1) |
| 285 | if subject.startswith('Subject: '): |
| 286 | subject = subject[len('Subject: '):] |
| 287 | else: |
| 288 | raise ValueError('Email template does not begin with "Subject:" line.') |
| 289 | |
| 290 | email_task = dict(to=sender_addr, subject=subject, body=body, |
| 291 | from_addr=emailfmt.NoReplyAddress()) |
| 292 | logging.info('sending email error reply: %r', email_task) |
| 293 | |
| 294 | return [email_task] |
| 295 | |
| 296 | |
| 297 | BAD_WRAP_RE = re.compile('=\r\n') |
| 298 | BAD_EQ_RE = re.compile('=3D') |
| 299 | |
Adrià Vilanova Martínez | de94280 | 2022-07-15 14:06:55 +0200 | [diff] [blame] | 300 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 301 | class BouncedEmail(object): |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 302 | """Handler to notice when email to given user is bouncing.""" |
| 303 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 304 | # For docs on AppEngine's bounce email see: |
| 305 | # https://cloud.google.com/appengine/docs/standard/python3/reference |
| 306 | # /services/bundled/google/appengine/api/mail/BounceNotification |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 307 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 308 | def __init__(self, services=None): |
| 309 | self.services = services or flask.current_app.config['services'] |
| 310 | |
| 311 | def postBouncedEmail(self): |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 312 | try: |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 313 | # Context: https://crbug.com/monorail/11083 |
| 314 | bounce_message = BounceNotification(flask.request.form) |
| 315 | self.receive(bounce_message) |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 316 | except AttributeError: |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 317 | # Context: https://crbug.com/monorail/2105 |
| 318 | raw_message = flask.request.form.get('raw-message') |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 319 | logging.info('raw_message %r', raw_message) |
| 320 | raw_message = BAD_WRAP_RE.sub('', raw_message) |
| 321 | raw_message = BAD_EQ_RE.sub('=', raw_message) |
| 322 | logging.info('fixed raw_message %r', raw_message) |
| 323 | mime_message = email.message_from_string(raw_message) |
| 324 | logging.info('get_payload gives %r', mime_message.get_payload()) |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 325 | new_form_dict = flask.request.form.copy() |
| 326 | new_form_dict['raw-message'] = mime_message |
| 327 | # Retry with mime_message |
| 328 | bounce_message = BounceNotification(new_form_dict) |
| 329 | self.receive(bounce_message) |
| 330 | return '' |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 331 | |
| 332 | |
| 333 | def receive(self, bounce_message): |
| 334 | email_addr = bounce_message.original.get('to') |
| 335 | logging.info('Bounce was sent to: %r', email_addr) |
| 336 | |
| 337 | # TODO(crbug.com/monorail/8727): The problem is likely no longer happening. |
| 338 | # but we are adding permanent logging so we don't have to keep adding |
| 339 | # expriring logpoints. |
| 340 | if '@intel' in email_addr: # both intel.com and intel-partner. |
Adrià Vilanova Martínez | ac4a644 | 2022-05-15 19:05:13 +0200 | [diff] [blame] | 341 | logging.info('bounce notification: %r', bounce_message.notification) |
| 342 | logging.info('bounce message original: %r', bounce_message.original) |
| 343 | # The original message's headers are the closest we get to the |
| 344 | # servers involved in the failed communication. |
| 345 | original_message = bounce_message.original_raw_message.original |
| 346 | if original_message is not None: |
| 347 | logging.info( |
| 348 | 'bounce message original headers: %r', original_message.items()) |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 349 | |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 350 | services = self.services |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 351 | cnxn = sql.MonorailConnection() |
| 352 | |
| 353 | try: |
| 354 | user_id = services.user.LookupUserID(cnxn, email_addr) |
| 355 | user = services.user.GetUser(cnxn, user_id) |
| 356 | user.email_bounce_timestamp = int(time.time()) |
| 357 | services.user.UpdateUser(cnxn, user_id, user) |
| 358 | except exceptions.NoSuchUserException: |
| 359 | logging.info('User %r not found, ignoring', email_addr) |
Adrià Vilanova Martínez | 9f9ade5 | 2022-10-10 23:20:11 +0200 | [diff] [blame] | 360 | logging.info('Received bounce post ... [%s]', flask.request) |
Copybara | 854996b | 2021-09-07 19:36:02 +0000 | [diff] [blame] | 361 | logging.info('Bounce original: %s', bounce_message.original) |
| 362 | logging.info('Bounce notification: %s', bounce_message.notification) |