blob: d9b2c37a5bd02d2b180b568444ae5cae061f9233 [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Handler to process inbound email with issue comments and commands."""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import email
import logging
import os
import re
import time
import six
from six.moves import urllib
import flask
import ezt
from google.appengine.api import mail
if six.PY2:
from google.appengine.ext.webapp.mail_handlers import BounceNotification
else:
from google.appengine.api.mail import BounceNotification
import settings
from features import alert2issue
from features import commitlogcommands
from features import notify_helpers
from framework import authdata
from framework import emailfmt
from framework import exceptions
from framework import framework_constants
from framework import monorailcontext
from framework import permissions
from framework import sql
from framework import template_helpers
from proto import project_pb2
TEMPLATE_PATH_BASE = framework_constants.TEMPLATE_PATH
MSG_TEMPLATES = {
'banned': 'features/inboundemail-banned.ezt',
'body_too_long': 'features/inboundemail-body-too-long.ezt',
'project_not_found': 'features/inboundemail-project-not-found.ezt',
'not_a_reply': 'features/inboundemail-not-a-reply.ezt',
'no_account': 'features/inboundemail-no-account.ezt',
'no_artifact': 'features/inboundemail-no-artifact.ezt',
'no_perms': 'features/inboundemail-no-perms.ezt',
'replies_disabled': 'features/inboundemail-replies-disabled.ezt',
}
class InboundEmail(object):
"""Servlet to handle inbound email messages."""
def __init__(self, services=None):
self.services = services or flask.current_app.config['services']
self._templates = {}
self.request = flask.request
for name, template_path in MSG_TEMPLATES.items():
self._templates[name] = template_helpers.MonorailTemplate(
TEMPLATE_PATH_BASE + template_path,
compress_whitespace=False, base_format=ezt.FORMAT_RAW)
def HandleInboundEmail(self, project_addr=None):
if self.request.method == 'POST':
self.post(project_addr)
elif self.request.method == 'GET':
self.get(project_addr)
return ''
def get(self, project_addr=None):
logging.info('\n\n\nGET for InboundEmail and project_addr is %r',
project_addr)
self.Handler(
mail.InboundEmailMessage(self.request.get_data()),
urllib.parse.unquote(project_addr))
def post(self, project_addr=None):
logging.info('\n\n\nPOST for InboundEmail and project_addr is %r',
project_addr)
self.Handler(
mail.InboundEmailMessage(self.request.get_data()),
urllib.parse.unquote(project_addr))
def Handler(self, inbound_email_message, project_addr):
"""Process an inbound email message."""
msg = inbound_email_message.original
email_tasks = self.ProcessMail(msg, project_addr)
if email_tasks:
notify_helpers.AddAllEmailTasks(email_tasks)
def ProcessMail(self, msg, project_addr):
"""Process an inbound email message."""
# TODO(jrobbins): If the message is HUGE, don't even try to parse
# it. Silently give up.
(from_addr, to_addrs, cc_addrs, references, incident_id, subject,
body) = emailfmt.ParseEmailMessage(msg)
logging.info('Proj addr: %r', project_addr)
logging.info('From addr: %r', from_addr)
logging.info('Subject: %r', subject)
logging.info('To: %r', to_addrs)
logging.info('Cc: %r', cc_addrs)
logging.info('References: %r', references)
logging.info('Incident Id: %r', incident_id)
logging.info('Body: %r', body)
# If message body is very large, reject it and send an error email.
if emailfmt.IsBodyTooBigToParse(body):
return _MakeErrorMessageReplyTask(
project_addr, from_addr, self._templates['body_too_long'])
# Make sure that the project reply-to address is in the To: line.
if not emailfmt.IsProjectAddressOnToLine(project_addr, to_addrs):
return None
project_name, verb, trooper_queue = emailfmt.IdentifyProjectVerbAndLabel(
project_addr)
is_alert = bool(verb and verb.lower() == 'alert')
error_addr = from_addr
local_id = None
author_addr = from_addr
if is_alert:
error_addr = settings.alert_escalation_email
author_addr = settings.alert_service_account
else:
local_id = emailfmt.IdentifyIssue(project_name, subject)
if not local_id:
logging.info('Could not identify issue: %s %s', project_addr, subject)
# No error message, because message was probably not intended for us.
return None
cnxn = sql.MonorailConnection()
if self.services.cache_manager:
self.services.cache_manager.DoDistributedInvalidation(cnxn)
project = self.services.project.GetProjectByName(cnxn, project_name)
# Authenticate the author_addr and perm check.
try:
mc = monorailcontext.MonorailContext(
self.services, cnxn=cnxn, requester=author_addr, autocreate=is_alert)
mc.LookupLoggedInUserPerms(project)
except exceptions.NoSuchUserException:
return _MakeErrorMessageReplyTask(
project_addr, error_addr, self._templates['no_account'])
# TODO(zhangtiff): Add separate email templates for alert error cases.
if not project or project.state != project_pb2.ProjectState.LIVE:
return _MakeErrorMessageReplyTask(
project_addr, error_addr, self._templates['project_not_found'])
if not project.process_inbound_email:
return _MakeErrorMessageReplyTask(
project_addr, error_addr, self._templates['replies_disabled'],
project_name=project_name)
# Verify that this is a reply to a notification that we could have sent.
is_development = os.environ['SERVER_SOFTWARE'].startswith('Development')
if not (is_alert or is_development):
for ref in references:
if emailfmt.ValidateReferencesHeader(ref, project, from_addr, subject):
break # Found a message ID that we could have sent.
if emailfmt.ValidateReferencesHeader(
ref, project, from_addr.lower(), subject):
break # Also match all-lowercase from-address.
else: # for-else: if loop completes with no valid reference found.
return _MakeErrorMessageReplyTask(
project_addr, from_addr, self._templates['not_a_reply'])
# Note: If the issue summary line is changed, a new thread is created,
# and replies to the old thread will no longer work because the subject
# line hash will not match, which seems reasonable.
if mc.auth.user_pb.banned:
logging.info('Banned user %s tried to post to %s',
from_addr, project_addr)
return _MakeErrorMessageReplyTask(
project_addr, error_addr, self._templates['banned'])
# If the email is an alert, switch to the alert handling path.
if is_alert:
alert2issue.ProcessEmailNotification(
self.services, cnxn, project, project_addr, from_addr,
mc.auth, subject, body, incident_id, msg, trooper_queue)
return None
# This email is a response to an email about a comment.
self.ProcessIssueReply(
mc, project, local_id, project_addr, body)
return None
def ProcessIssueReply(
self, mc, project, local_id, project_addr, body):
"""Examine an issue reply email body and add a comment to the issue.
Args:
mc: MonorailContext with cnxn and the requester email, user_id, perms.
project: Project PB for the project containing the issue.
local_id: int ID of the issue being replied to.
project_addr: string email address used for outbound emails from
that project.
body: string email body text of the reply email.
Returns:
A list of follow-up work items, e.g., to notify other users of
the new comment, or to notify the user that their reply was not
processed.
Side-effect:
Adds a new comment to the issue, if no error is reported.
"""
try:
issue = self.services.issue.GetIssueByLocalID(
mc.cnxn, project.project_id, local_id)
except exceptions.NoSuchIssueException:
issue = None
if not issue or issue.deleted:
# The referenced issue was not found, e.g., it might have been
# deleted, or someone messed with the subject line. Reject it.
return _MakeErrorMessageReplyTask(
project_addr, mc.auth.email, self._templates['no_artifact'],
artifact_phrase='issue %d' % local_id,
project_name=project.project_name)
can_view = mc.perms.CanUsePerm(
permissions.VIEW, mc.auth.effective_ids, project,
permissions.GetRestrictions(issue))
can_comment = mc.perms.CanUsePerm(
permissions.ADD_ISSUE_COMMENT, mc.auth.effective_ids, project,
permissions.GetRestrictions(issue))
if not can_view or not can_comment:
return _MakeErrorMessageReplyTask(
project_addr, mc.auth.email, self._templates['no_perms'],
artifact_phrase='issue %d' % local_id,
project_name=project.project_name)
# TODO(jrobbins): if the user does not have EDIT_ISSUE and the inbound
# email tries to make an edit, send back an error message.
lines = body.strip().split('\n')
uia = commitlogcommands.UpdateIssueAction(local_id)
uia.Parse(mc.cnxn, project.project_name, mc.auth.user_id, lines,
self.services, strip_quoted_lines=True)
uia.Run(mc, self.services)
def _MakeErrorMessageReplyTask(
project_addr, sender_addr, template, **callers_page_data):
"""Return a new task to send an error message email.
Args:
project_addr: string email address that the inbound email was delivered to.
sender_addr: string email address of user who sent the email that we could
not process.
template: EZT template used to generate the email error message. The
first line of this generated text will be used as the subject line.
callers_page_data: template data dict for body of the message.
Returns:
A list with a single Email task that can be enqueued to
actually send the email.
Raises:
ValueError: if the template does begin with a "Subject:" line.
"""
email_data = {
'project_addr': project_addr,
'sender_addr': sender_addr
}
email_data.update(callers_page_data)
generated_lines = template.GetResponse(email_data)
subject, body = generated_lines.split('\n', 1)
if subject.startswith('Subject: '):
subject = subject[len('Subject: '):]
else:
raise ValueError('Email template does not begin with "Subject:" line.')
email_task = dict(to=sender_addr, subject=subject, body=body,
from_addr=emailfmt.NoReplyAddress())
logging.info('sending email error reply: %r', email_task)
return [email_task]
BAD_WRAP_RE = re.compile('=\r\n')
BAD_EQ_RE = re.compile('=3D')
class BouncedEmail(object):
"""Handler to notice when email to given user is bouncing."""
# For docs on AppEngine's bounce email see:
# https://cloud.google.com/appengine/docs/standard/python3/reference
# /services/bundled/google/appengine/api/mail/BounceNotification
def __init__(self, services=None):
self.services = services or flask.current_app.config['services']
def postBouncedEmail(self):
try:
# Context: https://crbug.com/monorail/11083
bounce_message = BounceNotification(flask.request.form)
self.receive(bounce_message)
except AttributeError:
# Context: https://crbug.com/monorail/2105
raw_message = flask.request.form.get('raw-message')
logging.info('raw_message %r', raw_message)
raw_message = BAD_WRAP_RE.sub('', raw_message)
raw_message = BAD_EQ_RE.sub('=', raw_message)
logging.info('fixed raw_message %r', raw_message)
mime_message = email.message_from_string(raw_message)
logging.info('get_payload gives %r', mime_message.get_payload())
new_form_dict = flask.request.form.copy()
new_form_dict['raw-message'] = mime_message
# Retry with mime_message
bounce_message = BounceNotification(new_form_dict)
self.receive(bounce_message)
return ''
def receive(self, bounce_message):
email_addr = bounce_message.original.get('to')
logging.info('Bounce was sent to: %r', email_addr)
# TODO(crbug.com/monorail/8727): The problem is likely no longer happening.
# but we are adding permanent logging so we don't have to keep adding
# expriring logpoints.
if '@intel' in email_addr: # both intel.com and intel-partner.
logging.info('bounce notification: %r', bounce_message.notification)
logging.info('bounce message original: %r', bounce_message.original)
# The original message's headers are the closest we get to the
# servers involved in the failed communication.
original_message = bounce_message.original_raw_message.original
if original_message is not None:
logging.info(
'bounce message original headers: %r', original_message.items())
services = self.services
cnxn = sql.MonorailConnection()
try:
user_id = services.user.LookupUserID(cnxn, email_addr)
user = services.user.GetUser(cnxn, user_id)
user.email_bounce_timestamp = int(time.time())
services.user.UpdateUser(cnxn, user_id, user)
except exceptions.NoSuchUserException:
logging.info('User %r not found, ignoring', email_addr)
logging.info('Received bounce post ... [%s]', flask.request)
logging.info('Bounce original: %s', bounce_message.original)
logging.info('Bounce notification: %s', bounce_message.notification)