api/monorail_servicer.py

# Copyright 2018 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import functools
try:
  import html
except ImportError:
  import cgi as html
import logging
from six.moves import http_client
import sys
import time
from google.appengine.api import oauth

from google.appengine.api import users
from google.protobuf import json_format
from components.prpc import codes

import settings
from framework import authdata
from framework import exceptions
from framework import framework_constants
from framework import monitoring
from framework import monorailcontext
from framework import ratelimiter
from framework import permissions
from framework import sql
from framework import xsrf
from services import client_config_svc
from services import features_svc


# Header for XSRF token to protect cookie-based auth users.
XSRF_TOKEN_HEADER = 'x-xsrf-token'
# Header for test account email.  Only accepted for local dev server.
TEST_ACCOUNT_HEADER = 'x-test-account'
# Optional header to help us understand why certain calls were made.
REASON_HEADER = 'x-reason'
# Optional header to help prevent double updates.
REQUEST_ID_HEADER = 'x-request-id'

# TODO(https://crbug.com/1346473)
_PRPC_TO_HTTP_STATUS = {
  codes.StatusCode.OK: http_client.OK,
  codes.StatusCode.CANCELLED: http_client.NO_CONTENT,
  codes.StatusCode.INVALID_ARGUMENT: http_client.BAD_REQUEST,
  codes.StatusCode.DEADLINE_EXCEEDED: http_client.SERVICE_UNAVAILABLE,
  codes.StatusCode.NOT_FOUND: http_client.NOT_FOUND,
  codes.StatusCode.ALREADY_EXISTS: http_client.CONFLICT,
  codes.StatusCode.PERMISSION_DENIED: http_client.FORBIDDEN,
  codes.StatusCode.RESOURCE_EXHAUSTED: http_client.SERVICE_UNAVAILABLE,
  codes.StatusCode.FAILED_PRECONDITION: http_client.PRECONDITION_FAILED,
  codes.StatusCode.OUT_OF_RANGE: http_client.BAD_REQUEST,
  codes.StatusCode.UNIMPLEMENTED: http_client.NOT_IMPLEMENTED,
  codes.StatusCode.INTERNAL: http_client.INTERNAL_SERVER_ERROR,
  codes.StatusCode.UNAVAILABLE: http_client.SERVICE_UNAVAILABLE,
  codes.StatusCode.UNAUTHENTICATED: http_client.UNAUTHORIZED,
}

def ConvertPRPCStatusToHTTPStatus(context):
  """pRPC uses internal codes 0..16, but we want to report HTTP codes."""
  return _PRPC_TO_HTTP_STATUS.get(context._code, 500)


def PRPCMethod(func):
  @functools.wraps(func)
  def wrapper(self, request, prpc_context, cnxn=None):
    return self.Run(
        func, request, prpc_context, cnxn=cnxn)

  wrapper.wrapped = func
  return wrapper


class MonorailServicer(object):
  """Abstract base class for API servicers.
  """

  def __init__(self, services, make_rate_limiter=True, xsrf_timeout=None):
    self.services = services
    if make_rate_limiter:
      self.rate_limiter = ratelimiter.ApiRateLimiter()
    else:
      self.rate_limiter = None
    # We allow subclasses to specify a different timeout. This allows the
    # RefreshToken method to check the token with a longer expiration and
    # generate a new one.
    self.xsrf_timeout = xsrf_timeout or xsrf.TOKEN_TIMEOUT_SEC

  def Run(
      self, handler, request, prpc_context,
      cnxn=None, perms=None, start_time=None, end_time=None):
    """Run a Do* method in an API context.

    Args:
      handler: API handler method to call with MonorailContext and request.
      request: API Request proto object.
      prpc_context: pRPC context object with status code.
      cnxn: Optional connection to SQL database.
      perms: PermissionSet passed in during testing.
      start_time: Int timestamp passed in during testing.
      end_time: Int timestamp passed in during testing.

    Returns:
      The response proto returned from the handler or None if that
      method raised an exception that we handle.

    Raises:
      Only programming errors should be raised as exceptions.  All
      execptions for permission checks and input validation that are
      raised in the Do* method are converted into pRPC status codes.
    """
    start_time = start_time or time.time()
    cnxn = cnxn or sql.MonorailConnection()
    if self.services.cache_manager:
      self.services.cache_manager.DoDistributedInvalidation(cnxn)

    response = None
    client_id = None  # TODO(jrobbins): consider using client ID.
    requester_auth = None
    metadata = dict(prpc_context.invocation_metadata())
    mc = monorailcontext.MonorailContext(self.services, cnxn=cnxn, perms=perms)
    try:
      self.AssertBaseChecks(request, metadata)
      requester_auth = self.GetAndAssertRequesterAuth(
          cnxn, metadata, self.services)
      logging.info('request proto is:\n%r\n', request)
      logging.info('requester is %r', requester_auth.email)

      if self.rate_limiter:
        self.rate_limiter.CheckStart(
            client_id, requester_auth.email, start_time)
      mc.auth = requester_auth
      if not perms:
        mc.LookupLoggedInUserPerms(self.GetRequestProject(mc.cnxn, request))
      response = handler(self, mc, request)

    except Exception as e:
      if not self.ProcessException(e, prpc_context, mc):
        raise
    finally:
      if mc:
        mc.CleanUp()
      if self.rate_limiter and requester_auth and requester_auth.email:
        end_time = end_time or time.time()
        self.rate_limiter.CheckEnd(
            client_id, requester_auth.email, end_time, start_time)
      self.RecordMonitoringStats(start_time, request, response, prpc_context)

    return response

  def _GetAllowedEmailDomainAuth(self, cnxn, services):
    """Checks if the requester's email is found in api_allowed_email_domains
       and is authorized by the custom monorail scope.

    Args:
      cnxn: connection to the SQL database.
      services: connections to backend services.

    Returns:
      A new AuthData object if the method determines the requester is allowed
      to access the API, otherwise, None.
    """
    try:
      # Note: get_current_user(scopes) returns the User with the User's email.
      # So, in addition to requesting any scope listed in 'scopes', it also
      # always requests the email scope.
      monorail_scope_user = oauth.get_current_user(
          framework_constants.MONORAIL_SCOPE)
      logging.info('monorail scope user %r', monorail_scope_user)
      # TODO(b/144508063): remove this workaround.
      authorized_scopes = oauth.get_authorized_scopes(
          framework_constants.MONORAIL_SCOPE)
      if framework_constants.MONORAIL_SCOPE not in authorized_scopes:
        raise oauth.Error('Work around for b/144508063')
      logging.info(authorized_scopes)
      if (monorail_scope_user and monorail_scope_user.email().endswith(
          settings.api_allowed_email_domains)):
        logging.info('User %r authenticated with Oauth and monorail',
                     monorail_scope_user.email())
        return authdata.AuthData.FromEmail(
            cnxn, monorail_scope_user.email(), services)
    except oauth.Error as ex:
      logging.info('oauth.Error for monorail scope: %s' % ex)
    return None

  def GetAndAssertRequesterAuth(self, cnxn, metadata, services):
    """Gets the requester identity and checks if the user has permission
       to make the request.
       Any users successfully authenticated with oauth must be allowlisted or
       have accounts with the domains in api_allowed_email_domains.
       Users identified using cookie-based auth must have valid XSRF tokens.
       Test accounts ending with @example.com are only allowed in the
       local_mode.

    Args:
      cnxn: connection to the SQL database.
      metadata: metadata sent by the client.
      services: connections to backend services.

    Returns:
      A new AuthData object representing a signed in or anonymous user.

    Raises:
      exceptions.NoSuchUserException: If the requester does not exist
      permissions.BannedUserException: If the user has been banned from the site
      permissions.PermissionException: If the user is not authorized with the
        Monorail scope, is not allowlisted, and has an invalid token.
    """
    # TODO(monorail:6538): Move different authentication methods into separate
    # functions.
    requester_auth = None
    # When running on localhost, allow request to specify test account.
    if TEST_ACCOUNT_HEADER in metadata:
      if not settings.local_mode:
        raise exceptions.InputException(
            'x-test-account only accepted in local_mode')
      # For local development, we accept any request.
      # TODO(jrobbins): make this more realistic by requiring a fake XSRF token.
      test_account = metadata[TEST_ACCOUNT_HEADER]
      if not test_account.endswith('@example.com'):
        raise exceptions.InputException(
            'test_account must end with @example.com')
      logging.info('Using test_account: %r' % test_account)
      requester_auth = authdata.AuthData.FromEmail(cnxn, test_account, services)

    # Oauth for users with email domains in api_allowed_email_domains.
    if not requester_auth:
      requester_auth = self._GetAllowedEmailDomainAuth(cnxn, services)

    # Oauth for allowlisted users
    if not requester_auth:
      try:
        client_id = oauth.get_client_id(framework_constants.OAUTH_SCOPE)
        user = oauth.get_current_user(framework_constants.OAUTH_SCOPE)
        if user:
          auth_client_ids, auth_emails = (
              client_config_svc.GetClientConfigSvc().GetClientIDEmails())
          logging.info('Oauth requester %s', user.email())
          # Check if email or client_id is allowlisted
          if (user.email() in auth_emails) or (client_id in auth_client_ids):
            logging.info('Client %r is allowlisted', user.email())
            requester_auth = authdata.AuthData.FromEmail(
                cnxn, user.email(), services)
      except oauth.Error as ex:
        logging.info('Got oauth error: %r', ex)

    # Cookie-based auth for signed in and anonymous users.
    if not requester_auth:
      # Check for signed in user
      user = users.get_current_user()
      if user:
        logging.info('Using cookie user: %r', user.email())
        requester_auth = authdata.AuthData.FromEmail(
            cnxn, user.email(), services)
      else:
        # Create AuthData for anonymous user.
        requester_auth = authdata.AuthData.FromEmail(cnxn, None, services)

      # Cookie-based auth signed-in and anon users need to have the XSRF
      # token validate.
      try:
        token = metadata.get(XSRF_TOKEN_HEADER)
        xsrf.ValidateToken(
            token, requester_auth.user_id, xsrf.XHR_SERVLET_PATH,
            timeout=self.xsrf_timeout)
      except xsrf.TokenIncorrect:
        raise permissions.PermissionException(
            'Requester %s does not have permission to make this request.'
            % requester_auth.email)

    if permissions.IsBanned(requester_auth.user_pb, requester_auth.user_view):
      raise permissions.BannedUserException(
          'The user %s has been banned from using this site' %
          requester_auth.email)

    return requester_auth

  def AssertBaseChecks(self, request, metadata):
    """Reject requests that we refuse to serve."""
    # TODO(jrobbins): Add read_only check as an exception raised in sql.py.
    if (settings.read_only and
        not request.__class__.__name__.startswith(('Get', 'List'))):
      raise permissions.PermissionException(
          'This request is not allowed in read-only mode')

    if REASON_HEADER in metadata:
      logging.info('Request reason: %r', metadata[REASON_HEADER])
    if REQUEST_ID_HEADER in metadata:
      # TODO(jrobbins): Ignore requests with duplicate request_ids.
      logging.info('request_id: %r', metadata[REQUEST_ID_HEADER])

  def GetRequestProject(self, cnxn, request):
    """Return the Project business object that the user is viewing or None."""
    if hasattr(request, 'project_name'):
      project = self.services.project.GetProjectByName(
          cnxn, request.project_name)
      if not project:
        logging.info(
            'Request has project_name: %r but it does not exist.',
            request.project_name)
        return None
      return project
    else:
      return None

  def ProcessException(self, e, prpc_context, mc):
    """Return True if we convert an exception to a pRPC status code."""
    logging.exception(e)
    exc_type = type(e)
    if exc_type == exceptions.NoSuchUserException:
      prpc_context.set_code(codes.StatusCode.NOT_FOUND)
      prpc_context.set_details('The user does not exist.')
    elif exc_type == exceptions.NoSuchProjectException:
      prpc_context.set_code(codes.StatusCode.NOT_FOUND)
      prpc_context.set_details('The project does not exist.')
    elif exc_type == exceptions.NoSuchTemplateException:
      prpc_context.set_code(codes.StatusCode.NOT_FOUND)
      prpc_context.set_details('The template does not exist.')
    elif exc_type == exceptions.NoSuchIssueException:
      prpc_context.set_code(codes.StatusCode.NOT_FOUND)
      prpc_context.set_details('The issue does not exist.')
    elif exc_type == exceptions.NoSuchCommentException:
      prpc_context.set_code(codes.StatusCode.INVALID_ARGUMENT)
      prpc_context.set_details('No such comment')
    elif exc_type == exceptions.NoSuchComponentException:
      prpc_context.set_code(codes.StatusCode.NOT_FOUND)
      prpc_context.set_details('The component does not exist.')
    elif exc_type == permissions.BannedUserException:
      prpc_context.set_code(codes.StatusCode.PERMISSION_DENIED)
      prpc_context.set_details('The requesting user has been banned.')
    elif exc_type == permissions.PermissionException:
      logging.info('perms is %r', mc.perms)
      prpc_context.set_code(codes.StatusCode.PERMISSION_DENIED)
      prpc_context.set_details('Permission denied.')
    elif exc_type == exceptions.GroupExistsException:
      prpc_context.set_code(codes.StatusCode.ALREADY_EXISTS)
      prpc_context.set_details('The user group already exists.')
    elif exc_type == features_svc.HotlistAlreadyExists:
      prpc_context.set_code(codes.StatusCode.ALREADY_EXISTS)
      prpc_context.set_details('A hotlist with that name already exists.')
    elif exc_type == exceptions.FieldDefAlreadyExists:
      prpc_context.set_code(codes.StatusCode.ALREADY_EXISTS)
      prpc_context.set_details('A field def with that name already exists.')
    elif exc_type == exceptions.InvalidComponentNameException:
      prpc_context.set_code(codes.StatusCode.INVALID_ARGUMENT)
      prpc_context.set_details('That component name is invalid.')
    elif exc_type == exceptions.FilterRuleException:
      prpc_context.set_code(codes.StatusCode.INVALID_ARGUMENT)
      prpc_context.set_details('Violates filter rule that should error.')
    elif exc_type == exceptions.InputException:
      prpc_context.set_code(codes.StatusCode.INVALID_ARGUMENT)
      prpc_context.set_details(
          'Invalid arguments: %s' % html.escape(str(e), quote=True))
    elif exc_type == ratelimiter.ApiRateLimitExceeded:
      prpc_context.set_code(codes.StatusCode.PERMISSION_DENIED)
      prpc_context.set_details('The requester has exceeded API quotas limit.')
    elif exc_type == oauth.InvalidOAuthTokenError:
      prpc_context.set_code(codes.StatusCode.UNAUTHENTICATED)
      prpc_context.set_details(
          'The oauth token was not valid or must be refreshed.')
    elif exc_type == xsrf.TokenIncorrect:
      logging.info('Bad XSRF token: %r', str(e))
      prpc_context.set_code(codes.StatusCode.INVALID_ARGUMENT)
      prpc_context.set_details('Bad XSRF token.')
    else:
      prpc_context.set_code(codes.StatusCode.INTERNAL)
      prpc_context.set_details('Potential programming error.')
      return False  # Re-raise any exception from programming errors.
    return True  # It if was one of the cases above, don't reraise.

  def RecordMonitoringStats(
      self, start_time, request, response, prpc_context, now=None):
    """Record monitoring info about this request."""
    now = now or time.time()
    elapsed_ms = int((now - start_time) * 1000)
    method_name = request.__class__.__name__
    if method_name.endswith('Request'):
      method_name = method_name[:-len('Request')]

    fields = monitoring.GetCommonFields(
        # pRPC uses its own statuses, but we report HTTP status codes.
        ConvertPRPCStatusToHTTPStatus(prpc_context),
        # Use the API name, not the request path, to prevent an explosion in
        # possible field values.
        'monorail.v0.' + method_name)

    monitoring.AddServerDurations(elapsed_ms, fields)
    monitoring.IncrementServerResponseStatusCount(fields)
    monitoring.AddServerRequesteBytes(
        len(json_format.MessageToJson(request)), fields)
    response_length = 0
    if response:
      response_length = len(json_format.MessageToJson(response))
      monitoring.AddServerResponseBytes(response_length, fields)