Merge branch 'main' into avm99963-monorail
Merged commit 4137ed7879acadbf891e8c471108acb874dae886.
GitOrigin-RevId: b6100ffc5b1da355a35f37b13fcaaf746ee8b307
diff --git a/framework/excessiveactivity.py b/framework/excessiveactivity.py
index 3737e9a..c9933db 100644
--- a/framework/excessiveactivity.py
+++ b/framework/excessiveactivity.py
@@ -12,10 +12,10 @@
from __future__ import division
from __future__ import absolute_import
-from framework import servlet
+from framework import flaskservlet
-class ExcessiveActivity(servlet.Servlet):
+class ExcessiveActivity(flaskservlet.FlaskServlet):
"""ExcessiveActivity page shows an error message."""
_PAGE_TEMPLATE = 'framework/excessive-activity-page.ezt'
diff --git a/framework/flaskservlet.py b/framework/flaskservlet.py
new file mode 100644
index 0000000..9599884
--- /dev/null
+++ b/framework/flaskservlet.py
@@ -0,0 +1,873 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+"""Base classes for Monorail Flask servlets.
+
+This is derived from servlet.py
+This base class provides handler methods that conveniently drive
+the process of parsing the request, checking base permisssion,
+gathering common page information, gathering page-specific information,
+and adding on-page debugging information (when appropriate).
+Subclasses can simply implement the page-specific logic.
+
+Summary of page classes:
+ FlaskServlet: abstract base class for all Monorail flask servlets.
+"""
+
+import gc
+import os
+import httplib
+import logging
+import time
+from businesslogic import work_env
+
+import ezt
+from features import features_bizobj, hotlist_views
+import flask
+from project import project_constants
+from proto import project_pb2
+from search import query2ast
+
+import settings
+from framework import alerts, exceptions, framework_helpers, urls
+from framework import framework_views, servlet_helpers
+from framework import framework_constants
+from framework import monorailrequest
+from framework import permissions
+from framework import ratelimiter
+from framework import template_helpers
+from framework import xsrf
+from third_party import httpagentparser
+
+from google.appengine.api import app_identity
+from google.appengine.api import modules
+from google.appengine.api import users
+from tracker import tracker_views
+
+NONCE_LENGTH = 32
+
+if not settings.unit_test_mode:
+ import MySQLdb
+
+class FlaskServlet(object):
+ """Base class for all Monorail flask servlets.
+
+ Defines a framework of methods that build up parts of the EZT page data.
+
+ Subclasses should override GatherPageData and/or ProcessFormData to
+ handle requests.
+ """
+ _MAIN_TAB_MODE = None # Normally overridden in subclasses to be one of these:
+
+ MAIN_TAB_ISSUES = 't2'
+ IN_TAB_PEOPLE = 't3'
+ MAIN_TAB_PROCESS = 't4'
+ MAIN_TAB_UPDATES = 't5'
+ MAIN_TAB_ADMIN = 't6'
+ PROCESS_TAB_SUMMARY = 'st1'
+ PROCESS_TAB_STATUSES = 'st3'
+ PROCESS_TAB_LABELS = 'st4'
+ PROCESS_TAB_RULES = 'st5'
+ PROCESS_TAB_TEMPLATES = 'st6'
+ PROCESS_TAB_COMPONENTS = 'st7'
+ PROCESS_TAB_VIEWS = 'st8'
+ ADMIN_TAB_META = 'st1'
+ ADMIN_TAB_ADVANCED = 'st9'
+ HOTLIST_TAB_ISSUES = 'ht2'
+ HOTLIST_TAB_PEOPLE = 'ht3'
+ HOTLIST_TAB_DETAILS = 'ht4'
+
+ # Most forms require a security token, however if a form is really
+ # just redirecting to a search GET request without writing any data,
+ # subclass can override this to allow anonymous use.
+ CHECK_SECURITY_TOKEN = True
+
+ # Some pages might be posted to by clients outside of Monorail.
+ # ie: The issue entry page, by the issue filing wizard. In these cases,
+ # we can allow an xhr-scoped XSRF token to be used to post to the page.
+ ALLOW_XHR = False
+
+ # This value should not typically be overridden.
+ _TEMPLATE_PATH = framework_constants.TEMPLATE_PATH
+
+ _PAGE_TEMPLATE = None # Normally overridden in subclasses.
+ _ELIMINATE_BLANK_LINES = False
+
+ _MISSING_PERMISSIONS_TEMPLATE = 'sitewide/403-page.ezt'
+
+ def __init__(self, services=None, content_type='text/html; charset=UTF-8'):
+ """Load and parse the template, saving it for later use."""
+ if self._PAGE_TEMPLATE: # specified in subclasses
+ template_path = self._TEMPLATE_PATH + self._PAGE_TEMPLATE
+ self.template = template_helpers.GetTemplate(
+ template_path, eliminate_blank_lines=self._ELIMINATE_BLANK_LINES)
+ else:
+ self.template = None
+
+ self._missing_permissions_template = template_helpers.MonorailTemplate(
+ self._TEMPLATE_PATH + self._MISSING_PERMISSIONS_TEMPLATE)
+ self.services = services or flask.current_app.config['services']
+ self.content_type = content_type
+ self.mr = None
+ self.request = flask.request
+ self.response = None
+ self.ratelimiter = ratelimiter.RateLimiter()
+
+ def handler(self):
+ """Do common stuff then dispatch the request to get() or put() methods."""
+ self.response = flask.make_response()
+ handler_start_time = time.time()
+ logging.info('\n\n\n Flask Request handler: %r', self)
+
+ #TODO: add the ts_mon.NonCumulativeDistributionMetric
+ # count0, count1, count2 = gc.get_count()
+ # logging.info('gc counts: %d %d %d', count0, count1, count2)
+ # GC_COUNT.add(count0, {'generation': 0})
+ # GC_COUNT.add(count1, {'generation': 1})
+ # GC_COUNT.add(count2, {'generation': 2})
+
+ self.mr = monorailrequest.MonorailRequest(self.services)
+ self.response.headers.add(
+ 'Strict-Transport-Security', 'max-age=31536000; includeSubDomains')
+
+ if 'X-Cloud-Trace-Context' in self.request.headers:
+ self.mr.profiler.trace_context = (
+ self.request.headers.get('X-Cloud-Trace-Context'))
+
+ if self.services.cache_manager:
+ try:
+ with self.mr.profiler.Phase('distributed invalidation'):
+ self.services.cache_manager.DoDistributedInvalidation(self.mr.cnxn)
+
+ except MySQLdb.OperationalError as e:
+ logging.exception(e)
+ page_data = {
+ 'http_response_code': httplib.SERVICE_UNAVAILABLE,
+ 'requested_url': self.request.url,
+ }
+ self.template = template_helpers.GetTemplate(
+ 'templates/framework/database-maintenance.ezt',
+ eliminate_blank_lines=self._ELIMINATE_BLANK_LINES)
+ self.template.WriteFlaskResponse(
+ self.response, page_data, content_type='text/html')
+ return self.response
+
+ try:
+ self.ratelimiter.CheckStart(self.request)
+
+ with self.mr.profiler.Phase('parsing request and doing lookups'):
+ self.mr.ParseFlaskRequest(self.request, self.services)
+
+ self.response.headers['X-Frame-Options'] = 'SAMEORIGIN'
+
+ if self.request.method == 'POST':
+ self.post()
+ elif self.request.method == 'GET':
+ self.get()
+ except exceptions.NoSuchUserException as e:
+ logging.info('Trapped NoSuchUserException %s', e)
+ flask.abort(404, 'user not found')
+
+ except exceptions.NoSuchGroupException as e:
+ logging.warning('Trapped NoSuchGroupException %s', e)
+ flask.abort(404, 'user group not found')
+
+ except exceptions.InputException as e:
+ logging.info('Rejecting invalid input: %r', e)
+ self.response.status_code = httplib.BAD_REQUEST
+
+ except exceptions.NoSuchProjectException as e:
+ logging.info('Rejecting invalid request: %r', e)
+ self.response.status_code = httplib.NOT_FOUND
+
+ except xsrf.TokenIncorrect as e:
+ logging.info('Bad XSRF token: %r', e.message)
+ self.response.status_code = httplib.BAD_REQUEST
+
+ except permissions.BannedUserException as e:
+ logging.warning('The user has been banned')
+ url = framework_helpers.FormatAbsoluteURL(
+ self.mr, urls.BANNED, include_project=False, copy_params=False)
+ self.redirect(url, abort=True)
+
+ except ratelimiter.RateLimitExceeded as e:
+ logging.info('RateLimitExceeded Exception %s', e)
+ self.response.status_code = httplib.BAD_REQUEST
+ self.response.response = 'Slow your roll.'
+
+ finally:
+ self.mr.CleanUp()
+ self.ratelimiter.CheckEnd(self.request, time.time(), handler_start_time)
+
+ total_processing_time = time.time() - handler_start_time
+ logging.info(
+ 'Processed request in %d ms', int(total_processing_time * 1000))
+
+ end_count0, end_count1, end_count2 = gc.get_count()
+ logging.info('gc counts: %d %d %d', end_count0, end_count1, end_count2)
+ # TODO: get the GC event back
+ # if (end_count0 < count0) or (end_count1 < count1) or(end_count2 < count2):
+ # GC_EVENT_REQUEST.increment()
+
+ if settings.enable_profiler_logging:
+ self.mr.profiler.LogStats()
+
+ return self.response
+
+ def get(self):
+ """Collect page-specific and generic info, then render the page.
+
+ Args:
+ Any path components parsed by webapp2 will be in kwargs, but we do
+ our own parsing later anyway, so ignore them for now.
+ """
+ page_data = {}
+ nonce = framework_helpers.MakeRandomKey(length=NONCE_LENGTH)
+ try:
+ csp_header = 'Content-Security-Policy'
+ csp_scheme = 'https:'
+ if settings.local_mode:
+ csp_header = 'Content-Security-Policy-Report-Only'
+ csp_scheme = 'http:'
+ user_agent_str = self.mr.request.headers.get('User-Agent', '')
+ ua = httpagentparser.detect(user_agent_str)
+ browser, browser_major_version = 'Unknown browser', 0
+ if ua.has_key('browser'):
+ browser = ua['browser']['name']
+ try:
+ browser_major_version = int(ua['browser']['version'].split('.')[0])
+ except ValueError:
+ logging.warn('Could not parse version: %r', ua['browser']['version'])
+ csp_supports_report_sample = (
+ (browser == 'Chrome' and browser_major_version >= 59) or
+ (browser == 'Opera' and browser_major_version >= 46))
+ version_base = servlet_helpers.VersionBaseURL(self.mr.request)
+ self.response.headers.add(
+ csp_header,
+ (
+ "default-src %(scheme)s ; "
+ "script-src"
+ " %(rep_samp)s" # Report 40 chars of any inline violation.
+ " 'unsafe-inline'" # Only counts in browsers that lack CSP2.
+ " 'strict-dynamic'" # Allows <script nonce> to load more.
+ " %(version_base)s/static/dist/"
+ " 'self' 'nonce-%(nonce)s'; "
+ "child-src 'none'; "
+ "frame-src accounts.google.com" # All used by gapi.js auth.
+ " content-issuetracker.corp.googleapis.com"
+ " login.corp.google.com up.corp.googleapis.com"
+ # Used by Google Feedback
+ " feedback.googleusercontent.com"
+ " www.google.com; "
+ "img-src %(scheme)s data: blob: ; "
+ "style-src %(scheme)s 'unsafe-inline'; "
+ "object-src 'none'; "
+ "base-uri 'self'; " # Used by Google Feedback
+ "report-uri /csp.do" % {
+ 'nonce':
+ nonce,
+ 'scheme':
+ csp_scheme,
+ 'rep_samp':
+ "'report-sample'" if csp_supports_report_sample else '',
+ 'version_base':
+ version_base,
+ }))
+
+ # add the function to get data and render page
+ page_data.update(self._GatherFlagData(self.mr))
+
+ # Page-specific work happens in this call.
+ page_data.update(self._DoPageProcessing(self.mr, nonce))
+
+ self._AddHelpDebugPageData(page_data)
+
+ with self.mr.profiler.Phase('rendering template'):
+ self._RenderResponse(page_data)
+
+ except (servlet_helpers.MethodNotSupportedError, NotImplementedError) as e:
+ # Instead of these pages throwing 500s display the 404 message and log.
+ # The motivation of this is to minimize 500s on the site to keep alerts
+ # meaningful during fuzzing. For more context see
+ # https://bugs.chromium.org/p/monorail/issues/detail?id=659
+ logging.warning('Trapped NotImplementedError %s', e)
+ flask.abort(404, 'invalid page')
+ except query2ast.InvalidQueryError as e:
+ logging.warning('Trapped InvalidQueryError: %s', e)
+ logging.exception(e)
+ msg = e.message if e.message else 'invalid query'
+ flask.abort(400, msg)
+ except permissions.PermissionException as e:
+ logging.warning('Trapped PermissionException %s', e)
+ logging.warning('mr.auth.user_id is %s', self.mr.auth.user_id)
+ logging.warning('mr.auth.effective_ids is %s', self.mr.auth.effective_ids)
+ logging.warning('mr.perms is %s', self.mr.perms)
+ if not self.mr.auth.user_id:
+ # If not logged in, let them log in
+ url = servlet_helpers.SafeCreateLoginURL(self.mr)
+ self.redirect(url, abort=True)
+ else:
+ # Display the missing permissions template.
+ page_data = {
+ 'reason': e.message,
+ 'http_response_code': httplib.FORBIDDEN,
+ }
+ with self.mr.profiler.Phase('gather base data'):
+ page_data.update(self.GatherBaseData(self.mr, nonce))
+ self._AddHelpDebugPageData(page_data)
+ self._missing_permissions_template.WriteFlaskResponse(
+ self.response, page_data, content_type=self.content_type)
+
+ def post(self):
+ logging.info('process post request')
+ try:
+ # Page-specific work happens in this call.
+ self._DoFormProcessing(self.request, self.mr)
+
+ except permissions.PermissionException as e:
+ logging.warning('Trapped permission-related exception "%s".', e)
+ self.response.status_code = httplib.BAD_REQUEST
+
+ def _RenderResponse(self, page_data):
+ logging.info('rendering response len(page_data) is %r', len(page_data))
+ self.template.WriteFlaskResponse(
+ self.response, page_data, content_type=self.content_type)
+
+ def _GatherFlagData(self, mr):
+ page_data = {
+ 'project_stars_enabled':
+ ezt.boolean(settings.enable_project_stars),
+ 'user_stars_enabled':
+ ezt.boolean(settings.enable_user_stars),
+ 'can_create_project':
+ ezt.boolean(permissions.CanCreateProject(mr.perms)),
+ 'can_create_group':
+ ezt.boolean(permissions.CanCreateGroup(mr.perms)),
+ }
+
+ return page_data
+
+ def _DoCommonRequestProcessing(self, request, mr):
+ """Do common processing dependent on having the user and project pbs."""
+ with mr.profiler.Phase('basic processing'):
+ self._CheckForMovedProject(mr, request)
+ self.AssertBasePermission(mr)
+
+ # pylint: disable=unused-argument
+ def _DoPageProcessing(self, mr, nonce):
+ """Do user lookups and gather page-specific ezt data."""
+ with mr.profiler.Phase('common request data'):
+
+ self._DoCommonRequestProcessing(self.request, mr)
+
+ self._MaybeRedirectToBrandedDomain(self.request, mr.project_name)
+
+ page_data = self.GatherBaseData(mr, nonce)
+
+ with mr.profiler.Phase('page processing'):
+ page_data.update(self.GatherPageData(mr))
+ page_data.update(mr.form_overrides)
+ template_helpers.ExpandLabels(page_data)
+ self._RecordVisitTime(mr)
+
+ return page_data
+
+ def _DoFormProcessing(self, request, mr):
+ """Do user lookups and handle form data."""
+ self._DoCommonRequestProcessing(request, mr)
+
+ if self.CHECK_SECURITY_TOKEN:
+ try:
+ xsrf.ValidateToken(
+ request.values.get('token'), mr.auth.user_id, request.path)
+ except xsrf.TokenIncorrect as err:
+ if self.ALLOW_XHR:
+ xsrf.ValidateToken(
+ request.values.get('token'), mr.auth.user_id, 'xhr')
+ else:
+ raise err
+
+ redirect_url = self.ProcessFormData(mr, request.values)
+
+ # Most forms redirect the user to a new URL on success. If no
+ # redirect_url was returned, the form handler must have already
+ # sent a response. E.g., bounced the user back to the form with
+ # invalid form fields highlighted.
+ if redirect_url:
+ self.redirect(redirect_url, abort=True)
+ else:
+ assert self.response.response
+
+ def ProcessFormData(self, mr, post_data):
+ """Handle form data and redirect appropriately.
+
+ Args:
+ mr: commonly used info parsed from the request.
+ post_data: HTML form data from the request.
+
+ Returns:
+ String URL to redirect the user to, or None if response was already sent.
+ """
+ raise servlet_helpers.MethodNotSupportedError()
+
+ def _FormHandlerURL(self, path):
+ """Return the form handler for the main form on a page."""
+ if path.endswith('/'):
+ return path + 'edit.do'
+ elif path.endswith('.do'):
+ return path # This happens as part of PleaseCorrect().
+ else:
+ return path + '.do'
+
+ # pylint: disable=unused-argument
+ def GatherPageData(self, mr):
+ """Return a dict of page-specific ezt data."""
+ raise servlet_helpers.MethodNotSupportedError()
+
+ def GatherBaseData(self, mr, nonce):
+ """Return a dict of info used on almost all pages."""
+ project = mr.project
+
+ project_summary = ''
+ project_alert = None
+ project_read_only = False
+ project_home_page = ''
+ project_thumbnail_url = ''
+ if project:
+ project_summary = project.summary
+ project_alert = servlet_helpers.CalcProjectAlert(project)
+ project_read_only = project.read_only_reason
+ project_home_page = project.home_page
+ project_thumbnail_url = tracker_views.LogoView(project).thumbnail_url
+
+ with work_env.WorkEnv(mr, self.services) as we:
+ is_project_starred = False
+ project_view = None
+ if mr.project:
+ if permissions.UserCanViewProject(mr.auth.user_pb,
+ mr.auth.effective_ids, mr.project):
+ is_project_starred = we.IsProjectStarred(mr.project_id)
+ project_view = template_helpers.PBProxy(mr.project)
+
+ grid_x_attr = None
+ grid_y_attr = None
+ hotlist_view = None
+ if mr.hotlist:
+ users_by_id = framework_views.MakeAllUserViews(
+ mr.cnxn, self.services.user,
+ features_bizobj.UsersInvolvedInHotlists([mr.hotlist]))
+ hotlist_view = hotlist_views.HotlistView(
+ mr.hotlist, mr.perms, mr.auth, mr.viewed_user_auth.user_id,
+ users_by_id,
+ self.services.hotlist_star.IsItemStarredBy(
+ mr.cnxn, mr.hotlist.hotlist_id, mr.auth.user_id))
+ grid_x_attr = mr.x.lower()
+ grid_y_attr = mr.y.lower()
+
+ app_version = os.environ.get('CURRENT_VERSION_ID')
+
+ viewed_username = None
+ if mr.viewed_user_auth.user_view:
+ viewed_username = mr.viewed_user_auth.user_view.username
+
+ config = None
+ if mr.project_id and self.services.config:
+ with mr.profiler.Phase('getting config'):
+ config = self.services.config.GetProjectConfig(mr.cnxn, mr.project_id)
+ grid_x_attr = (mr.x or config.default_x_attr).lower()
+ grid_y_attr = (mr.y or config.default_y_attr).lower()
+
+ viewing_self = mr.auth.user_id == mr.viewed_user_auth.user_id
+ offer_saved_queries_subtab = (
+ viewing_self or mr.auth.user_pb and mr.auth.user_pb.is_site_admin)
+
+ login_url = servlet_helpers.SafeCreateLoginURL(mr)
+ logout_url = servlet_helpers.SafeCreateLogoutURL(mr)
+ logout_url_goto_home = users.create_logout_url('/')
+ version_base = servlet_helpers.VersionBaseURL(mr.request)
+
+ base_data = {
+ # EZT does not have constants for True and False, so we pass them in.
+ 'True':
+ ezt.boolean(True),
+ 'False':
+ ezt.boolean(False),
+ 'local_mode':
+ ezt.boolean(settings.local_mode),
+ 'site_name':
+ settings.site_name,
+ 'show_search_metadata':
+ ezt.boolean(False),
+ 'page_template':
+ self._PAGE_TEMPLATE,
+ 'main_tab_mode':
+ self._MAIN_TAB_MODE,
+ 'project_summary':
+ project_summary,
+ 'project_home_page':
+ project_home_page,
+ 'project_thumbnail_url':
+ project_thumbnail_url,
+ 'hotlist_id':
+ mr.hotlist_id,
+ 'hotlist':
+ hotlist_view,
+ 'hostport':
+ mr.request.host,
+ 'absolute_base_url':
+ '%s://%s' % (mr.request.scheme, mr.request.host),
+ 'project_home_url':
+ None,
+ 'link_rel_canonical':
+ None, # For specifying <link rel="canonical">
+ 'projectname':
+ mr.project_name,
+ 'project':
+ project_view,
+ 'project_is_restricted':
+ ezt.boolean(servlet_helpers.ProjectIsRestricted(mr)),
+ 'offer_contributor_list':
+ ezt.boolean(permissions.CanViewContributorList(mr, mr.project)),
+ 'logged_in_user':
+ mr.auth.user_view,
+ 'form_token':
+ None, # Set to a value below iff the user is logged in.
+ 'form_token_path':
+ None,
+ 'token_expires_sec':
+ None,
+ 'xhr_token':
+ None, # Set to a value below iff the user is logged in.
+ 'flag_spam_token':
+ None,
+ 'nonce':
+ nonce,
+ 'perms':
+ mr.perms,
+ 'warnings':
+ mr.warnings,
+ 'errors':
+ mr.errors,
+ 'viewed_username':
+ viewed_username,
+ 'viewed_user':
+ mr.viewed_user_auth.user_view,
+ 'viewed_user_pb':
+ template_helpers.PBProxy(mr.viewed_user_auth.user_pb),
+ 'viewing_self':
+ ezt.boolean(viewing_self),
+ 'viewed_user_id':
+ mr.viewed_user_auth.user_id,
+ 'offer_saved_queries_subtab':
+ ezt.boolean(offer_saved_queries_subtab),
+ 'currentPageURL':
+ mr.current_page_url,
+ 'currentPageURLEncoded':
+ mr.current_page_url_encoded,
+ 'login_url':
+ login_url,
+ 'logout_url':
+ logout_url,
+ 'logout_url_goto_home':
+ logout_url_goto_home,
+ 'continue_issue_id':
+ mr.continue_issue_id,
+ 'feedback_email':
+ settings.feedback_email,
+ 'category_css':
+ None, # Used to specify a category of stylesheet
+ 'category2_css':
+ None, # specify a 2nd category of stylesheet if needed.
+ 'page_css':
+ None, # Used to add a stylesheet to a specific page.
+ 'can':
+ mr.can,
+ 'query':
+ mr.query,
+ 'colspec':
+ None,
+ 'sortspec':
+ mr.sort_spec,
+
+ # Options for issuelist display
+ 'grid_x_attr':
+ grid_x_attr,
+ 'grid_y_attr':
+ grid_y_attr,
+ 'grid_cell_mode':
+ mr.cells,
+ 'grid_mode':
+ None,
+ 'list_mode':
+ None,
+ 'chart_mode':
+ None,
+ 'is_cross_project':
+ ezt.boolean(False),
+
+ # for project search (some also used in issue search)
+ 'start':
+ mr.start,
+ 'num':
+ mr.num,
+ 'groupby':
+ mr.group_by_spec,
+ 'q_field_size':
+ (
+ min(
+ framework_constants.MAX_ARTIFACT_SEARCH_FIELD_SIZE,
+ max(
+ framework_constants.MIN_ARTIFACT_SEARCH_FIELD_SIZE,
+ len(mr.query) + framework_constants.AUTOSIZE_STEP))),
+ 'mode':
+ None, # Display mode, e.g., grid mode.
+ 'ajah':
+ mr.ajah,
+ 'table_title':
+ mr.table_title,
+ 'alerts':
+ alerts.AlertsView(mr), # For alert.ezt
+ 'project_alert':
+ project_alert,
+ 'title':
+ None, # First part of page title
+ 'title_summary':
+ None, # Appended to title on artifact detail pages
+ 'project_read_only':
+ ezt.boolean(project_read_only),
+ 'site_read_only':
+ ezt.boolean(settings.read_only),
+ 'banner_time':
+ servlet_helpers.GetBannerTime(settings.banner_time),
+ 'read_only':
+ ezt.boolean(settings.read_only or project_read_only),
+ 'site_banner_message':
+ settings.banner_message,
+ 'robots_no_index':
+ None,
+ 'analytics_id':
+ settings.analytics_id,
+ 'is_project_starred':
+ ezt.boolean(is_project_starred),
+ 'version_base':
+ version_base,
+ 'app_version':
+ app_version,
+ 'gapi_client_id':
+ settings.gapi_client_id,
+ 'viewing_user_page':
+ ezt.boolean(False),
+ 'old_ui_url':
+ None,
+ 'new_ui_url':
+ None,
+ 'is_member':
+ ezt.boolean(False),
+ }
+
+ if mr.project:
+ base_data['project_home_url'] = '/p/%s' % mr.project_name
+
+ # Always add xhr-xsrf token because even anon users need some
+ # pRPC methods, e.g., autocomplete, flipper, and charts.
+ base_data['token_expires_sec'] = xsrf.TokenExpiresSec()
+ base_data['xhr_token'] = xsrf.GenerateToken(
+ mr.auth.user_id, xsrf.XHR_SERVLET_PATH)
+ # Always add other anti-xsrf tokens when the user is logged in.
+ if mr.auth.user_id:
+ form_token_path = self._FormHandlerURL(mr.request.path)
+ form_token_path = '/'
+ base_data['form_token'] = xsrf.GenerateToken(
+ mr.auth.user_id, form_token_path)
+ base_data['form_token_path'] = form_token_path
+
+ return base_data
+
+ def _AddHelpDebugPageData(self, page_data):
+ with self.mr.profiler.Phase('help and debug data'):
+ page_data.update(self.GatherHelpData(self.mr, page_data))
+ page_data.update(self.GatherDebugData(self.mr, page_data))
+
+ # pylint: disable=unused-argument
+ def GatherHelpData(self, mr, page_data):
+ """Return a dict of values to drive on-page user help.
+ Subclasses can override this function
+ Args:
+ mr: common information parsed from the HTTP request.
+ page_data: Dictionary of base and page template data.
+
+ Returns:
+ A dict of values to drive on-page user help, to be added to page_data.
+ """
+ help_data = {
+ 'cue': None, # for cues.ezt
+ 'account_cue': None, # for cues.ezt
+ }
+ dismissed = []
+ if mr.auth.user_pb:
+ with work_env.WorkEnv(mr, self.services) as we:
+ userprefs = we.GetUserPrefs(mr.auth.user_id)
+ dismissed = [pv.name for pv in userprefs.prefs if pv.value == 'true']
+ if (mr.auth.user_pb.vacation_message and
+ 'you_are_on_vacation' not in dismissed):
+ help_data['cue'] = 'you_are_on_vacation'
+ if (mr.auth.user_pb.email_bounce_timestamp and
+ 'your_email_bounced' not in dismissed):
+ help_data['cue'] = 'your_email_bounced'
+ if mr.auth.user_pb.linked_parent_id:
+ # This one is not dismissable.
+ help_data['account_cue'] = 'switch_to_parent_account'
+ parent_email = self.services.user.LookupUserEmail(
+ mr.cnxn, mr.auth.user_pb.linked_parent_id)
+ help_data['parent_email'] = parent_email
+
+ return help_data
+
+ def GatherDebugData(self, mr, page_data):
+ """Return debugging info for display at the very bottom of the page."""
+ if mr.debug_enabled:
+ debug = [servlet_helpers.ContextDebugCollection('Page data', page_data)]
+ debug = [('none', 'recorded')]
+ return {
+ 'dbg': 'on',
+ 'debug': debug,
+ 'profiler': mr.profiler,
+ }
+ else:
+ if '?' in mr.current_page_url:
+ debug_url = mr.current_page_url + '&debug=1'
+ else:
+ debug_url = mr.current_page_url + '?debug=1'
+
+ return {
+ 'debug_uri': debug_url,
+ 'dbg': 'off',
+ 'debug': [('none', 'recorded')],
+ }
+
+ def _CheckForMovedProject(self, mr, request):
+ """If the project moved, redirect there or to an informational page."""
+ if not mr.project:
+ return # We are on a site-wide or user page.
+ if not mr.project.moved_to:
+ return # This project has not moved.
+ admin_url = '/p/%s%s' % (mr.project_name, urls.ADMIN_META)
+ if request.path.startswith(admin_url):
+ return # It moved, but we are near the page that can un-move it.
+
+ logging.info(
+ 'project %s has moved: %s', mr.project.project_name,
+ mr.project.moved_to)
+
+ moved_to = mr.project.moved_to
+ if project_constants.RE_PROJECT_NAME.match(moved_to):
+ # Use the redir query parameter to avoid redirect loops.
+ if mr.redir is None:
+ url = framework_helpers.FormatMovedProjectURL(mr, moved_to)
+ if '?' in url:
+ url += '&redir=1'
+ else:
+ url += '?redir=1'
+ logging.info('trusted move to a new project on our site')
+ self.redirect(url, abort=True)
+
+ logging.info('not a trusted move, will display link to user to click')
+ # Attach the project name as a url param instead of generating a /p/
+ # link to the destination project.
+ url = framework_helpers.FormatAbsoluteURL(
+ mr,
+ urls.PROJECT_MOVED,
+ include_project=False,
+ copy_params=False,
+ project=mr.project_name)
+ self.redirect(url, abort=True)
+
+ def _MaybeRedirectToBrandedDomain(self, request, project_name):
+ """If we are live and the project should be branded, check request host."""
+ if request.values.get('redir'):
+ return # Avoid any chance of a redirect loop.
+ if not project_name:
+ return
+ needed_domain = framework_helpers.GetNeededDomain(
+ project_name, request.host)
+ if not needed_domain:
+ return
+
+ url = 'https://%s%s' % (needed_domain, request.full_path)
+ if '?' in url:
+ url += '&redir=1'
+ else:
+ url += '?redir=1'
+ logging.info('branding redirect to url %r', url)
+ self.redirect(url, abort=True)
+
+ def AssertBasePermission(self, mr):
+ """Make sure that the logged in user has permission to view this page.
+
+ Subclasses should call super, then check additional permissions
+ and raise a PermissionException if the user is not authorized to
+ do something.
+
+ Args:
+ mr: commonly used info parsed from the request.
+
+ Raises:
+ PermissionException: If the user does not have permisssion to view
+ the current page.
+ """
+ servlet_helpers.AssertBasePermission(mr)
+
+ def CheckPerm(self, mr, perm, art=None, granted_perms=None):
+ """Return True if the user can use the requested permission."""
+ return servlet_helpers.CheckPerm(
+ mr, perm, art=art, granted_perms=granted_perms)
+
+ def MakePagePerms(self, mr, art, *perm_list, **kwargs):
+ """Make an EZTItem with a set of permissions needed in a given template.
+
+ Args:
+ mr: commonly used info parsed from the request.
+ art: a project artifact, such as an issue.
+ *perm_list: any number of permission names that are referenced
+ in the EZT template.
+ **kwargs: dictionary that may include 'granted_perms' list of permissions
+ granted to the current user specifically on the current page.
+
+ Returns:
+ An EZTItem with one attribute for each permission and the value
+ of each attribute being an ezt.boolean(). True if the user
+ is permitted to do that action on the given artifact, or
+ False if not.
+ """
+ granted_perms = kwargs.get('granted_perms')
+ page_perms = template_helpers.EZTItem()
+ for perm in perm_list:
+ setattr(
+ page_perms, perm,
+ ezt.boolean(
+ self.CheckPerm(mr, perm, art=art, granted_perms=granted_perms)))
+
+ return page_perms
+
+ def redirect(self, url, abort=False):
+ if abort:
+ flask.redirect(url, code=302)
+ flask.abort(302)
+ else:
+ flask.redirect(url)
+
+ def PleaseCorrect(self, mr, **echo_data):
+ """Show the same form again so that the user can correct their input."""
+ mr.PrepareForReentry(echo_data)
+ self.get()
+
+ def _RecordVisitTime(self, mr, now=None):
+ """Record the signed in user's last visit time, if possible."""
+ now = now or int(time.time())
+ if not settings.read_only and mr.auth.user_id:
+ user_pb = mr.auth.user_pb
+ if (user_pb.last_visit_timestamp <
+ now - framework_constants.VISIT_RESOLUTION):
+ user_pb.last_visit_timestamp = now
+ self.services.user.UpdateUser(mr.cnxn, user_pb.user_id, user_pb)
diff --git a/framework/monitoring.py b/framework/monitoring.py
index 6ddeeb9..6407e2d 100644
--- a/framework/monitoring.py
+++ b/framework/monitoring.py
@@ -4,8 +4,14 @@
"""Monitoring ts_mon custom to monorail."""
+import os
+import sys
+lib_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')
+
+from google.cloud import logging
from infra_libs import ts_mon
from framework import framework_helpers
+import settings
def GetCommonFields(status, name, is_robot=False):
@@ -24,8 +30,10 @@
ts_mon.StringField('client_email'),
ts_mon.StringField('version')])
-def IncrementAPIRequestsCount(version, client_id, client_email=None):
- # type: (str, str, Optional[str]) -> None
+
+def IncrementAPIRequestsCount(
+ version, client_id, client_email=None, handler='none'):
+ # type: (str, str, Optional[str], Optional[str]) -> None
"""Increment the request count in ts_mon."""
if not client_email:
client_email = 'anonymous'
@@ -40,6 +48,18 @@
}
API_REQUESTS_COUNT.increment_by(1, fields)
+ if not settings.unit_test_mode:
+ logging_client = logging.Client()
+ logger = logging_client.logger("request_log")
+ logger.log_struct(
+ {
+ 'log_type': "IncrementAPIRequestsCount",
+ 'client_id': client_id,
+ 'client_email': client_email,
+ 'requests_count': str(API_REQUESTS_COUNT.get(fields)),
+ 'endpoint': handler
+ })
+
# 90% of durations are in the range 11-1873ms. Growth factor 10^0.06 puts that
# range into 37 buckets. Max finite bucket value is 12 minutes.
diff --git a/framework/monorailrequest.py b/framework/monorailrequest.py
index e51aa15..94dd9d6 100644
--- a/framework/monorailrequest.py
+++ b/framework/monorailrequest.py
@@ -270,7 +270,6 @@
if self.viewed_username:
self._LookupViewedUser(services)
self._LookupLoggedInUser(services)
- # TODO(jrobbins): re-implement HandleLurkerViewingSelf()
if not self.hotlist:
self._LookupHotlist(services)
@@ -287,6 +286,62 @@
if settings.local_mode or 'staging' in request.host:
self.use_cached_searches = False
+ def ParseFlaskRequest(self, request, services, do_user_lookups=True):
+ """Parse tons of useful info from the given flask request object.
+
+ Args:
+ request: flask Request object w/ path and query params.
+ services: connections to backend servers including DB.
+ do_user_lookups: Set to False to disable lookups during testing.
+ """
+ with self.profiler.Phase('basic parsing'):
+ self.request = request
+ self.current_page_url = request.url
+ self.current_page_url_encoded = urllib.quote_plus(self.current_page_url)
+
+ # Only accept a hostport from the request that looks valid.
+ if not _HOSTPORT_RE.match(request.host):
+ raise exceptions.InputException(
+ 'request.host looks funny: %r', request.host)
+
+ logging.info('Flask Request: %s', self.current_page_url)
+
+ with self.profiler.Phase('path parsing'):
+ (viewed_user_val, self.project_name, self.hotlist_id,
+ self.hotlist_name) = _ParsePathIdentifiers(self.request.url)
+ self.viewed_username = _GetViewedEmail(
+ viewed_user_val, self.cnxn, services)
+ with self.profiler.Phase('qs parsing'):
+ self._ParseQueryParameters()
+ with self.profiler.Phase('overrides parsing'):
+ self._ParseFormOverrides()
+
+ if not self.project: # It can be already set in unit tests.
+ self._LookupProject(services)
+ if self.project_id and services.config:
+ self.config = services.config.GetProjectConfig(self.cnxn, self.project_id)
+
+ if do_user_lookups:
+ if self.viewed_username:
+ self._LookupViewedUser(services)
+ self._LookupLoggedInUser(services)
+
+ if not self.hotlist:
+ self._LookupHotlist(services)
+
+ if self.query is None:
+ self.query = self._CalcDefaultQuery()
+
+ prod_debug_allowed = self.perms.HasPerm(
+ permissions.VIEW_DEBUG, self.auth.user_id, None)
+ self.debug_enabled = (
+ request.args.get('debug') and
+ (settings.local_mode or prod_debug_allowed))
+ # temporary option for perf testing on staging instance.
+ if request.args.get('disable_cache'):
+ if settings.local_mode or 'staging' in request.host:
+ self.use_cached_searches = False
+
def _CalcDefaultQuery(self):
"""When URL has no q= param, return the default for members or ''."""
if (self.can == 2 and self.project and self.auth.effective_ids and
@@ -538,7 +593,11 @@
def GetParam(self, query_param_name, default_value=None,
antitamper_re=None):
"""Get a query parameter from the URL as a utf8 string."""
- value = self.request.params.get(query_param_name)
+ value = None
+ if hasattr(self.request, 'params'):
+ value = self.request.params.get(query_param_name)
+ else:
+ value = self.request.args.get(query_param_name)
assert value is None or isinstance(value, six.text_type)
using_default = value is None
if using_default:
@@ -557,7 +616,11 @@
def GetIntParam(self, query_param_name, default_value=None):
"""Get an integer param from the URL or default."""
- value = self.request.params.get(query_param_name)
+ value = None
+ if hasattr(self.request, 'params'):
+ value = self.request.params.get(query_param_name)
+ else:
+ value = self.request.args.get(query_param_name)
if value is None or value == '':
return default_value
@@ -574,7 +637,11 @@
def GetListParam(self, query_param_name, default_value=None):
"""Get a list of strings from the URL or default."""
- params = self.request.params.get(query_param_name)
+ params = None
+ if hasattr(self.request, 'params'):
+ params = self.request.params.get(query_param_name)
+ else:
+ params = self.request.args.get(query_param_name)
if params is None:
return default_value
if not params:
@@ -594,7 +661,12 @@
def GetBoolParam(self, query_param_name, default_value=None):
"""Get a boolean param from the URL or default."""
- value = self.request.params.get(query_param_name)
+ value = None
+ if hasattr(self.request, 'params'):
+ value = self.request.params.get(query_param_name)
+ else:
+ value = self.request.args.get(query_param_name)
+
if value is None:
return default_value
diff --git a/framework/servlet.py b/framework/servlet.py
index 1ed6935..e1c0cf1 100644
--- a/framework/servlet.py
+++ b/framework/servlet.py
@@ -90,17 +90,6 @@
# 'cloudtrace', 'v1', credentials=credentials)
# except Exception as e:
# logging.warning('could not get trace service: %s', e)
-
-
-class MethodNotSupportedError(NotImplementedError):
- """An exception class for indicating that the method is not supported.
-
- Used by GatherPageData and ProcessFormData to indicate that GET and POST,
- respectively, are not supported methods on the given Servlet.
- """
- pass
-
-
class Servlet(webapp2.RequestHandler):
"""Base class for all Monorail servlets.
@@ -318,7 +307,7 @@
csp_supports_report_sample = (
(browser == 'Chrome' and browser_major_version >= 59) or
(browser == 'Opera' and browser_major_version >= 46))
- version_base = _VersionBaseURL(self.mr.request)
+ version_base = servlet_helpers.VersionBaseURL(self.mr.request)
self.response.headers.add(csp_header,
("default-src %(scheme)s ; "
"script-src"
@@ -355,7 +344,7 @@
with self.mr.profiler.Phase('rendering template'):
self._RenderResponse(page_data)
- except (MethodNotSupportedError, NotImplementedError) as e:
+ except (servlet_helpers.MethodNotSupportedError, NotImplementedError) as e:
# Instead of these pages throwing 500s display the 404 message and log.
# The motivation of this is to minimize 500s on the site to keep alerts
# meaningful during fuzzing. For more context see
@@ -374,7 +363,7 @@
logging.warning('mr.perms is %s', self.mr.perms)
if not self.mr.auth.user_id:
# If not logged in, let them log in
- url = _SafeCreateLoginURL(self.mr)
+ url = servlet_helpers.SafeCreateLoginURL(self.mr)
self.redirect(url, abort=True)
else:
# Display the missing permissions template.
@@ -388,12 +377,6 @@
self._missing_permissions_template.WriteResponse(
self.response, page_data, content_type=self.content_type)
- def SetCacheHeaders(self, response):
- """Set headers to allow the response to be cached."""
- headers = framework_helpers.StaticCacheHeaders()
- for name, value in headers:
- response.headers[name] = value
-
def GetTemplate(self, _page_data):
"""Get the template to use for writing the http response.
@@ -437,7 +420,7 @@
Returns:
String URL to redirect the user to, or None if response was already sent.
"""
- raise MethodNotSupportedError()
+ raise servlet_helpers.MethodNotSupportedError()
def post(self, **kwargs):
"""Parse the request, check base perms, and call form-specific code."""
@@ -607,7 +590,7 @@
project_thumbnail_url = ''
if project:
project_summary = project.summary
- project_alert = _CalcProjectAlert(project)
+ project_alert = servlet_helpers.CalcProjectAlert(project)
project_read_only = project.read_only_reason
project_home_page = project.home_page
project_thumbnail_url = tracker_views.LogoView(project).thumbnail_url
@@ -653,10 +636,10 @@
offer_saved_queries_subtab = (
viewing_self or mr.auth.user_pb and mr.auth.user_pb.is_site_admin)
- login_url = _SafeCreateLoginURL(mr)
- logout_url = _SafeCreateLogoutURL(mr)
+ login_url = servlet_helpers.SafeCreateLoginURL(mr)
+ logout_url = servlet_helpers.SafeCreateLogoutURL(mr)
logout_url_goto_home = users.create_logout_url('/')
- version_base = _VersionBaseURL(mr.request)
+ version_base = servlet_helpers.VersionBaseURL(mr.request)
base_data = {
# EZT does not have constants for True and False, so we pass them in.
@@ -697,7 +680,7 @@
'project':
project_view,
'project_is_restricted':
- ezt.boolean(_ProjectIsRestricted(mr)),
+ ezt.boolean(servlet_helpers.ProjectIsRestricted(mr)),
'offer_contributor_list':
ezt.boolean(permissions.CanViewContributorList(mr, mr.project)),
'logged_in_user':
@@ -784,10 +767,13 @@
mr.num,
'groupby':
mr.group_by_spec,
- 'q_field_size': (min(
- framework_constants.MAX_ARTIFACT_SEARCH_FIELD_SIZE,
- max(framework_constants.MIN_ARTIFACT_SEARCH_FIELD_SIZE,
- len(mr.query) + framework_constants.AUTOSIZE_STEP))),
+ 'q_field_size':
+ (
+ min(
+ framework_constants.MAX_ARTIFACT_SEARCH_FIELD_SIZE,
+ max(
+ framework_constants.MIN_ARTIFACT_SEARCH_FIELD_SIZE,
+ len(mr.query) + framework_constants.AUTOSIZE_STEP))),
'mode':
None, # Display mode, e.g., grid mode.
'ajah':
@@ -866,7 +852,7 @@
def GatherPageData(self, mr):
"""Return a dict of page-specific ezt data."""
- raise MethodNotSupportedError()
+ raise servlet_helpers.MethodNotSupportedError()
# pylint: disable=unused-argument
def GatherHelpData(self, mr, page_data):
@@ -907,7 +893,7 @@
def GatherDebugData(self, mr, page_data):
"""Return debugging info for display at the very bottom of the page."""
if mr.debug_enabled:
- debug = [_ContextDebugCollection('Page data', page_data)]
+ debug = [servlet_helpers.ContextDebugCollection('Page data', page_data)]
return {
'dbg': 'on',
'debug': debug,
@@ -939,109 +925,3 @@
now - framework_constants.VISIT_RESOLUTION):
user_pb.last_visit_timestamp = now
self.services.user.UpdateUser(mr.cnxn, user_pb.user_id, user_pb)
-
-
-def _CalcProjectAlert(project):
- """Return a string to be shown as red text explaning the project state."""
-
- project_alert = None
-
- if project.read_only_reason:
- project_alert = 'READ-ONLY: %s.' % project.read_only_reason
- if project.moved_to:
- project_alert = 'This project has moved to: %s.' % project.moved_to
- elif project.delete_time:
- delay_seconds = project.delete_time - time.time()
- delay_days = delay_seconds // framework_constants.SECS_PER_DAY
- if delay_days <= 0:
- project_alert = 'Scheduled for deletion today.'
- else:
- days_word = 'day' if delay_days == 1 else 'days'
- project_alert = (
- 'Scheduled for deletion in %d %s.' % (delay_days, days_word))
- elif project.state == project_pb2.ProjectState.ARCHIVED:
- project_alert = 'Project is archived: read-only by members only.'
-
- return project_alert
-
-
-class _ContextDebugItem(object):
- """Wrapper class to generate on-screen debugging output."""
-
- def __init__(self, key, val):
- """Store the key and generate a string for the value."""
- self.key = key
- if isinstance(val, list):
- nested_debug_strs = [self.StringRep(v) for v in val]
- self.val = '[%s]' % ', '.join(nested_debug_strs)
- else:
- self.val = self.StringRep(val)
-
- def StringRep(self, val):
- """Make a useful string representation of the given value."""
- try:
- return val.DebugString()
- except Exception:
- try:
- return str(val.__dict__)
- except Exception:
- return repr(val)
-
-
-class _ContextDebugCollection(object):
- """Attach a title to a dictionary for exporting as a table of debug info."""
-
- def __init__(self, title, collection):
- self.title = title
- self.collection = [_ContextDebugItem(key, collection[key])
- for key in sorted(collection.keys())]
-
-
-def _ProjectIsRestricted(mr):
- """Return True if the mr has a 'private' project."""
- return (mr.project and
- mr.project.access != project_pb2.ProjectAccess.ANYONE)
-
-
-def _SafeCreateLoginURL(mr, continue_url=None):
- """Make a login URL w/ a detailed continue URL, otherwise use a short one."""
- continue_url = continue_url or mr.current_page_url
- try:
- url = users.create_login_url(continue_url)
- except users.RedirectTooLongError:
- if mr.project_name:
- url = users.create_login_url('/p/%s' % mr.project_name)
- else:
- url = users.create_login_url('/')
-
- # Give the user a choice of existing accounts in their session
- # or the option to add an account, even if they are currently
- # signed in to exactly one account.
- if mr.auth.user_id:
- # Notice: this makes assuptions about the output of users.create_login_url,
- # which can change at any time. See https://crbug.com/monorail/3352.
- url = url.replace('/ServiceLogin', '/AccountChooser', 1)
- return url
-
-
-def _SafeCreateLogoutURL(mr):
- """Make a logout URL w/ a detailed continue URL, otherwise use a short one."""
- try:
- return users.create_logout_url(mr.current_page_url)
- except users.RedirectTooLongError:
- if mr.project_name:
- return users.create_logout_url('/p/%s' % mr.project_name)
- else:
- return users.create_logout_url('/')
-
-
-def _VersionBaseURL(request):
- """Return a version-specific URL that we use to load static assets."""
- if settings.local_mode:
- version_base = '%s://%s' % (request.scheme, request.host)
- else:
- version_base = '%s://%s-dot-%s' % (
- request.scheme, modules.get_current_version_name(),
- app_identity.get_default_version_hostname())
-
- return version_base
diff --git a/framework/servlet_helpers.py b/framework/servlet_helpers.py
index 68eb0c4..89fe587 100644
--- a/framework/servlet_helpers.py
+++ b/framework/servlet_helpers.py
@@ -8,28 +8,83 @@
from __future__ import division
from __future__ import absolute_import
+import settings
import calendar
import datetime
import logging
import urllib
+import time
+from framework import framework_constants
from framework import framework_bizobj
from framework import framework_helpers
from framework import permissions
from framework import template_helpers
from framework import urls
from framework import xsrf
+from proto import project_pb2
+
+from google.appengine.api import app_identity
+from google.appengine.api import modules
+from google.appengine.api import users
_ZERO = datetime.timedelta(0)
+
+class MethodNotSupportedError(NotImplementedError):
+ """An exception class for indicating that the method is not supported.
+
+ Used by GatherPageData and ProcessFormData in Servlet.
+ """
+ pass
+
+
+class _ContextDebugItem(object):
+ """Wrapper class to generate on-screen debugging output."""
+
+ def __init__(self, key, val):
+ """Store the key and generate a string for the value."""
+ self.key = key
+ if isinstance(val, list):
+ nested_debug_strs = [self.StringRep(v) for v in val]
+ self.val = '[%s]' % ', '.join(nested_debug_strs)
+ else:
+ self.val = self.StringRep(val)
+
+ def StringRep(self, val):
+ """Make a useful string representation of the given value."""
+ try:
+ return val.DebugString()
+ except Exception:
+ try:
+ return str(val.__dict__)
+ except Exception:
+ return repr(val)
+
+
+class ContextDebugCollection(object):
+ """Attach a title to a dictionary for exporting as a table of debug info."""
+
+ def __init__(self, title, collection):
+ self.title = title
+ self.collection = [
+ _ContextDebugItem(key, collection[key])
+ for key in sorted(collection.keys())
+ ]
+
+
class _UTCTimeZone(datetime.tzinfo):
- """UTC"""
- def utcoffset(self, _dt):
- return _ZERO
- def tzname(self, _dt):
- return "UTC"
- def dst(self, _dt):
- return _ZERO
+ """UTC"""
+
+ def utcoffset(self, _dt):
+ return _ZERO
+
+ def tzname(self, _dt):
+ return "UTC"
+
+ def dst(self, _dt):
+ return _ZERO
+
_UTC = _UTCTimeZone()
@@ -132,21 +187,11 @@
case. Otherewise it will be a fully qualified URL that includes some
query string parameters.
"""
+ # TODO: remove the custom_issue_entry_url since its no longer
if not config.custom_issue_entry_url:
return '/p/%s/issues/entry' % (mr.project_name)
- base_url = config.custom_issue_entry_url
- sep = '&' if '?' in base_url else '?'
- token = xsrf.GenerateToken(
- mr.auth.user_id, '/p/%s%s%s' % (mr.project_name, urls.ISSUE_ENTRY, '.do'))
- role_name = framework_helpers.GetRoleName(mr.auth.effective_ids, mr.project)
-
- continue_url = urllib.quote(framework_helpers.FormatAbsoluteURL(
- mr, urls.ISSUE_ENTRY + '.do'))
-
- return '%s%stoken=%s&role=%s&continue=%s' % (
- base_url, sep, urllib.quote(token),
- urllib.quote(role_name or ''), continue_url)
+ return '/p/chromium/issues/wizard'
def IssueListURL(mr, config, query_string=None):
@@ -158,3 +203,76 @@
if config and config.member_default_query:
url += '?q=' + urllib.quote_plus(config.member_default_query)
return url
+
+
+def ProjectIsRestricted(mr):
+ """Return True if the mr has a 'private' project."""
+ return (mr.project and mr.project.access != project_pb2.ProjectAccess.ANYONE)
+
+
+def SafeCreateLoginURL(mr, continue_url=None):
+ """Make a login URL w/ a detailed continue URL, otherwise use a short one."""
+ continue_url = continue_url or mr.current_page_url
+ try:
+ url = users.create_login_url(continue_url)
+ except users.RedirectTooLongError:
+ if mr.project_name:
+ url = users.create_login_url('/p/%s' % mr.project_name)
+ else:
+ url = users.create_login_url('/')
+
+ # Give the user a choice of existing accounts in their session
+ # or the option to add an account, even if they are currently
+ # signed in to exactly one account.
+ if mr.auth.user_id:
+ # Notice: this makes assuptions about the output of users.create_login_url,
+ # which can change at any time. See https://crbug.com/monorail/3352.
+ url = url.replace('/ServiceLogin', '/AccountChooser', 1)
+ return url
+
+
+def SafeCreateLogoutURL(mr):
+ """Make a logout URL w/ a detailed continue URL, otherwise use a short one."""
+ try:
+ return users.create_logout_url(mr.current_page_url)
+ except users.RedirectTooLongError:
+ if mr.project_name:
+ return users.create_logout_url('/p/%s' % mr.project_name)
+ else:
+ return users.create_logout_url('/')
+
+
+def VersionBaseURL(request):
+ """Return a version-specific URL that we use to load static assets."""
+ if settings.local_mode:
+ version_base = '%s://%s' % (request.scheme, request.host)
+ else:
+ version_base = '%s://%s-dot-%s' % (
+ request.scheme, modules.get_current_version_name(),
+ app_identity.get_default_version_hostname())
+
+ return version_base
+
+
+def CalcProjectAlert(project):
+ """Return a string to be shown as red text explaining the project state."""
+
+ project_alert = None
+
+ if project.read_only_reason:
+ project_alert = 'READ-ONLY: %s.' % project.read_only_reason
+ if project.moved_to:
+ project_alert = 'This project has moved to: %s.' % project.moved_to
+ elif project.delete_time:
+ delay_seconds = project.delete_time - time.time()
+ delay_days = delay_seconds // framework_constants.SECS_PER_DAY
+ if delay_days <= 0:
+ project_alert = 'Scheduled for deletion today.'
+ else:
+ days_word = 'day' if delay_days == 1 else 'days'
+ project_alert = (
+ 'Scheduled for deletion in %d %s.' % (delay_days, days_word))
+ elif project.state == project_pb2.ProjectState.ARCHIVED:
+ project_alert = 'Project is archived: read-only by members only.'
+
+ return project_alert
diff --git a/framework/template_helpers.py b/framework/template_helpers.py
index 5f383c3..52a25d7 100644
--- a/framework/template_helpers.py
+++ b/framework/template_helpers.py
@@ -166,6 +166,20 @@
response.write(whole_page)
logging.info('wrote response in %dms', int((time.time() - start) * 1000))
+ def WriteFlaskResponse(self, response, data, content_type=None):
+ """Write the parsed and filled in template to http server."""
+ if content_type:
+ response.content_type = content_type
+
+ response.status_code = data.get('http_response_code', httplib.OK)
+ whole_page = self.GetResponse(data)
+ if data.get('prevent_sniffing'):
+ for sniff_pattern, sniff_replacement in SNIFFABLE_PATTERNS.items():
+ whole_page = whole_page.replace(sniff_pattern, sniff_replacement)
+ start = time.time()
+ response.response = whole_page
+ logging.info('wrote response in %dms', int((time.time() - start) * 1000))
+
def GetResponse(self, data):
"""Generate the text from the template and return it as a string."""
template = self.GetTemplate()
diff --git a/framework/test/flask_servlet_test.py b/framework/test/flask_servlet_test.py
new file mode 100644
index 0000000..443f919
--- /dev/null
+++ b/framework/test/flask_servlet_test.py
@@ -0,0 +1,108 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+"""Unit tests for servlet base class module."""
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+
+import time
+import mock
+import unittest
+import logging
+
+from google.appengine.ext import testbed
+
+from framework import flaskservlet, framework_constants, servlet_helpers
+from framework import xsrf
+from proto import project_pb2
+from proto import tracker_pb2
+from proto import user_pb2
+from services import service_manager
+from testing import fake
+from testing import testing_helpers
+
+
+class TestableFlaskServlet(flaskservlet.FlaskServlet):
+ """A tiny concrete subclass of abstract class Servlet."""
+
+ def __init__(self, services=None, do_post_redirect=True):
+ super(TestableFlaskServlet, self).__init__(services=services)
+ self.do_post_redirect = do_post_redirect
+ self.seen_post_data = None
+
+
+class FlaskServletTest(unittest.TestCase):
+
+ def setUp(self):
+ services = service_manager.Services(
+ project=fake.ProjectService(),
+ project_star=fake.ProjectStarService(),
+ user=fake.UserService(),
+ usergroup=fake.UserGroupService())
+ services.user.TestAddUser('user@example.com', 111)
+ self.page_class = flaskservlet.FlaskServlet(services=services)
+ self.testbed = testbed.Testbed()
+ self.testbed.activate()
+ self.testbed.init_user_stub()
+ self.testbed.init_memcache_stub()
+ self.testbed.init_datastore_v3_stub()
+
+ def tearDown(self):
+ self.testbed.deactivate()
+
+ def testDefaultValues(self):
+ self.assertEqual(None, self.page_class._MAIN_TAB_MODE)
+ self.assertTrue(self.page_class._TEMPLATE_PATH.endswith('/templates/'))
+ self.assertEqual(None, self.page_class._PAGE_TEMPLATE)
+
+ @mock.patch('flask.abort')
+ def testCheckForMovedProject_NoRedirect(self, mock_abort):
+ project = fake.Project(
+ project_name='proj', state=project_pb2.ProjectState.LIVE)
+ request, mr = testing_helpers.GetRequestObjects(
+ path='/p/proj', project=project)
+ self.page_class._CheckForMovedProject(mr, request)
+ mock_abort.assert_not_called()
+
+ request, mr = testing_helpers.GetRequestObjects(
+ path='/p/proj/source/browse/p/adminAdvanced', project=project)
+ self.page_class._CheckForMovedProject(mr, request)
+ mock_abort.assert_not_called()
+
+ @mock.patch('flask.abort')
+ def testCheckForMovedProject_Redirect(self, mock_abort):
+ project = fake.Project(project_name='proj', moved_to='http://example.com')
+ request, mr = testing_helpers.GetRequestObjects(
+ path='/p/proj', project=project)
+ self.page_class._CheckForMovedProject(mr, request)
+ mock_abort.assert_called_once_with(302)
+
+ def testGatherBaseData(self):
+ project = self.page_class.services.project.TestAddProject(
+ 'testproj', state=project_pb2.ProjectState.LIVE)
+ project.cached_content_timestamp = 12345
+
+ (_request, mr) = testing_helpers.GetRequestObjects(
+ path='/p/testproj/feeds', project=project)
+ nonce = '1a2b3c4d5e6f7g'
+
+ base_data = self.page_class.GatherBaseData(mr, nonce)
+
+ self.assertEqual(base_data['nonce'], nonce)
+ self.assertEqual(base_data['projectname'], 'testproj')
+ self.assertEqual(base_data['project'].cached_content_timestamp, 12345)
+ self.assertEqual(base_data['project_alert'], None)
+
+ self.assertTrue(base_data['currentPageURL'].endswith('/p/testproj/feeds'))
+ self.assertTrue(
+ base_data['currentPageURLEncoded'].endswith('%2Fp%2Ftestproj%2Ffeeds'))
+
+ def testGatherHelpData_Normal(self):
+ project = fake.Project(project_name='proj')
+ _request, mr = testing_helpers.GetRequestObjects(
+ path='/p/proj', project=project)
+ help_data = self.page_class.GatherHelpData(mr, {})
+ self.assertEqual(None, help_data['cue'])
+ self.assertEqual(None, help_data['account_cue'])
diff --git a/framework/test/servlet_helpers_test.py b/framework/test/servlet_helpers_test.py
index a2fe687..19f4ea4 100644
--- a/framework/test/servlet_helpers_test.py
+++ b/framework/test/servlet_helpers_test.py
@@ -9,6 +9,7 @@
from __future__ import absolute_import
import unittest
+import settings
from google.appengine.ext import testbed
@@ -75,6 +76,16 @@
permissions.PermissionException,
servlet_helpers.AssertBasePermission, mr)
+ def testPermForProject(self):
+ project = project_pb2.Project()
+ project.project_name = 'proj'
+ project.access = project_pb2.ProjectAccess.MEMBERS_ONLY
+ _, mr = testing_helpers.GetRequestObjects(path='/p/proj/', project=project)
+ mr.auth.user_pb.email = settings.borg_service_account
+ project_perm = servlet_helpers.CheckPermForProject(
+ mr, permissions.CREATE_GROUP, project)
+ self.assertTrue(project_perm)
+
FORM_URL = 'http://example.com/issues/form.php'
@@ -110,10 +121,7 @@
self.config.custom_issue_entry_url = FORM_URL
url = servlet_helpers.ComputeIssueEntryURL(mr, self.config)
- self.assertTrue(url.startswith(FORM_URL))
- self.assertIn('token=', url)
- self.assertIn('role=', url)
- self.assertIn('continue=', url)
+ self.assertIn('/issues/wizard', url)
class IssueListURLTest(unittest.TestCase):
@@ -166,3 +174,60 @@
url = servlet_helpers.IssueListURL(mr, self.config, query_string='q=Pri=1')
self.assertEqual('/p/proj/issues/list?q=Pri=1', url)
+
+class ProjectIsRestrictedTest(unittest.TestCase):
+
+ def setUp(self):
+ self.project = project_pb2.Project()
+ self.project.project_name = 'proj'
+ self.config = tracker_pb2.ProjectIssueConfig()
+ self.testbed = testbed.Testbed()
+ self.testbed.activate()
+ self.testbed.init_user_stub()
+ self.testbed.init_memcache_stub()
+ self.testbed.init_datastore_v3_stub()
+
+ def tearDown(self):
+ self.testbed.deactivate()
+
+ def testProjectIsRestricted(self):
+ self.project.access = project_pb2.ProjectAccess.MEMBERS_ONLY
+ _request, mr = testing_helpers.GetRequestObjects(
+ path='/p/proj/issues/detail?id=123&q=term', project=self.project)
+ isRestrict = servlet_helpers.ProjectIsRestricted(mr)
+ self.assertTrue(isRestrict)
+
+ def testProjectIsNotRestricted(self):
+ self.project.access = project_pb2.ProjectAccess.ANYONE
+ _request, mr = testing_helpers.GetRequestObjects(
+ path='/p/proj/issues/detail?id=123&q=term', project=self.project)
+ isRestrict = servlet_helpers.ProjectIsRestricted(mr)
+ self.assertFalse(isRestrict)
+
+
+class ComputerCreateUrl(unittest.TestCase):
+
+ def setUp(self):
+ self.project = project_pb2.Project()
+ self.project.project_name = 'proj'
+ self.config = tracker_pb2.ProjectIssueConfig()
+ self.testbed = testbed.Testbed()
+ self.testbed.activate()
+ self.testbed.init_user_stub()
+ self.testbed.init_memcache_stub()
+ self.testbed.init_datastore_v3_stub()
+
+ def tearDown(self):
+ self.testbed.deactivate()
+
+ def testCreateLoginUrl(self):
+ _, mr = testing_helpers.GetRequestObjects(
+ path='/p/proj/issues/detail?id=123&q=term', project=self.project)
+ url = servlet_helpers.SafeCreateLoginURL(mr, '/continue')
+ self.assertIn('/continue', url)
+
+ def testCreateLogoutUrl(self):
+ _, mr = testing_helpers.GetRequestObjects(
+ path='/p/proj/issues/detail?id=123&q=term', project=self.project)
+ url = servlet_helpers.SafeCreateLogoutURL(mr)
+ self.assertIn('/Logout', url)
diff --git a/framework/test/servlet_test.py b/framework/test/servlet_test.py
index 40d5ed2..694e493 100644
--- a/framework/test/servlet_test.py
+++ b/framework/test/servlet_test.py
@@ -17,7 +17,7 @@
import webapp2
-from framework import framework_constants
+from framework import framework_constants, servlet_helpers
from framework import servlet
from framework import xsrf
from proto import project_pb2
@@ -227,23 +227,23 @@
project = fake.Project(
project_name='alerttest', state=project_pb2.ProjectState.LIVE)
- project_alert = servlet._CalcProjectAlert(project)
+ project_alert = servlet_helpers.CalcProjectAlert(project)
self.assertEqual(project_alert, None)
project.state = project_pb2.ProjectState.ARCHIVED
- project_alert = servlet._CalcProjectAlert(project)
+ project_alert = servlet_helpers.CalcProjectAlert(project)
self.assertEqual(
project_alert,
'Project is archived: read-only by members only.')
delete_time = int(time.time() + framework_constants.SECS_PER_DAY * 1.5)
project.delete_time = delete_time
- project_alert = servlet._CalcProjectAlert(project)
+ project_alert = servlet_helpers.CalcProjectAlert(project)
self.assertEqual(project_alert, 'Scheduled for deletion in 1 day.')
delete_time = int(time.time() + framework_constants.SECS_PER_DAY * 2.5)
project.delete_time = delete_time
- project_alert = servlet._CalcProjectAlert(project)
+ project_alert = servlet_helpers.CalcProjectAlert(project)
self.assertEqual(project_alert, 'Scheduled for deletion in 2 days.')
def testCheckForMovedProject_NoRedirect(self):
@@ -441,10 +441,10 @@
proj.access = project_pb2.ProjectAccess.ANYONE
proj.state = project_pb2.ProjectState.LIVE
- self.assertFalse(servlet._ProjectIsRestricted(mr))
+ self.assertFalse(servlet_helpers.ProjectIsRestricted(mr))
proj.state = project_pb2.ProjectState.ARCHIVED
- self.assertFalse(servlet._ProjectIsRestricted(mr))
+ self.assertFalse(servlet_helpers.ProjectIsRestricted(mr))
def testRestrictedProject(self):
proj = project_pb2.Project()
@@ -453,14 +453,15 @@
proj.state = project_pb2.ProjectState.LIVE
proj.access = project_pb2.ProjectAccess.MEMBERS_ONLY
- self.assertTrue(servlet._ProjectIsRestricted(mr))
+ self.assertTrue(servlet_helpers.ProjectIsRestricted(mr))
+
class VersionBaseTest(unittest.TestCase):
@mock.patch('settings.local_mode', True)
def testLocalhost(self):
request = webapp2.Request.blank('/', base_url='http://localhost:8080')
- actual = servlet._VersionBaseURL(request)
+ actual = servlet_helpers.VersionBaseURL(request)
expected = 'http://localhost:8080'
self.assertEqual(expected, actual)
@@ -469,6 +470,6 @@
def testProd(self, mock_gdvh):
mock_gdvh.return_value = 'monorail-prod.appspot.com'
request = webapp2.Request.blank('/', base_url='https://bugs.chromium.org')
- actual = servlet._VersionBaseURL(request)
+ actual = servlet_helpers.VersionBaseURL(request)
expected = 'https://test-dot-monorail-prod.appspot.com'
self.assertEqual(expected, actual)