Merge branch 'main' into avm99963-monorail
Merged commit cd4b3b336f1f14afa02990fdc2eec5d9467a827e
GitOrigin-RevId: e67bbf185d5538e1472bb42e0abb2a141f88bac1
diff --git a/features/alert2issue.py b/features/alert2issue.py
index fbaf5d9..daf72ca 100644
--- a/features/alert2issue.py
+++ b/features/alert2issue.py
@@ -10,7 +10,7 @@
import itertools
import logging
-import rfc822
+import email.utils
import settings
from businesslogic import work_env
@@ -227,7 +227,7 @@
owner_email = owner_email.strip()
if not owner_email:
return framework_constants.NO_USER_SPECIFIED
- emails = [addr for _, addr in rfc822.AddressList(owner_email)]
+ emails = [addr for _, addr in email.utils.getaddresses([owner_email])]
return user_svc.LookupExistingUserIDs(
cnxn, emails).get(owner_email) or framework_constants.NO_USER_SPECIFIED
@@ -237,7 +237,7 @@
cc_emails = cc_emails.strip()
if not cc_emails:
return []
- emails = [addr for _, addr in rfc822.AddressList(cc_emails)]
+ emails = [addr for _, addr in email.utils.getaddresses([cc_emails])]
return [userID for _, userID
in user_svc.LookupExistingUserIDs(cnxn, emails).iteritems()
if userID is not None]
diff --git a/features/autolink.py b/features/autolink.py
index 2787b9c..67c898a 100644
--- a/features/autolink.py
+++ b/features/autolink.py
@@ -33,8 +33,8 @@
import logging
import re
-import urllib
-import urlparse
+from six.moves import urllib
+from six.moves.urllib.parse import urlparse
import settings
from features import autolink_constants
diff --git a/features/banspammer.py b/features/banspammer.py
index 4b66251..a6be311 100644
--- a/features/banspammer.py
+++ b/features/banspammer.py
@@ -13,6 +13,7 @@
import time
from framework import cloud_tasks_helpers
+from framework import flaskservlet
from framework import framework_helpers
from framework import permissions
from framework import jsonfeed
@@ -56,7 +57,11 @@
mr, mr.viewed_user_auth.user_view.profile_url, include_project=False,
saved=1, ts=int(time.time()))
+ # def PostBanSpammerPage(self, **kwargs):
+ # return self.handler(**kwargs)
+
+# when convert to flask switch jsonfeed.FlaskInternalTask
class BanSpammerTask(jsonfeed.InternalTask):
"""This task will update all of the comments and issues created by the
target user with is_spam=True, and also add a manual verdict attached
@@ -91,7 +96,18 @@
self.services.issue, self.services.user, comment.id,
reporter_id, is_spammer)
+ # remove the self.response.body when convert to flask
self.response.body = json.dumps({
'comments': len(comments),
'issues': len(issues),
})
+ # return json.dumps({
+ # 'comments': len(comments),
+ # 'issues': len(issues),
+ # })
+
+ # def GetBanSpammer(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostBanSpammer(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/component_helpers.py b/features/component_helpers.py
deleted file mode 100644
index 1392f0b..0000000
--- a/features/component_helpers.py
+++ /dev/null
@@ -1,127 +0,0 @@
-# Copyright 2018 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-
-import json
-import logging
-import re
-
-import settings
-import cloudstorage
-
-from features import generate_dataset
-from framework import framework_helpers
-from services import ml_helpers
-from tracker import tracker_bizobj
-
-from googleapiclient import discovery
-from oauth2client.client import GoogleCredentials
-
-
-MODEL_NAME = 'projects/{}/models/{}'.format(
- settings.classifier_project_id, settings.component_model_name)
-
-
-def _GetTopWords(trainer_name): # pragma: no cover
- # TODO(carapew): Use memcache to get top words rather than storing as a
- # variable.
- credentials = GoogleCredentials.get_application_default()
- storage = discovery.build('storage', 'v1', credentials=credentials)
- request = storage.objects().get_media(
- bucket=settings.component_ml_bucket,
- object=trainer_name + '/topwords.txt')
- response = request.execute()
-
- # This turns the top words list into a dictionary for faster feature
- # generation.
- return {word: idx for idx, word in enumerate(response.split())}
-
-
-def _GetComponentsByIndex(trainer_name):
- # TODO(carapew): Memcache the index mapping file.
- mapping_path = '/%s/%s/component_index.json' % (
- settings.component_ml_bucket, trainer_name)
- logging.info('Mapping path full name: %r', mapping_path)
-
- with cloudstorage.open(mapping_path, 'r') as index_mapping_file:
- logging.info('Index component mapping opened')
- mapping = index_mapping_file.read()
- logging.info(mapping)
- return json.loads(mapping)
-
-
-@framework_helpers.retry(3)
-def _GetComponentPrediction(ml_engine, instance):
- """Predict the component from the default model based on the provided text.
-
- Args:
- ml_engine: An ML Engine instance for making predictions.
- instance: The dict object returned from ml_helpers.GenerateFeaturesRaw
- containing the features generated from the provided text.
-
- Returns:
- The index of the component with the highest score. ML engine's predict
- api returns a dict of the format
- {'predictions': [{'classes': ['0', '1', ...], 'scores': [.00234, ...]}]}
- where each class has a score at the same index. Classes are sequential,
- so the index of the highest score also happens to be the component's
- index.
- """
- body = {'instances': [{'inputs': instance['word_features']}]}
- request = ml_engine.projects().predict(name=MODEL_NAME, body=body)
- response = request.execute()
-
- logging.info('ML Engine API response: %r' % response)
- scores = response['predictions'][0]['scores']
-
- return scores.index(max(scores))
-
-
-def PredictComponent(raw_text, config):
- """Get the component ID predicted for the given text.
-
- Args:
- raw_text: The raw text for which we want to predict a component.
- config: The config of the project. Used to decide if the predicted component
- is valid.
-
- Returns:
- The component ID predicted for the provided component, or None if no
- component was predicted.
- """
- # Set-up ML engine.
- ml_engine = ml_helpers.setup_ml_engine()
-
- # Gets the timestamp number from the folder containing the model's trainer
- # in order to get the correct files for mappings and features.
- request = ml_engine.projects().models().get(name=MODEL_NAME)
- response = request.execute()
-
- version = re.search(r'v_(\d+)', response['defaultVersion']['name']).group(1)
- trainer_name = 'component_trainer_%s' % version
-
- top_words = _GetTopWords(trainer_name)
- components_by_index = _GetComponentsByIndex(trainer_name)
- logging.info('Length of top words list: %s', len(top_words))
-
- clean_text = generate_dataset.CleanText(raw_text)
- instance = ml_helpers.GenerateFeaturesRaw(
- [clean_text], settings.component_features, top_words)
-
- # Get the component id with the highest prediction score. Component ids are
- # stored in GCS as strings, but represented in the app as longs.
- best_score_index = _GetComponentPrediction(ml_engine, instance)
- component_id = components_by_index.get(str(best_score_index))
- if component_id:
- component_id = int(component_id)
-
- # The predicted component id might not exist.
- if tracker_bizobj.FindComponentDefByID(component_id, config) is None:
- return None
-
- return component_id
diff --git a/features/componentexport.py b/features/componentexport.py
deleted file mode 100644
index cadb6a8..0000000
--- a/features/componentexport.py
+++ /dev/null
@@ -1,59 +0,0 @@
-# Copyright 2018 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-""" Tasks and handlers for maintaining the spam classifier model. These
- should be run via cron and task queue rather than manually.
-"""
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-
-import cloudstorage
-import datetime
-import logging
-import webapp2
-
-from google.appengine.api import app_identity
-
-from features.generate_dataset import build_component_dataset
-from framework import cloud_tasks_helpers
-from framework import servlet
-from framework import urls
-
-
-class ComponentTrainingDataExport(webapp2.RequestHandler):
- """Trigger a training data export task"""
- def get(self):
- logging.info('Training data export requested.')
- task = {
- 'app_engine_http_request':
- {
- 'http_method': 'GET',
- 'relative_uri': urls.COMPONENT_DATA_EXPORT_TASK,
- }
- }
- cloud_tasks_helpers.create_task(task, queue='componentexport')
-
-
-class ComponentTrainingDataExportTask(servlet.Servlet):
- """Export training data for issues and their assigned components, to be used
- to train a model later.
- """
- def get(self):
- logging.info('Training data export initiated.')
- bucket_name = app_identity.get_default_gcs_bucket_name()
- logging.info('Bucket name: %s', bucket_name)
- date_str = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-
- logging.info('Opening cloud storage')
- gcs_file = cloudstorage.open('/' + bucket_name
- + '/component_training_data/'
- + date_str + '.csv',
- content_type='text/csv', mode='w')
-
- logging.info('GCS file opened')
-
- gcs_file = build_component_dataset(self.services.issue, gcs_file)
-
- gcs_file.close()
diff --git a/features/dateaction.py b/features/dateaction.py
index a525db1..169f582 100644
--- a/features/dateaction.py
+++ b/features/dateaction.py
@@ -39,6 +39,8 @@
TEMPLATE_PATH = framework_constants.TEMPLATE_PATH
+
+# TODO: change to FlaskInternalTask when convert to Flask
class DateActionCron(jsonfeed.InternalTask):
"""Find and process issues with date-type values that arrived today."""
@@ -84,6 +86,12 @@
urls.ISSUE_DATE_ACTION_TASK + '.do', params)
cloud_tasks_helpers.create_task(task)
+ # def GetDateActionCron(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostDateActionCron(self, **kwargs):
+ # return self.handler(**kwargs)
+
def _GetTimestampRange(now):
"""Return a (min, max) timestamp range for today."""
@@ -225,3 +233,9 @@
field, timestamp = ping
date_str = timestr.TimestampToDateWidgetStr(timestamp)
return 'The %s date has arrived: %s' % (field.field_name, date_str)
+
+ # def GetIssueDateActionTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostIssueDateActionTask(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/filterrules.py b/features/filterrules.py
index 3b1277e..724d7e2 100644
--- a/features/filterrules.py
+++ b/features/filterrules.py
@@ -15,6 +15,7 @@
from tracker import tracker_constants
+# TODO: change to FlaskInternalTask when convert to flask
class RecomputeDerivedFieldsTask(jsonfeed.InternalTask):
"""JSON servlet that recomputes derived fields on a batch of issues."""
@@ -35,7 +36,14 @@
'success': True,
}
+ # def GetRecomputeDerivedFieldsTask(self, **kwargs):
+ # return self.handler(**kwargs)
+ # def PostRecomputeDerivedFieldsTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+
+# TODO: change to FlaskInternalTask when convert to Flask
class ReindexQueueCron(jsonfeed.InternalTask):
"""JSON servlet that reindexes some issues each minute, as needed."""
@@ -48,3 +56,9 @@
return {
'num_reindexed': num_reindexed,
}
+
+ # def GetReindexQueueCron(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostReindexQueueCron(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/hotlistcreate.py b/features/hotlistcreate.py
index 448697b..fa8946f 100644
--- a/features/hotlistcreate.py
+++ b/features/hotlistcreate.py
@@ -12,16 +12,14 @@
import time
import re
-from features import features_constants
from features import hotlist_helpers
from framework import exceptions
+from framework import flaskservlet
from framework import framework_bizobj
from framework import framework_helpers
from framework import permissions
from framework import servlet
-from framework import urls
from services import features_svc
-from proto import api_pb2_v1
_MSG_HOTLIST_NAME_NOT_AVAIL = 'You already have a hotlist with that name.'
@@ -115,3 +113,9 @@
mr, hotlist_helpers.GetURLOfHotlist(
mr.cnxn, hotlist, self.services.user),
include_project=False)
+
+ # def GetCreateHotlist(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostCreateHotlist(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/hotlistdetails.py b/features/hotlistdetails.py
index d3bf3b2..f9c0435 100644
--- a/features/hotlistdetails.py
+++ b/features/hotlistdetails.py
@@ -14,6 +14,7 @@
from features import hotlist_helpers
from framework import framework_bizobj
+from framework import flaskservlet
from framework import framework_helpers
from framework import servlet
from framework import permissions
@@ -32,7 +33,7 @@
"""A page with hotlist details and editing options."""
_PAGE_TEMPLATE = 'features/hotlist-details-page.ezt'
- _MAIN_TAB_MODE = servlet.Servlet.HOTLIST_TAB_DETAILS
+ _MAIN_TAB_MODE = flaskservlet.FlaskServlet.HOTLIST_TAB_DETAILS
def AssertBasePermission(self, mr):
super(HotlistDetails, self).AssertBasePermission(mr)
@@ -121,3 +122,9 @@
if 'default_col_spec' in post_data:
default_col_spec = post_data['default_col_spec']
return summary, description, name, default_col_spec
+
+ # def GetHotlistDetailsPage(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostHotlistDetailsPage(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/hotlistissues.py b/features/hotlistissues.py
index 8743772..78ba007 100644
--- a/features/hotlistissues.py
+++ b/features/hotlistissues.py
@@ -20,6 +20,7 @@
from features import features_constants
from features import hotlist_helpers
from framework import exceptions
+from framework import flaskservlet
from framework import servlet
from framework import sorting
from framework import permissions
@@ -46,7 +47,7 @@
"""HotlistIssues is a page that shows the issues of one hotlist."""
_PAGE_TEMPLATE = 'features/hotlist-issues-page.ezt'
- _MAIN_TAB_MODE = servlet.Servlet.HOTLIST_TAB_ISSUES
+ _MAIN_TAB_MODE = flaskservlet.FlaskServlet.HOTLIST_TAB_ISSUES
def AssertBasePermission(self, mr):
"""Check that the user has permission to even visit this page."""
@@ -347,3 +348,9 @@
url_params=url_params)})
return grid_view_data
+
+ # def GetHotlistIssuesPage(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostHotlistIssuesPage(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/hotlistissuescsv.py b/features/hotlistissuescsv.py
index 3ae3f3b..2b35dad 100644
--- a/features/hotlistissuescsv.py
+++ b/features/hotlistissuescsv.py
@@ -60,3 +60,6 @@
page_data = hotlistissues.HotlistIssues.GatherPageData(self, mr)
return csv_helpers.ReformatRowsForCSV(
mr, page_data, '%d/csv' % mr.hotlist_id)
+
+ # def GetHotlistIssuesCsvPage(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/hotlistpeople.py b/features/hotlistpeople.py
index 1eb00ff..c574469 100644
--- a/features/hotlistpeople.py
+++ b/features/hotlistpeople.py
@@ -16,6 +16,7 @@
from features import hotlist_helpers
from features import hotlist_views
from framework import framework_helpers
+from framework import flaskservlet
from framework import framework_views
from framework import paginate
from framework import permissions
@@ -30,7 +31,7 @@
_PAGE_TEMPLATE = 'project/people-list-page.ezt'
# Note: using the project's peoplelist page template. minor edits were
# to make it compatible with HotlistPeopleList
- _MAIN_TAB_MODE = servlet.Servlet.HOTLIST_TAB_PEOPLE
+ _MAIN_TAB_MODE = flaskservlet.FlaskServlet.HOTLIST_TAB_PEOPLE
def AssertBasePermission(self, mr):
super(HotlistPeopleList, self).AssertBasePermission(mr)
@@ -218,6 +219,8 @@
def ProcessRemoveMembers(self, mr, post_data, hotlist_url):
"""Process the user's request to remove members."""
+ #TODO: convert for flask
+ #remove_strs = post_data.getlist('remove')
remove_strs = post_data.getall('remove')
logging.info('remove_strs = %r', remove_strs)
remove_ids = set(
@@ -250,3 +253,9 @@
mr, '%s%s' % (
hotlist_url, urls.HOTLIST_PEOPLE),
saved=1, ts=int(time.time()), include_project=False)
+
+ # def GetHotlistPeoplePage(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostHotlistPeoplePage(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/inboundemail.py b/features/inboundemail.py
index 6326dde..d9c36d3 100644
--- a/features/inboundemail.py
+++ b/features/inboundemail.py
@@ -13,17 +13,15 @@
import os
import re
import time
-import urllib
+from six.moves import urllib
import ezt
-from google.appengine.api import mail
from google.appengine.ext.webapp.mail_handlers import BounceNotificationHandler
import webapp2
import settings
-from businesslogic import work_env
from features import alert2issue
from features import commitlogcommands
from features import notify_helpers
@@ -36,7 +34,6 @@
from framework import sql
from framework import template_helpers
from proto import project_pb2
-from tracker import tracker_helpers
TEMPLATE_PATH_BASE = framework_constants.TEMPLATE_PATH
@@ -65,17 +62,25 @@
TEMPLATE_PATH_BASE + template_path,
compress_whitespace=False, base_format=ezt.FORMAT_RAW)
+ # def HandleInboundEmail(self, project_addr=None):
+ # if self.request.method == 'POST':
+ # self.post(project_addr)
+ # elif self.request.method == 'GET':
+ # self.get(project_addr)
+
def get(self, project_addr=None):
logging.info('\n\n\nGET for InboundEmail and project_addr is %r',
project_addr)
- self.Handler(mail.InboundEmailMessage(self.request.body),
- urllib.unquote(project_addr))
+ self.Handler(
+ mail.InboundEmailMessage(self.request.body),
+ urllib.parse.unquote(project_addr))
def post(self, project_addr=None):
logging.info('\n\n\nPOST for InboundEmail and project_addr is %r',
project_addr)
- self.Handler(mail.InboundEmailMessage(self.request.body),
- urllib.unquote(project_addr))
+ self.Handler(
+ mail.InboundEmailMessage(self.request.body),
+ urllib.parse.unquote(project_addr))
def Handler(self, inbound_email_message, project_addr):
"""Process an inbound email message."""
@@ -287,6 +292,7 @@
BAD_WRAP_RE = re.compile('=\r\n')
BAD_EQ_RE = re.compile('=3D')
+
class BouncedEmail(BounceNotificationHandler):
"""Handler to notice when email to given user is bouncing."""
diff --git a/features/notify.py b/features/notify.py
index c285c76..425041e 100644
--- a/features/notify.py
+++ b/features/notify.py
@@ -219,6 +219,12 @@
return email_tasks
+ # def GetNotifyIssueChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostNotifyIssueChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
class NotifyBlockingChangeTask(notify_helpers.NotifyTaskBase):
"""JSON servlet that notifies appropriate users after a blocking change."""
@@ -350,6 +356,12 @@
return one_issue_email_tasks
+ # def GetNotifyBlockingChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostNotifyBlockingChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
class NotifyBulkChangeTask(notify_helpers.NotifyTaskBase):
"""JSON servlet that notifies appropriate users after a bulk edit."""
@@ -712,6 +724,12 @@
return subject, body
+ # def GetNotifyBulkChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostNotifyBulkChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
# For now, this class will not be used to send approval comment notifications
# TODO(jojwang): monorail:3588, it might make sense for this class to handle
@@ -901,6 +919,12 @@
return list(set(recipient_ids))
+ # def GetNotifyApprovalChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostNotifyApprovalChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
class NotifyRulesDeletedTask(notify_helpers.NotifyTaskBase):
"""JSON servlet that sends one email."""
@@ -967,7 +991,14 @@
return email_tasks
+ # def GetNotifyRulesDeletedTask(self, **kwargs):
+ # return self.handler(**kwargs)
+ # def PostNotifyRulesDeletedTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+
+# TODO: change to FlaskInternalTask when convert to flask
class OutboundEmailTask(jsonfeed.InternalTask):
"""JSON servlet that sends one email.
@@ -1053,3 +1084,9 @@
return dict(
sender=sender, to=to, subject=subject, body=body, html_body=html_body,
reply_to=reply_to, references=references)
+
+ # def GetOutboundEmailTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostOutboundEmailTask(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/notify_helpers.py b/features/notify_helpers.py
index f22ed38..5f77307 100644
--- a/features/notify_helpers.py
+++ b/features/notify_helpers.py
@@ -123,6 +123,7 @@
return notified
+# TODO: change to FlaskInternalTask when convert to flask
class NotifyTaskBase(jsonfeed.InternalTask):
"""Abstract base class for notification task handler."""
diff --git a/features/pubsub.py b/features/pubsub.py
index a74ff22..86bd3ba 100644
--- a/features/pubsub.py
+++ b/features/pubsub.py
@@ -26,6 +26,7 @@
from framework import jsonfeed
+# TODO: change to FlaskInternalTask when convert to flask
class PublishPubsubIssueChangeTask(jsonfeed.InternalTask):
"""JSON servlet that pushes issue update messages onto a pub/sub topic."""
@@ -70,6 +71,12 @@
return {}
+ # def GetPublishPubsubIssueChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostPublishPubsubIssueChangeTask(self, **kwargs):
+ # return self.handler(**kwargs)
+
def set_up_pubsub_api():
"""Attempts to build and return a pub/sub API client."""
diff --git a/features/rerankhotlist.py b/features/rerankhotlist.py
index fe235db..74365f6 100644
--- a/features/rerankhotlist.py
+++ b/features/rerankhotlist.py
@@ -19,6 +19,7 @@
from tracker import rerank_helpers
+# TODO: convert to FLaskJsonFeed while conver to flask
class RerankHotlistIssue(jsonfeed.JsonFeed):
"""Rerank an issue in a hotlist."""
@@ -134,3 +135,9 @@
lower, higher = features_bizobj.SplitHotlistIssueRanks(
mr.target_id, mr.split_above, untouched_items)
return rerank_helpers.GetInsertRankings(lower, higher, mr.moved_ids)
+
+ # def GetRerankHotlistIssuePage(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostRerankHotlistIssuePage(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/savedqueries.py b/features/savedqueries.py
index 5cc1bc8..fb99fcf 100644
--- a/features/savedqueries.py
+++ b/features/savedqueries.py
@@ -15,6 +15,7 @@
from features import savedqueries_helpers
from framework import framework_helpers
+from framework import flaskservlet
from framework import permissions
from framework import servlet
from framework import urls
@@ -74,3 +75,9 @@
return framework_helpers.FormatAbsoluteURL(
mr, '/u/%s%s' % (mr.viewed_username, urls.SAVED_QUERIES),
include_project=False, saved=1, ts=int(time.time()))
+
+ # def GetSavedQueriesPage(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostSavedQueriesPage(self, **kwargs):
+ # return self.handler(**kwargs)
diff --git a/features/spammodel.py b/features/spammodel.py
deleted file mode 100644
index dc5e715..0000000
--- a/features/spammodel.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# Copyright 2016 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-""" Tasks and handlers for maintaining the spam classifier model. These
- should be run via cron and task queue rather than manually.
-"""
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-
-import csv
-import logging
-import webapp2
-import cloudstorage
-import json
-
-from datetime import date
-from datetime import datetime
-from datetime import timedelta
-from google.appengine.api import app_identity
-
-from framework import cloud_tasks_helpers
-from framework import gcs_helpers
-from framework import servlet
-from framework import urls
-
-class TrainingDataExport(webapp2.RequestHandler):
- """Trigger a training data export task"""
- def get(self):
- task = cloud_tasks_helpers.generate_simple_task(
- urls.SPAM_DATA_EXPORT_TASK + '.do', {})
- cloud_tasks_helpers.create_task(task)
-
-
-BATCH_SIZE = 1000
-
-class TrainingDataExportTask(servlet.Servlet):
- """Export any human-labeled ham or spam from the previous day. These
- records will be used by a subsequent task to create an updated model.
- """
- CHECK_SECURITY_TOKEN = False
-
- def ProcessFormData(self, mr, post_data):
- logging.info("Training data export initiated.")
-
- bucket_name = app_identity.get_default_gcs_bucket_name()
- date_str = date.today().isoformat()
- export_target_path = '/' + bucket_name + '/spam_training_data/' + date_str
- total_issues = 0
-
- with cloudstorage.open(export_target_path, mode='w',
- content_type=None, options=None, retry_params=None) as gcs_file:
-
- csv_writer = csv.writer(gcs_file, delimiter=',', quotechar='"',
- quoting=csv.QUOTE_ALL, lineterminator='\n')
-
- since = datetime.now() - timedelta(days=7)
-
- # TODO: Further pagination.
- issues, first_comments, _count = (
- self.services.spam.GetTrainingIssues(
- mr.cnxn, self.services.issue, since, offset=0, limit=BATCH_SIZE))
- total_issues += len(issues)
- for issue in issues:
- # Cloud Prediction API doesn't allow newlines in the training data.
- fixed_summary = issue.summary.replace('\r\n', ' ')
- fixed_comment = first_comments[issue.issue_id].replace('\r\n', ' ')
- email = self.services.user.LookupUserEmail(mr.cnxn, issue.reporter_id)
- csv_writer.writerow([
- 'spam' if issue.is_spam else 'ham',
- fixed_summary.encode('utf-8'), fixed_comment.encode('utf-8'), email,
- ])
-
- comments = (
- self.services.spam.GetTrainingComments(
- mr.cnxn, self.services.issue, since, offset=0, limit=BATCH_SIZE))
- total_comments = len(comments)
- for comment in comments:
- # Cloud Prediction API doesn't allow newlines in the training data.
- fixed_comment = comment.content.replace('\r\n', ' ')
- email = self.services.user.LookupUserEmail(mr.cnxn, comment.user_id)
- csv_writer.writerow([
- 'spam' if comment.is_spam else 'ham',
- # Comments don't have summaries, so it's blank:
- '', fixed_comment.encode('utf-8'), email
- ])
-
- self.response.body = json.dumps({
- "exported_issue_count": total_issues,
- "exported_comment_count": total_comments,
- })
diff --git a/features/spamtraining.py b/features/spamtraining.py
deleted file mode 100644
index 625fa53..0000000
--- a/features/spamtraining.py
+++ /dev/null
@@ -1,63 +0,0 @@
-"""Cron job to train spam model with all spam data."""
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-
-import logging
-import settings
-import time
-
-from googleapiclient import discovery
-from googleapiclient import errors
-from google.appengine.api import app_identity
-from oauth2client.client import GoogleCredentials
-import webapp2
-
-class TrainSpamModelCron(webapp2.RequestHandler):
-
- """Submit a job to ML Engine which uploads a spam classification model by
- training on an already packaged trainer.
- """
- def get(self):
-
- credentials = GoogleCredentials.get_application_default()
- ml = discovery.build('ml', 'v1', credentials=credentials)
-
- app_id = app_identity.get_application_id()
- project_id = 'projects/%s' % (app_id)
- job_id = 'spam_trainer_%d' % time.time()
- training_input = {
- 'scaleTier': 'BASIC',
- 'packageUris': [
- settings.trainer_staging
- if app_id == "monorail-staging" else
- settings.trainer_prod
- ],
- 'pythonModule': 'trainer.task',
- 'args': [
- '--train-steps',
- '1000',
- '--verbosity',
- 'DEBUG',
- '--gcs-bucket',
- 'monorail-prod.appspot.com',
- '--gcs-prefix',
- 'spam_training_data',
- '--trainer-type',
- 'spam'
- ],
- 'region': 'us-central1',
- 'jobDir': 'gs://%s-mlengine/%s' % (app_id, job_id),
- 'runtimeVersion': '1.2'
- }
- job_info = {
- 'jobId': job_id,
- 'trainingInput': training_input
- }
- request = ml.projects().jobs().create(parent=project_id, body=job_info)
-
- try:
- response = request.execute()
- logging.info(response)
- except errors.HttpError, err:
- logging.error(err._get_reason())
diff --git a/features/test/banspammer_test.py b/features/test/banspammer_test.py
index e6fceff..edf7aba 100644
--- a/features/test/banspammer_test.py
+++ b/features/test/banspammer_test.py
@@ -12,7 +12,7 @@
import mock
import os
import unittest
-import urllib
+from six.moves import urllib
import webapp2
import settings
@@ -74,7 +74,7 @@
'app_engine_http_request':
{
'relative_uri': urls.BAN_SPAMMER_TASK + '.do',
- 'body': urllib.urlencode(params),
+ 'body': urllib.parse.urlencode(params),
'headers': {
'Content-type': 'application/x-www-form-urlencoded'
}
diff --git a/features/test/component_helpers_test.py b/features/test/component_helpers_test.py
deleted file mode 100644
index aa6c761..0000000
--- a/features/test/component_helpers_test.py
+++ /dev/null
@@ -1,145 +0,0 @@
-# Copyright 2018 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file or at
-# https://developers.google.com/open-source/licenses/bsd
-
-"""Unit tests for component prediction endpoints."""
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-
-import json
-import mock
-import sys
-import unittest
-
-from services import service_manager
-from testing import fake
-
-# Mock cloudstorage before it's imported by component_helpers
-sys.modules['cloudstorage'] = mock.Mock()
-from features import component_helpers
-
-
-class FakeMLEngine(object):
- def __init__(self, test):
- self.test = test
- self.expected_features = None
- self.scores = None
- self._execute_response = None
-
- def projects(self):
- return self
-
- def models(self):
- return self
-
- def predict(self, name, body):
- self.test.assertEqual(component_helpers.MODEL_NAME, name)
- self.test.assertEqual(
- {'instances': [{'inputs': self.expected_features}]}, body)
- self._execute_response = {'predictions': [{'scores': self.scores}]}
- return self
-
- def get(self, name):
- self.test.assertEqual(component_helpers.MODEL_NAME, name)
- self._execute_response = {'defaultVersion': {'name': 'v_1234'}}
- return self
-
- def execute(self):
- response = self._execute_response
- self._execute_response = None
- return response
-
-
-class ComponentHelpersTest(unittest.TestCase):
-
- def setUp(self):
- self.services = service_manager.Services(
- config=fake.ConfigService(),
- user=fake.UserService())
- self.project = fake.Project(project_name='proj')
-
- self._ml_engine = FakeMLEngine(self)
- self._top_words = None
- self._components_by_index = None
-
- mock.patch(
- 'services.ml_helpers.setup_ml_engine', lambda: self._ml_engine).start()
- mock.patch(
- 'features.component_helpers._GetTopWords',
- lambda _: self._top_words).start()
- mock.patch('cloudstorage.open', self.cloudstorageOpen).start()
- mock.patch('settings.component_features', 5).start()
-
- self.addCleanup(mock.patch.stopall)
-
- def cloudstorageOpen(self, name, mode):
- """Create a file mock that returns self._components_by_index when read."""
- open_fn = mock.mock_open(read_data=json.dumps(self._components_by_index))
- return open_fn(name, mode)
-
- def testPredict_Normal(self):
- """Test normal case when predicted component exists."""
- component_id = self.services.config.CreateComponentDef(
- cnxn=None, project_id=self.project.project_id, path='Ruta>Baga',
- docstring='', deprecated=False, admin_ids=[], cc_ids=[], created=None,
- creator_id=None, label_ids=[])
- config = self.services.config.GetProjectConfig(
- None, self.project.project_id)
-
- self._top_words = {
- 'foo': 0,
- 'bar': 1,
- 'baz': 2}
- self._components_by_index = {
- '0': '123',
- '1': str(component_id),
- '2': '789'}
- self._ml_engine.expected_features = [3, 0, 1, 0, 0]
- self._ml_engine.scores = [5, 10, 3]
-
- text = 'foo baz foo foo'
-
- self.assertEqual(
- component_id, component_helpers.PredictComponent(text, config))
-
- def testPredict_UnknownComponentIndex(self):
- """Test case where the prediction is not in components_by_index."""
- config = self.services.config.GetProjectConfig(
- None, self.project.project_id)
-
- self._top_words = {
- 'foo': 0,
- 'bar': 1,
- 'baz': 2}
- self._components_by_index = {
- '0': '123',
- '1': '456',
- '2': '789'}
- self._ml_engine.expected_features = [3, 0, 1, 0, 0]
- self._ml_engine.scores = [5, 10, 3, 1000]
-
- text = 'foo baz foo foo'
-
- self.assertIsNone(component_helpers.PredictComponent(text, config))
-
- def testPredict_InvalidComponentIndex(self):
- """Test case where the prediction is not a valid component id."""
- config = self.services.config.GetProjectConfig(
- None, self.project.project_id)
-
- self._top_words = {
- 'foo': 0,
- 'bar': 1,
- 'baz': 2}
- self._components_by_index = {
- '0': '123',
- '1': '456',
- '2': '789'}
- self._ml_engine.expected_features = [3, 0, 1, 0, 0]
- self._ml_engine.scores = [5, 10, 3]
-
- text = 'foo baz foo foo'
-
- self.assertIsNone(component_helpers.PredictComponent(text, config))
diff --git a/features/test/componentexport_test.py b/features/test/componentexport_test.py
deleted file mode 100644
index 0e5fbf8..0000000
--- a/features/test/componentexport_test.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2020 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-"""Tests for the componentexport module."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import mock
-import unittest
-import webapp2
-
-import settings
-from features import componentexport
-from framework import urls
-
-
-class ComponentTrainingDataExportTest(unittest.TestCase):
-
- def test_handler_definition(self):
- instance = componentexport.ComponentTrainingDataExport()
- self.assertIsInstance(instance, webapp2.RequestHandler)
-
- @mock.patch('framework.cloud_tasks_helpers._get_client')
- def test_enqueues_task(self, get_client_mock):
- componentexport.ComponentTrainingDataExport().get()
-
- queue = 'componentexport'
- task = {
- 'app_engine_http_request':
- {
- 'http_method': 'GET',
- 'relative_uri': urls.COMPONENT_DATA_EXPORT_TASK
- }
- }
-
- get_client_mock().queue_path.assert_called_with(
- settings.app_id, settings.CLOUD_TASKS_REGION, queue)
- get_client_mock().create_task.assert_called_once()
- ((_parent, called_task), _kwargs) = get_client_mock().create_task.call_args
- self.assertEqual(called_task, task)
diff --git a/features/test/filterrules_helpers_test.py b/features/test/filterrules_helpers_test.py
index 99d22b7..a68c279 100644
--- a/features/test/filterrules_helpers_test.py
+++ b/features/test/filterrules_helpers_test.py
@@ -10,8 +10,8 @@
import mock
import unittest
-import urllib
-import urlparse
+from six.moves import urllib
+from six.moves.urllib.parse import parse_qs
import settings
from features import filterrules_helpers
@@ -139,7 +139,7 @@
'app_engine_http_request':
{
'relative_uri': urls.RECOMPUTE_DERIVED_FIELDS_TASK + '.do',
- 'body': urllib.urlencode(params),
+ 'body': urllib.parse.urlencode(params),
'headers':
{
'Content-type': 'application/x-www-form-urlencoded'
@@ -177,7 +177,7 @@
'relative_uri')
self.assertEqual(relative_uri, urls.RECOMPUTE_DERIVED_FIELDS_TASK + '.do')
encoded_params = called_task.get('app_engine_http_request').get('body')
- params = {k: v[0] for k, v in urlparse.parse_qs(encoded_params).items()}
+ params = {k: v[0] for k, v in parse_qs(encoded_params).items()}
self.assertEqual(params['project_id'], str(self.project.project_id))
self.assertEqual(
params['lower_bound'], str(12345 // self.BLOCK * self.BLOCK + 1))
@@ -188,7 +188,7 @@
'relative_uri')
self.assertEqual(relative_uri, urls.RECOMPUTE_DERIVED_FIELDS_TASK + '.do')
encoded_params = called_task.get('app_engine_http_request').get('body')
- params = {k: v[0] for k, v in urlparse.parse_qs(encoded_params).items()}
+ params = {k: v[0] for k, v in parse_qs(encoded_params).items()}
self.assertEqual(params['project_id'], str(self.project.project_id))
self.assertEqual(params['lower_bound'], str(1))
self.assertEqual(params['upper_bound'], str(self.BLOCK + 1))
diff --git a/features/test/inboundemail_test.py b/features/test/inboundemail_test.py
index 6c13827..0eaa281 100644
--- a/features/test/inboundemail_test.py
+++ b/features/test/inboundemail_test.py
@@ -15,6 +15,7 @@
import mox
import time
+from google.appengine.api import mail
from google.appengine.ext.webapp.mail_handlers import BounceNotificationHandler
import settings
@@ -36,7 +37,6 @@
class InboundEmailTest(unittest.TestCase):
-
def setUp(self):
self.cnxn = 'fake cnxn'
self.services = service_manager.Services(
@@ -358,28 +358,6 @@
self.mox.UnsetStubs()
self.mox.ResetAll()
- def testPost_Normal(self):
- """Normally, our post() just calls BounceNotificationHandler post()."""
- self.mox.StubOutWithMock(BounceNotificationHandler, 'post')
- BounceNotificationHandler.post()
- self.mox.ReplayAll()
-
- self.servlet.post()
- self.mox.VerifyAll()
-
- def testPost_Exception(self):
- """Our post() method works around an escaping bug."""
- self.servlet.request = webapp2.Request.blank(
- '/', POST={'raw-message': 'this is an email message'})
-
- self.mox.StubOutWithMock(BounceNotificationHandler, 'post')
- BounceNotificationHandler.post().AndRaise(AttributeError())
- BounceNotificationHandler.post()
- self.mox.ReplayAll()
-
- self.servlet.post()
- self.mox.VerifyAll()
-
def testReceive_Normal(self):
"""Find the user that bounced and set email_bounce_timestamp."""
self.assertEqual(0, self.user.email_bounce_timestamp)
diff --git a/features/test/notify_test.py b/features/test/notify_test.py
index 00de106..9ddcce7 100644
--- a/features/test/notify_test.py
+++ b/features/test/notify_test.py
@@ -26,8 +26,6 @@
from tracker import attachment_helpers
from tracker import tracker_bizobj
-from third_party import cloudstorage
-
def MakeTestIssue(project_id, local_id, owner_id, reporter_id, is_spam=False):
issue = tracker_pb2.Issue()
@@ -62,8 +60,6 @@
project_id=12345, local_id=2, owner_id=2, reporter_id=1)
self.services.issue.TestAddIssue(self.issue1)
- self._old_gcs_open = cloudstorage.open
- cloudstorage.open = fake.gcs_open
self.orig_sign_attachment_id = attachment_helpers.SignAttachmentID
attachment_helpers.SignAttachmentID = (
lambda aid: 'signed_%d' % aid)
@@ -74,7 +70,6 @@
self.testbed.init_datastore_v3_stub()
def tearDown(self):
- cloudstorage.open = self._old_gcs_open
attachment_helpers.SignAttachmentID = self.orig_sign_attachment_id
def get_filtered_task_call_args(self, create_task_mock, relative_uri):
diff --git a/features/test/send_notifications_test.py b/features/test/send_notifications_test.py
index 435a67d..b15fb23 100644
--- a/features/test/send_notifications_test.py
+++ b/features/test/send_notifications_test.py
@@ -10,7 +10,7 @@
import mock
import unittest
-import urlparse
+from six.moves.urllib.parse import parse_qs
from features import send_notifications
from framework import urls
@@ -31,9 +31,7 @@
(args, _kwargs) = call
path = args[0]['app_engine_http_request']['relative_uri']
encoded_params = args[0]['app_engine_http_request']['body']
- params = {
- k: v[0] for k, v in urlparse.parse_qs(encoded_params, True).items()
- }
+ params = {k: v[0] for k, v in parse_qs(encoded_params, True).items()}
return path, params
@mock.patch('framework.cloud_tasks_helpers.create_task')
diff --git a/features/test/spammodel_test.py b/features/test/spammodel_test.py
deleted file mode 100644
index 3e99c8f..0000000
--- a/features/test/spammodel_test.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright 2020 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-"""Tests for the spammodel module."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import mock
-import unittest
-import webapp2
-
-from features import spammodel
-from framework import urls
-
-
-class TrainingDataExportTest(unittest.TestCase):
-
- def test_handler_definition(self):
- instance = spammodel.TrainingDataExport()
- self.assertIsInstance(instance, webapp2.RequestHandler)
-
- @mock.patch('framework.cloud_tasks_helpers._get_client')
- def test_enqueues_task(self, get_client_mock):
- spammodel.TrainingDataExport().get()
- task = {
- 'app_engine_http_request':
- {
- 'relative_uri': urls.SPAM_DATA_EXPORT_TASK + '.do',
- 'body': '',
- 'headers': {
- 'Content-type': 'application/x-www-form-urlencoded'
- }
- }
- }
- get_client_mock().create_task.assert_called_once()
- ((_parent, called_task), _kwargs) = get_client_mock().create_task.call_args
- self.assertEqual(called_task, task)
diff --git a/features/userhotlists.py b/features/userhotlists.py
index 330ab73..65e2d9d 100644
--- a/features/userhotlists.py
+++ b/features/userhotlists.py
@@ -14,6 +14,7 @@
from features import hotlist_views
from framework import framework_views
from framework import servlet
+from framework import flaskservlet
class UserHotlists(servlet.Servlet):
@@ -81,3 +82,9 @@
help_data = super(UserHotlists, self).GatherHelpData(mr, page_data)
help_data['cue'] = 'explain_hotlist_starring'
return help_data
+
+ # def GetUserHotlistsPage(self, **kwargs):
+ # return self.handler(**kwargs)
+
+ # def PostUserHotlistsPage(self, **kwargs):
+ # return self.handler(**kwargs)