Merge branch 'main' into avm99963-monorail
Merged commit 3779da353b36d43cf778e7d4f468097714dd4540
GitOrigin-RevId: 6451a5c6b75afb0fd1f37b3f14521148d0722ea8
diff --git a/services/cachemanager_svc.py b/services/cachemanager_svc.py
index 02ad6dd..753bffa 100644
--- a/services/cachemanager_svc.py
+++ b/services/cachemanager_svc.py
@@ -44,6 +44,7 @@
import logging
from framework import jsonfeed
+from framework import logger
from framework import sql
@@ -119,19 +120,26 @@
def StoreInvalidateRows(self, cnxn, kind, keys):
"""Store rows to let all jobs know to invalidate the given keys."""
assert kind in INVALIDATE_KIND_VALUES
+ logger.log(
+ {
+ 'log_type': 'cache/invalidate/rows',
+ 'kind': kind,
+ 'count': len(keys),
+ 'keys': str(keys),
+ })
self.invalidate_tbl.InsertRows(
cnxn, ['kind', 'cache_key'], [(kind, key) for key in keys])
def StoreInvalidateAll(self, cnxn, kind):
"""Store a value to tell all jobs to invalidate all items of this kind."""
+ logger.log({'log_type': 'cache/invalidate/all', 'kind': kind})
last_timestep = self.invalidate_tbl.InsertRow(
cnxn, kind=kind, cache_key=INVALIDATE_ALL_KEYS)
self.invalidate_tbl.Delete(
cnxn, kind=kind, where=[('timestep < %s', [last_timestep])])
-# TODO: change to FlaskInternalTask when convert to Flask
-class RamCacheConsolidate(jsonfeed.InternalTask):
+class RamCacheConsolidate(jsonfeed.FlaskInternalTask):
"""Drop old Invalidate rows when there are too many of them."""
def HandleRequest(self, mr):
@@ -166,8 +174,5 @@
'new_count': new_count,
}
- # def GetRamCacheConsolidate(self, **kwargs):
- # return self.handler(**kwargs)
-
- # def PostRamCacheConsolidate(self, **kwargs):
- # return self.handler(**kwargs)
+ def GetRamCacheConsolidate(self, **kwargs):
+ return self.handler(**kwargs)
diff --git a/services/caches.py b/services/caches.py
index 35276a0..8869d61 100644
--- a/services/caches.py
+++ b/services/caches.py
@@ -27,6 +27,7 @@
import settings
from framework import framework_constants
+from framework import logger
DEFAULT_MAX_SIZE = 10000
@@ -253,6 +254,16 @@
self._WriteToMemcache(retrieved_dict)
still_missing_keys = [key for key in keys if key not in result_dict]
+ if still_missing_keys:
+ # The keys were not found in the caches or the DB.
+ logger.log(
+ {
+ 'log_type': 'database/missing_keys',
+ 'kind': self.cache.kind,
+ 'prefix': self.prefix,
+ 'count': len(still_missing_keys),
+ 'keys': str(still_missing_keys)
+ })
return result_dict, still_missing_keys
def LocalInvalidateAll(self):
@@ -329,6 +340,14 @@
def _DeleteFromMemcache(self, keys):
# type: (Sequence[str]) -> None
"""Delete key-values from memcache. """
+ logger.log(
+ {
+ 'log_type': 'cache/memcache/delete',
+ 'kind': self.cache.kind,
+ 'prefix': self.prefix,
+ 'count': len(keys),
+ 'keys': str(keys)
+ })
memcache.delete_multi(
[self._KeyToStr(key) for key in keys],
seconds=5,
diff --git a/services/client_config_svc.py b/services/client_config_svc.py
index ce85a95..d5d6a25 100644
--- a/services/client_config_svc.py
+++ b/services/client_config_svc.py
@@ -14,6 +14,7 @@
import time
from six.moves import urllib
import webapp2
+import flask
from google.appengine.api import app_identity
from google.appengine.api import urlfetch
@@ -45,108 +46,77 @@
configs = db.TextProperty()
-# Note: The cron job must have hit the servlet before this will work.
-# when convert to flask replace the webapp2.RequestHandler to Object
-class LoadApiClientConfigs(webapp2.RequestHandler):
+_CONFIG_LOADS = ts_mon.CounterMetric(
+ 'monorail/client_config_svc/loads', 'Results of fetches from luci-config.',
+ [ts_mon.BooleanField('success'),
+ ts_mon.StringField('type')])
- config_loads = ts_mon.CounterMetric(
- 'monorail/client_config_svc/loads',
- 'Results of fetches from luci-config.',
- [ts_mon.BooleanField('success'), ts_mon.StringField('type')])
- def get(self):
- global service_account_map
- global qpm_dict
- authorization_token, _ = app_identity.get_access_token(
+def _process_response(response):
+ try:
+ content = json.loads(response.content)
+ except ValueError:
+ logging.error('Response was not JSON: %r', response.content)
+ _CONFIG_LOADS.increment({'success': False, 'type': 'json-load-error'})
+ raise
+
+ try:
+ config_content = content['content']
+ except KeyError:
+ logging.error('JSON contained no content: %r', content)
+ _CONFIG_LOADS.increment({'success': False, 'type': 'json-key-error'})
+ raise
+
+ try:
+ content_text = base64.b64decode(config_content)
+ except TypeError:
+ logging.error('Content was not b64: %r', config_content)
+ _CONFIG_LOADS.increment({'success': False, 'type': 'b64-decode-error'})
+ raise
+
+ try:
+ cfg = api_clients_config_pb2.ClientCfg()
+ text_format.Merge(content_text, cfg)
+ except:
+ logging.error('Content was not a valid ClientCfg proto: %r', content_text)
+ _CONFIG_LOADS.increment({'success': False, 'type': 'proto-load-error'})
+ raise
+
+ return content_text
+
+
+def GetLoadApiClientConfigs():
+ global service_account_map
+ global qpm_dict
+ authorization_token, _ = app_identity.get_access_token(
framework_constants.OAUTH_SCOPE)
- response = urlfetch.fetch(
+ response = urlfetch.fetch(
LUCI_CONFIG_URL,
method=urlfetch.GET,
follow_redirects=False,
- headers={'Content-Type': 'application/json; charset=UTF-8',
- 'Authorization': 'Bearer ' + authorization_token})
+ headers={
+ 'Content-Type': 'application/json; charset=UTF-8',
+ 'Authorization': 'Bearer ' + authorization_token
+ })
- if response.status_code != 200:
- logging.error('Invalid response from luci-config: %r', response)
- self.config_loads.increment({'success': False, 'type': 'luci-cfg-error'})
- self.abort(500, 'Invalid response from luci-config')
+ if response.status_code != 200:
+ logging.error('Invalid response from luci-config: %r', response)
+ _CONFIG_LOADS.increment({'success': False, 'type': 'luci-cfg-error'})
+ flask.abort(500, 'Invalid response from luci-config')
- try:
- content_text = self._process_response(response)
- except Exception as e:
- self.abort(500, str(e))
+ try:
+ content_text = _process_response(response)
+ except Exception as e:
+ flask.abort(500, str(e))
- logging.info('luci-config content decoded: %r.', content_text)
- configs = ClientConfig(configs=content_text,
- key_name='api_client_configs')
- configs.put()
- service_account_map = None
- qpm_dict = None
- self.config_loads.increment({'success': True, 'type': 'success'})
+ logging.info('luci-config content decoded: %r.', content_text)
+ configs = ClientConfig(configs=content_text, key_name='api_client_configs')
+ configs.put()
+ service_account_map = None
+ qpm_dict = None
+ _CONFIG_LOADS.increment({'success': True, 'type': 'success'})
- def _process_response(self, response):
- try:
- content = json.loads(response.content)
- except ValueError:
- logging.error('Response was not JSON: %r', response.content)
- self.config_loads.increment({'success': False, 'type': 'json-load-error'})
- raise
-
- try:
- config_content = content['content']
- except KeyError:
- logging.error('JSON contained no content: %r', content)
- self.config_loads.increment({'success': False, 'type': 'json-key-error'})
- raise
-
- try:
- content_text = base64.b64decode(config_content)
- except TypeError:
- logging.error('Content was not b64: %r', config_content)
- self.config_loads.increment({'success': False,
- 'type': 'b64-decode-error'})
- raise
-
- try:
- cfg = api_clients_config_pb2.ClientCfg()
- text_format.Merge(content_text, cfg)
- except:
- logging.error('Content was not a valid ClientCfg proto: %r', content_text)
- self.config_loads.increment({'success': False,
- 'type': 'proto-load-error'})
- raise
-
- return content_text
-
- # def GetLoadApiClientConfigs(self):
- # global service_account_map
- # global qpm_dict
- # authorization_token, _ = app_identity.get_access_token(
- # framework_constants.OAUTH_SCOPE)
- # response = urlfetch.fetch(
- # LUCI_CONFIG_URL,
- # method=urlfetch.GET,
- # follow_redirects=False,
- # headers={'Content-Type': 'application/json; charset=UTF-8',
- # 'Authorization': 'Bearer ' + authorization_token})
-
- # if response.status_code != 200:
- # logging.error('Invalid response from luci-config: %r', response)
- # self.config_loads.increment({'success': False, 'type': 'luci-cfg-error'})
- # flask.abort(500, 'Invalid response from luci-config')
-
- # try:
- # content_text = self._process_response(response)
- # except Exception as e:
- # flask.abort(500, str(e))
-
- # logging.info('luci-config content decoded: %r.', content_text)
- # configs = ClientConfig(configs=content_text,
- # key_name='api_client_configs')
- # configs.put()
- # service_account_map = None
- # qpm_dict = None
- # self.config_loads.increment({'success': True, 'type': 'success'})
+ return ''
class ClientConfigService(object):
diff --git a/services/test/cachemanager_svc_test.py b/services/test/cachemanager_svc_test.py
index 20956e0..b84d33e 100644
--- a/services/test/cachemanager_svc_test.py
+++ b/services/test/cachemanager_svc_test.py
@@ -10,7 +10,10 @@
import unittest
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
from framework import sql
from services import cachemanager_svc
@@ -168,8 +171,7 @@
sql.SQLTableManager)
self.services = service_manager.Services(
cache_manager=self.cache_manager)
- self.servlet = cachemanager_svc.RamCacheConsolidate(
- 'req', 'res', services=self.services)
+ self.servlet = cachemanager_svc.RamCacheConsolidate(services=self.services)
def testHandleRequest_NothingToDo(self):
mr = testing_helpers.MakeMonorailRequest()
diff --git a/services/test/chart_svc_test.py b/services/test/chart_svc_test.py
index fbd87df..470bc80 100644
--- a/services/test/chart_svc_test.py
+++ b/services/test/chart_svc_test.py
@@ -10,7 +10,10 @@
from __future__ import absolute_import
import datetime
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
import re
import settings
import unittest
diff --git a/services/test/client_config_svc_test.py b/services/test/client_config_svc_test.py
index 5e9b87a..d8a305e 100644
--- a/services/test/client_config_svc_test.py
+++ b/services/test/client_config_svc_test.py
@@ -20,35 +20,32 @@
def __init__(self, content):
self.content = content
- def setUp(self):
- self.handler = client_config_svc.LoadApiClientConfigs()
-
def testProcessResponse_InvalidJSON(self):
r = self.FakeResponse('}{')
with self.assertRaises(ValueError):
- self.handler._process_response(r)
+ client_config_svc._process_response(r)
def testProcessResponse_NoContent(self):
r = self.FakeResponse('{"wrong-key": "some-value"}')
with self.assertRaises(KeyError):
- self.handler._process_response(r)
+ client_config_svc._process_response(r)
def testProcessResponse_NotB64(self):
# 'asd' is not a valid base64-encoded string.
r = self.FakeResponse('{"content": "asd"}')
with self.assertRaises(TypeError):
- self.handler._process_response(r)
+ client_config_svc._process_response(r)
def testProcessResponse_NotProto(self):
# 'asdf' is a valid base64-encoded string.
r = self.FakeResponse('{"content": "asdf"}')
with self.assertRaises(Exception):
- self.handler._process_response(r)
+ client_config_svc._process_response(r)
def testProcessResponse_Success(self):
with open(client_config_svc.CONFIG_FILE_PATH) as f:
r = self.FakeResponse('{"content": "%s"}' % base64.b64encode(f.read()))
- c = self.handler._process_response(r)
+ c = client_config_svc._process_response(r)
assert '123456789.apps.googleusercontent.com' in c
diff --git a/services/test/config_svc_test.py b/services/test/config_svc_test.py
index 6d1d941..dd2796c 100644
--- a/services/test/config_svc_test.py
+++ b/services/test/config_svc_test.py
@@ -13,7 +13,10 @@
import logging
import mock
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
from google.appengine.api import memcache
from google.appengine.ext import testbed
diff --git a/services/test/features_svc_test.py b/services/test/features_svc_test.py
index c80b819..d285152 100644
--- a/services/test/features_svc_test.py
+++ b/services/test/features_svc_test.py
@@ -9,7 +9,10 @@
from __future__ import absolute_import
import logging
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
import time
import unittest
import mock
diff --git a/services/test/fulltext_helpers_test.py b/services/test/fulltext_helpers_test.py
index 1e4f0c9..fbff1b8 100644
--- a/services/test/fulltext_helpers_test.py
+++ b/services/test/fulltext_helpers_test.py
@@ -10,7 +10,10 @@
import unittest
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
from google.appengine.api import search
diff --git a/services/test/issue_svc_test.py b/services/test/issue_svc_test.py
index b6fe682..fe41aa4 100644
--- a/services/test/issue_svc_test.py
+++ b/services/test/issue_svc_test.py
@@ -15,7 +15,10 @@
import unittest
from mock import patch, Mock, ANY
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
from google.appengine.api import search
from google.appengine.ext import testbed
diff --git a/services/test/project_svc_test.py b/services/test/project_svc_test.py
index 2eb7a2b..48de180 100644
--- a/services/test/project_svc_test.py
+++ b/services/test/project_svc_test.py
@@ -11,7 +11,10 @@
import time
import unittest
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
import mock
from google.appengine.ext import testbed
diff --git a/services/test/spam_svc_test.py b/services/test/spam_svc_test.py
index 67b53cf..351ec62 100644
--- a/services/test/spam_svc_test.py
+++ b/services/test/spam_svc_test.py
@@ -11,7 +11,10 @@
import mock
import unittest
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
from google.appengine.ext import testbed
diff --git a/services/test/star_svc_test.py b/services/test/star_svc_test.py
index 03a0d23..3a5ce74 100644
--- a/services/test/star_svc_test.py
+++ b/services/test/star_svc_test.py
@@ -10,7 +10,10 @@
import unittest
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
import mock
from google.appengine.ext import testbed
diff --git a/services/test/tracker_fulltext_test.py b/services/test/tracker_fulltext_test.py
index db8a7a7..a4c935e 100644
--- a/services/test/tracker_fulltext_test.py
+++ b/services/test/tracker_fulltext_test.py
@@ -10,7 +10,10 @@
import unittest
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
from google.appengine.api import search
diff --git a/services/test/user_svc_test.py b/services/test/user_svc_test.py
index 4a8eb16..323d3eb 100644
--- a/services/test/user_svc_test.py
+++ b/services/test/user_svc_test.py
@@ -11,7 +11,10 @@
import unittest
import mock
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
import time
from google.appengine.ext import testbed
diff --git a/services/test/usergroup_svc_test.py b/services/test/usergroup_svc_test.py
index 5bfd899..10b2c8a 100644
--- a/services/test/usergroup_svc_test.py
+++ b/services/test/usergroup_svc_test.py
@@ -12,7 +12,10 @@
import mock
import unittest
-import mox
+try:
+ from mox3 import mox
+except ImportError:
+ import mox
from google.appengine.ext import testbed