| # Copyright 2016 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """A class that provides persistence for Monorail's additional features. |
| |
| Business objects are described in tracker_pb2.py, features_pb2.py, and |
| tracker_bizobj.py. |
| """ |
| from __future__ import print_function |
| from __future__ import division |
| from __future__ import absolute_import |
| |
| import collections |
| import logging |
| import re |
| import time |
| |
| import settings |
| |
| from features import features_constants |
| from features import filterrules_helpers |
| from framework import exceptions |
| from framework import framework_bizobj |
| from framework import framework_constants |
| from framework import sql |
| from mrproto import features_pb2 |
| from services import caches |
| from services import config_svc |
| from tracker import tracker_bizobj |
| from tracker import tracker_constants |
| |
| QUICKEDITHISTORY_TABLE_NAME = 'QuickEditHistory' |
| QUICKEDITMOSTRECENT_TABLE_NAME = 'QuickEditMostRecent' |
| SAVEDQUERY_TABLE_NAME = 'SavedQuery' |
| PROJECT2SAVEDQUERY_TABLE_NAME = 'Project2SavedQuery' |
| SAVEDQUERYEXECUTESINPROJECT_TABLE_NAME = 'SavedQueryExecutesInProject' |
| USER2SAVEDQUERY_TABLE_NAME = 'User2SavedQuery' |
| FILTERRULE_TABLE_NAME = 'FilterRule' |
| HOTLIST_TABLE_NAME = 'Hotlist' |
| HOTLIST2ISSUE_TABLE_NAME = 'Hotlist2Issue' |
| HOTLIST2USER_TABLE_NAME = 'Hotlist2User' |
| |
| |
| QUICKEDITHISTORY_COLS = [ |
| 'user_id', 'project_id', 'slot_num', 'command', 'comment'] |
| QUICKEDITMOSTRECENT_COLS = ['user_id', 'project_id', 'slot_num'] |
| SAVEDQUERY_COLS = ['id', 'name', 'base_query_id', 'query'] |
| PROJECT2SAVEDQUERY_COLS = ['project_id', 'rank', 'query_id'] |
| SAVEDQUERYEXECUTESINPROJECT_COLS = ['query_id', 'project_id'] |
| USER2SAVEDQUERY_COLS = ['user_id', 'rank', 'query_id', 'subscription_mode'] |
| FILTERRULE_COLS = ['project_id', 'rank', 'predicate', 'consequence'] |
| HOTLIST_COLS = [ |
| 'id', 'name', 'summary', 'description', 'is_private', 'default_col_spec'] |
| HOTLIST_ABBR_COLS = ['id', 'name', 'summary', 'is_private'] |
| HOTLIST2ISSUE_COLS = [ |
| 'hotlist_id', 'issue_id', 'rank', 'adder_id', 'added', 'note'] |
| HOTLIST2USER_COLS = ['hotlist_id', 'user_id', 'role_name'] |
| |
| |
| # Regex for parsing one action in the filter rule consequence storage syntax. |
| CONSEQUENCE_RE = re.compile( |
| r'(default_status:(?P<default_status>[-.\w]+))|' |
| r'(default_owner_id:(?P<default_owner_id>\d+))|' |
| r'(add_cc_id:(?P<add_cc_id>\d+))|' |
| r'(add_label:(?P<add_label>[-.\w]+))|' |
| r'(add_notify:(?P<add_notify>[-.@\w]+))|' |
| r'(warning:(?P<warning>.+))|' # Warnings consume the rest of the string. |
| r'(error:(?P<error>.+))' # Errors consume the rest of the string. |
| ) |
| |
| class HotlistTwoLevelCache(caches.AbstractTwoLevelCache): |
| """Class to manage both RAM and memcache for Hotlist PBs.""" |
| |
| def __init__(self, cachemanager, features_service): |
| super(HotlistTwoLevelCache, self).__init__( |
| cachemanager, 'hotlist', 'hotlist:', features_pb2.Hotlist) |
| self.features_service = features_service |
| |
| def _DeserializeHotlists( |
| self, hotlist_rows, issue_rows, role_rows): |
| """Convert database rows into a dictionary of Hotlist PB keyed by ID. |
| |
| Args: |
| hotlist_rows: a list of hotlist rows from HOTLIST_TABLE_NAME. |
| issue_rows: a list of issue rows from HOTLIST2ISSUE_TABLE_NAME, |
| ordered by rank DESC, issue_id. |
| role_rows: a list of role rows from HOTLIST2USER_TABLE_NAME. |
| |
| Returns: |
| a dict mapping hotlist_id to hotlist PB""" |
| hotlist_dict = {} |
| |
| for hotlist_row in hotlist_rows: |
| (hotlist_id, hotlist_name, summary, description, is_private, |
| default_col_spec) = hotlist_row |
| hotlist = features_pb2.MakeHotlist( |
| hotlist_name, hotlist_id=hotlist_id, summary=summary, |
| description=description, is_private=bool(is_private), |
| default_col_spec=default_col_spec) |
| hotlist_dict[hotlist_id] = hotlist |
| |
| for (hotlist_id, issue_id, rank, adder_id, added, note) in issue_rows: |
| hotlist = hotlist_dict.get(hotlist_id) |
| if hotlist: |
| hotlist.items.append( |
| features_pb2.MakeHotlistItem(issue_id=issue_id, rank=rank, |
| adder_id=adder_id , date_added=added, |
| note=note)) |
| else: |
| logging.warning('hotlist %d not found', hotlist_id) |
| |
| for (hotlist_id, user_id, role_name) in role_rows: |
| hotlist = hotlist_dict.get(hotlist_id) |
| if not hotlist: |
| logging.warning('hotlist %d not found', hotlist_id) |
| elif role_name == 'owner': |
| hotlist.owner_ids.append(user_id) |
| elif role_name == 'editor': |
| hotlist.editor_ids.append(user_id) |
| elif role_name == 'follower': |
| hotlist.follower_ids.append(user_id) |
| else: |
| logging.info('unknown role name %s', role_name) |
| |
| return hotlist_dict |
| |
| def FetchItems(self, cnxn, keys): |
| """On RAM and memcache miss, hit the database to get missing hotlists.""" |
| hotlist_rows = self.features_service.hotlist_tbl.Select( |
| cnxn, cols=HOTLIST_COLS, is_deleted=False, id=keys) |
| issue_rows = self.features_service.hotlist2issue_tbl.Select( |
| cnxn, cols=HOTLIST2ISSUE_COLS, hotlist_id=keys, |
| order_by=[('rank DESC', []), ('issue_id', [])]) |
| role_rows = self.features_service.hotlist2user_tbl.Select( |
| cnxn, cols=HOTLIST2USER_COLS, hotlist_id=keys) |
| retrieved_dict = self._DeserializeHotlists( |
| hotlist_rows, issue_rows, role_rows) |
| return retrieved_dict |
| |
| |
| class HotlistIDTwoLevelCache(caches.AbstractTwoLevelCache): |
| """Class to manage both RAM and memcache for hotlist_ids. |
| |
| Keys for this cache are tuples (hotlist_name.lower(), owner_id). |
| This cache should be used to fetch hotlist_ids owned by users or |
| to check if a user owns a hotlist with a certain name, so the |
| hotlist_names in keys will always be in lowercase. |
| """ |
| |
| def __init__(self, cachemanager, features_service): |
| super(HotlistIDTwoLevelCache, self).__init__( |
| cachemanager, 'hotlist_id', 'hotlist_id:', int, |
| max_size=settings.issue_cache_max_size) |
| self.features_service = features_service |
| |
| def _MakeCache(self, cache_manager, kind, max_size=None): |
| """Override normal RamCache creation with ValueCentricRamCache.""" |
| return caches.ValueCentricRamCache(cache_manager, kind, max_size=max_size) |
| |
| def _KeyToStr(self, key): |
| """This cache uses pairs of (str, int) as keys. Convert them to strings.""" |
| return '%s,%d' % key |
| |
| def _StrToKey(self, key_str): |
| """This cache uses pairs of (str, int) as keys. |
| Convert them from strings. |
| """ |
| hotlist_name_str, owner_id_str = key_str.split(',') |
| return (hotlist_name_str, int(owner_id_str)) |
| |
| def _DeserializeHotlistIDs( |
| self, hotlist_rows, owner_rows, wanted_names_for_owners): |
| """Convert database rows into a dictionary of hotlist_ids keyed by ( |
| hotlist_name, owner_id). |
| |
| Args: |
| hotlist_rows: a list of hotlist rows [id, name] from HOTLIST for |
| with names we are interested in. |
| owner_rows: a list of role rows [hotlist_id, uwer_id] from HOTLIST2USER |
| for owners that we are interested in that own hotlists with names that |
| we are interested in. |
| wanted_names_for_owners: a dict of |
| {owner_id: [hotlist_name.lower(), ...], ...} |
| so we know which (hotlist_name, owner_id) keys to return. |
| |
| Returns: |
| A dict mapping (hotlist_name.lower(), owner_id) keys to hotlist_id values. |
| """ |
| hotlist_ids_dict = {} |
| if not hotlist_rows or not owner_rows: |
| return hotlist_ids_dict |
| |
| hotlist_to_owner_id = {} |
| |
| # Note: owner_rows contains hotlist owners that we are interested in, but |
| # may not own hotlists with names we are interested in. |
| for (hotlist_id, user_id) in owner_rows: |
| found_owner_id = hotlist_to_owner_id.get(hotlist_id) |
| if found_owner_id: |
| logging.warning( |
| 'hotlist %d has more than one owner: %d, %d', hotlist_id, user_id, |
| found_owner_id) |
| hotlist_to_owner_id[hotlist_id] = user_id |
| |
| # Note: hotlist_rows hotlists found in the owner_rows that have names |
| # we're interested in. |
| # We use wanted_names_for_owners to filter out hotlists in hotlist_rows |
| # that have a (hotlist_name, owner_id) pair we are not interested in. |
| for (hotlist_id, hotlist_name) in hotlist_rows: |
| owner_id = hotlist_to_owner_id.get(hotlist_id) |
| if owner_id: |
| if hotlist_name.lower() in wanted_names_for_owners.get(owner_id, []): |
| hotlist_ids_dict[(hotlist_name.lower(), owner_id)] = hotlist_id |
| |
| return hotlist_ids_dict |
| |
| def FetchItems(self, cnxn, keys): |
| """On RAM and memcache miss, hit the database.""" |
| hotlist_names, _owner_ids = zip(*keys) |
| # Keys may contain [(name1, user1), (name1, user2)] so we cast this to |
| # a set to make sure 'name1' is not repeated. |
| hotlist_names_set = set(hotlist_names) |
| # Pass this dict to _DeserializeHotlistIDs so it knows what hotlist names |
| # we're interested in for each owner. |
| wanted_names_for_owner = collections.defaultdict(list) |
| for hotlist_name, owner_id in keys: |
| wanted_names_for_owner[owner_id].append(hotlist_name.lower()) |
| |
| role_rows = self.features_service.hotlist2user_tbl.Select( |
| cnxn, cols=['hotlist_id', 'user_id'], |
| user_id=sorted(wanted_names_for_owner.keys()), role_name='owner') |
| |
| hotlist_ids = [row[0] for row in role_rows] |
| hotlist_rows = self.features_service.hotlist_tbl.Select( |
| cnxn, cols=['id', 'name'], id=hotlist_ids, is_deleted=False, |
| where=[('LOWER(name) IN (%s)' % sql.PlaceHolders(hotlist_names_set), |
| [name.lower() for name in sorted(hotlist_names_set)])]) |
| |
| return self._DeserializeHotlistIDs( |
| hotlist_rows, role_rows, wanted_names_for_owner) |
| |
| |
| class FeaturesService(object): |
| """The persistence layer for servlets in the features directory.""" |
| |
| def __init__(self, cache_manager, config_service): |
| """Initialize this object so that it is ready to use. |
| |
| Args: |
| cache_manager: local cache with distributed invalidation. |
| config_service: an instance of ConfigService. |
| """ |
| self.quickedithistory_tbl = sql.SQLTableManager(QUICKEDITHISTORY_TABLE_NAME) |
| self.quickeditmostrecent_tbl = sql.SQLTableManager( |
| QUICKEDITMOSTRECENT_TABLE_NAME) |
| |
| self.savedquery_tbl = sql.SQLTableManager(SAVEDQUERY_TABLE_NAME) |
| self.project2savedquery_tbl = sql.SQLTableManager( |
| PROJECT2SAVEDQUERY_TABLE_NAME) |
| self.savedqueryexecutesinproject_tbl = sql.SQLTableManager( |
| SAVEDQUERYEXECUTESINPROJECT_TABLE_NAME) |
| self.user2savedquery_tbl = sql.SQLTableManager(USER2SAVEDQUERY_TABLE_NAME) |
| |
| self.filterrule_tbl = sql.SQLTableManager(FILTERRULE_TABLE_NAME) |
| |
| self.hotlist_tbl = sql.SQLTableManager(HOTLIST_TABLE_NAME) |
| self.hotlist2issue_tbl = sql.SQLTableManager(HOTLIST2ISSUE_TABLE_NAME) |
| self.hotlist2user_tbl = sql.SQLTableManager(HOTLIST2USER_TABLE_NAME) |
| |
| self.saved_query_cache = caches.RamCache( |
| cache_manager, 'user', max_size=1000) |
| self.canned_query_cache = caches.RamCache( |
| cache_manager, 'project', max_size=1000) |
| |
| self.hotlist_2lc = HotlistTwoLevelCache(cache_manager, self) |
| self.hotlist_id_2lc = HotlistIDTwoLevelCache(cache_manager, self) |
| self.hotlist_user_to_ids = caches.RamCache(cache_manager, 'hotlist') |
| |
| self.config_service = config_service |
| |
| ### QuickEdit command history |
| |
| def GetRecentCommands(self, cnxn, user_id, project_id): |
| """Return recent command items for the "Redo" menu. |
| |
| Args: |
| cnxn: Connection to SQL database. |
| user_id: int ID of the current user. |
| project_id: int ID of the current project. |
| |
| Returns: |
| A pair (cmd_slots, recent_slot_num). cmd_slots is a list of |
| 3-tuples that can be used to populate the "Redo" menu of the |
| quick-edit dialog. recent_slot_num indicates which of those |
| slots should initially populate the command and comment fields. |
| """ |
| # Always start with the standard 5 commands. |
| history = tracker_constants.DEFAULT_RECENT_COMMANDS[:] |
| # If the user has modified any, then overwrite some standard ones. |
| history_rows = self.quickedithistory_tbl.Select( |
| cnxn, cols=['slot_num', 'command', 'comment'], |
| user_id=user_id, project_id=project_id) |
| for slot_num, command, comment in history_rows: |
| if slot_num < len(history): |
| history[slot_num - 1] = (command, comment) |
| |
| slots = [] |
| for idx, (command, comment) in enumerate(history): |
| slots.append((idx + 1, command, comment)) |
| |
| recent_slot_num = self.quickeditmostrecent_tbl.SelectValue( |
| cnxn, 'slot_num', default=1, user_id=user_id, project_id=project_id) |
| |
| return slots, recent_slot_num |
| |
| def StoreRecentCommand( |
| self, cnxn, user_id, project_id, slot_num, command, comment): |
| """Store the given command and comment in the user's command history.""" |
| self.quickedithistory_tbl.InsertRow( |
| cnxn, replace=True, user_id=user_id, project_id=project_id, |
| slot_num=slot_num, command=command, comment=comment) |
| self.quickeditmostrecent_tbl.InsertRow( |
| cnxn, replace=True, user_id=user_id, project_id=project_id, |
| slot_num=slot_num) |
| |
| def ExpungeQuickEditHistory(self, cnxn, project_id): |
| """Completely delete every users' quick edit history for this project.""" |
| self.quickeditmostrecent_tbl.Delete(cnxn, project_id=project_id) |
| self.quickedithistory_tbl.Delete(cnxn, project_id=project_id) |
| |
| def ExpungeQuickEditsByUsers(self, cnxn, user_ids, limit=None): |
| """Completely delete every given users' quick edits. |
| |
| This method will not commit the operations. This method will |
| not make changes to in-memory data. |
| """ |
| commit = False |
| self.quickeditmostrecent_tbl.Delete( |
| cnxn, user_id=user_ids, commit=commit, limit=limit) |
| self.quickedithistory_tbl.Delete( |
| cnxn, user_id=user_ids, commit=commit, limit=limit) |
| |
| ### Saved User and Project Queries |
| |
| def GetSavedQueries(self, cnxn, query_ids): |
| """Retrieve the specified SaveQuery PBs.""" |
| # TODO(jrobbins): RAM cache |
| if not query_ids: |
| return {} |
| saved_queries = {} |
| savedquery_rows = self.savedquery_tbl.Select( |
| cnxn, cols=SAVEDQUERY_COLS, id=query_ids) |
| for saved_query_tuple in savedquery_rows: |
| qid, name, base_id, query = saved_query_tuple |
| saved_queries[qid] = tracker_bizobj.MakeSavedQuery( |
| qid, name, base_id, query) |
| |
| sqeip_rows = self.savedqueryexecutesinproject_tbl.Select( |
| cnxn, cols=SAVEDQUERYEXECUTESINPROJECT_COLS, query_id=query_ids) |
| for query_id, project_id in sqeip_rows: |
| saved_queries[query_id].executes_in_project_ids.append(project_id) |
| |
| return saved_queries |
| |
| def GetSavedQuery(self, cnxn, query_id): |
| """Retrieve the specified SaveQuery PB.""" |
| saved_queries = self.GetSavedQueries(cnxn, [query_id]) |
| return saved_queries.get(query_id) |
| |
| def _GetUsersSavedQueriesDict(self, cnxn, user_ids): |
| """Return a dict of all SavedQuery PBs for the specified users.""" |
| results_dict, missed_uids = self.saved_query_cache.GetAll(user_ids) |
| |
| if missed_uids: |
| savedquery_rows = self.user2savedquery_tbl.Select( |
| cnxn, cols=SAVEDQUERY_COLS + ['user_id', 'subscription_mode'], |
| left_joins=[('SavedQuery ON query_id = id', [])], |
| order_by=[('rank', [])], user_id=missed_uids) |
| sqeip_dict = {} |
| if savedquery_rows: |
| query_ids = {row[0] for row in savedquery_rows} |
| sqeip_rows = self.savedqueryexecutesinproject_tbl.Select( |
| cnxn, cols=SAVEDQUERYEXECUTESINPROJECT_COLS, query_id=query_ids) |
| for qid, pid in sqeip_rows: |
| sqeip_dict.setdefault(qid, []).append(pid) |
| |
| for saved_query_tuple in savedquery_rows: |
| query_id, name, base_id, query, uid, sub_mode = saved_query_tuple |
| sq = tracker_bizobj.MakeSavedQuery( |
| query_id, name, base_id, query, subscription_mode=sub_mode, |
| executes_in_project_ids=sqeip_dict.get(query_id, [])) |
| results_dict.setdefault(uid, []).append(sq) |
| |
| self.saved_query_cache.CacheAll(results_dict) |
| return results_dict |
| |
| # TODO(jrobbins): change this termonology to "canned query" rather than |
| # "saved" throughout the application. |
| def GetSavedQueriesByUserID(self, cnxn, user_id): |
| """Return a list of SavedQuery PBs for the specified user.""" |
| saved_queries_dict = self._GetUsersSavedQueriesDict(cnxn, [user_id]) |
| saved_queries = saved_queries_dict.get(user_id, []) |
| return saved_queries[:] |
| |
| def GetCannedQueriesForProjects(self, cnxn, project_ids): |
| """Return a dict {project_id: [saved_query]} for the specified projects.""" |
| results_dict, missed_pids = self.canned_query_cache.GetAll(project_ids) |
| |
| if missed_pids: |
| cannedquery_rows = self.project2savedquery_tbl.Select( |
| cnxn, cols=['project_id'] + SAVEDQUERY_COLS, |
| left_joins=[('SavedQuery ON query_id = id', [])], |
| order_by=[('rank', [])], project_id=project_ids) |
| |
| for cq_row in cannedquery_rows: |
| project_id = cq_row[0] |
| canned_query_tuple = cq_row[1:] |
| results_dict.setdefault(project_id ,[]).append( |
| tracker_bizobj.MakeSavedQuery(*canned_query_tuple)) |
| |
| self.canned_query_cache.CacheAll(results_dict) |
| return results_dict |
| |
| def GetCannedQueriesByProjectID(self, cnxn, project_id): |
| """Return the list of SavedQueries for the specified project.""" |
| project_ids_to_canned_queries = self.GetCannedQueriesForProjects( |
| cnxn, [project_id]) |
| return project_ids_to_canned_queries.get(project_id, []) |
| |
| def _UpdateSavedQueries(self, cnxn, saved_queries, commit=True): |
| """Store the given SavedQueries to the DB.""" |
| savedquery_rows = [ |
| (sq.query_id or None, sq.name, sq.base_query_id, sq.query) |
| for sq in saved_queries] |
| existing_query_ids = [sq.query_id for sq in saved_queries if sq.query_id] |
| if existing_query_ids: |
| self.savedquery_tbl.Delete(cnxn, id=existing_query_ids, commit=commit) |
| |
| generated_ids = self.savedquery_tbl.InsertRows( |
| cnxn, SAVEDQUERY_COLS, savedquery_rows, commit=commit, |
| return_generated_ids=True) |
| if generated_ids: |
| logging.info('generated_ids are %r', generated_ids) |
| for sq in saved_queries: |
| generated_id = generated_ids.pop(0) |
| if not sq.query_id: |
| sq.query_id = generated_id |
| |
| def UpdateCannedQueries(self, cnxn, project_id, canned_queries): |
| """Update the canned queries for a project. |
| |
| Args: |
| cnxn: connection to SQL database. |
| project_id: int project ID of the project that contains these queries. |
| canned_queries: list of SavedQuery PBs to update. |
| """ |
| self.project2savedquery_tbl.Delete( |
| cnxn, project_id=project_id, commit=False) |
| self._UpdateSavedQueries(cnxn, canned_queries, commit=False) |
| project2savedquery_rows = [ |
| (project_id, rank, sq.query_id) |
| for rank, sq in enumerate(canned_queries)] |
| self.project2savedquery_tbl.InsertRows( |
| cnxn, PROJECT2SAVEDQUERY_COLS, project2savedquery_rows, |
| commit=False) |
| cnxn.Commit() |
| |
| self.canned_query_cache.Invalidate(cnxn, project_id) |
| |
| def UpdateUserSavedQueries(self, cnxn, user_id, saved_queries): |
| """Store the given saved_queries for the given user.""" |
| saved_query_ids = [sq.query_id for sq in saved_queries if sq.query_id] |
| self.savedqueryexecutesinproject_tbl.Delete( |
| cnxn, query_id=saved_query_ids, commit=False) |
| self.user2savedquery_tbl.Delete(cnxn, user_id=user_id, commit=False) |
| |
| self._UpdateSavedQueries(cnxn, saved_queries, commit=False) |
| user2savedquery_rows = [] |
| for rank, sq in enumerate(saved_queries): |
| user2savedquery_rows.append( |
| (user_id, rank, sq.query_id, sq.subscription_mode or 'noemail')) |
| |
| self.user2savedquery_tbl.InsertRows( |
| cnxn, USER2SAVEDQUERY_COLS, user2savedquery_rows, commit=False) |
| |
| sqeip_rows = [] |
| for sq in saved_queries: |
| for pid in sq.executes_in_project_ids: |
| sqeip_rows.append((sq.query_id, pid)) |
| |
| self.savedqueryexecutesinproject_tbl.InsertRows( |
| cnxn, SAVEDQUERYEXECUTESINPROJECT_COLS, sqeip_rows, commit=False) |
| cnxn.Commit() |
| |
| self.saved_query_cache.Invalidate(cnxn, user_id) |
| |
| ### Subscriptions |
| |
| def GetSubscriptionsInProjects(self, cnxn, project_ids): |
| """Return all saved queries for users that have any subscription there. |
| |
| Args: |
| cnxn: Connection to SQL database. |
| project_ids: list of int project IDs that contain the modified issues. |
| |
| Returns: |
| A dict {user_id: all_saved_queries, ...} for all users that have any |
| subscription in any of the specified projects. |
| """ |
| sqeip_join_str = ( |
| 'SavedQueryExecutesInProject ON ' |
| 'SavedQueryExecutesInProject.query_id = User2SavedQuery.query_id') |
| user_join_str = ( |
| 'User ON ' |
| 'User.user_id = User2SavedQuery.user_id') |
| now = int(time.time()) |
| absence_threshold = now - settings.subscription_timeout_secs |
| where = [ |
| ('(User.banned IS NULL OR User.banned = %s)', ['']), |
| ('User.last_visit_timestamp >= %s', [absence_threshold]), |
| ('(User.email_bounce_timestamp IS NULL OR ' |
| 'User.email_bounce_timestamp = %s)', [0]), |
| ] |
| # TODO(jrobbins): cache this since it rarely changes. |
| subscriber_rows = self.user2savedquery_tbl.Select( |
| cnxn, cols=['User2SavedQuery.user_id'], distinct=True, |
| joins=[(sqeip_join_str, []), (user_join_str, [])], |
| subscription_mode='immediate', project_id=project_ids, |
| where=where) |
| subscriber_ids = [row[0] for row in subscriber_rows] |
| logging.info('subscribers relevant to projects %r are %r', |
| project_ids, subscriber_ids) |
| user_ids_to_saved_queries = self._GetUsersSavedQueriesDict( |
| cnxn, subscriber_ids) |
| return user_ids_to_saved_queries |
| |
| def ExpungeSavedQueriesExecuteInProject(self, cnxn, project_id): |
| """Remove any references from saved queries to projects in the database.""" |
| self.savedqueryexecutesinproject_tbl.Delete(cnxn, project_id=project_id) |
| |
| savedquery_rows = self.project2savedquery_tbl.Select( |
| cnxn, cols=['query_id'], project_id=project_id) |
| savedquery_ids = [row[0] for row in savedquery_rows] |
| self.project2savedquery_tbl.Delete(cnxn, project_id=project_id) |
| self.savedquery_tbl.Delete(cnxn, id=savedquery_ids) |
| |
| def ExpungeSavedQueriesByUsers(self, cnxn, user_ids, limit=None): |
| """Completely delete every given users' saved queries. |
| |
| This method will not commit the operations. This method will |
| not make changes to in-memory data. |
| """ |
| commit = False |
| savedquery_rows = self.user2savedquery_tbl.Select( |
| cnxn, cols=['query_id'], user_id=user_ids, limit=limit) |
| savedquery_ids = [row[0] for row in savedquery_rows] |
| self.user2savedquery_tbl.Delete( |
| cnxn, query_id=savedquery_ids, commit=commit) |
| self.savedqueryexecutesinproject_tbl.Delete( |
| cnxn, query_id=savedquery_ids, commit=commit) |
| self.savedquery_tbl.Delete(cnxn, id=savedquery_ids, commit=commit) |
| |
| |
| ### Filter rules |
| |
| def _DeserializeFilterRules(self, filterrule_rows): |
| """Convert the given DB row tuples into PBs.""" |
| result_dict = collections.defaultdict(list) |
| |
| for filterrule_row in sorted(filterrule_rows): |
| project_id, _rank, predicate, consequence = filterrule_row |
| (default_status, default_owner_id, add_cc_ids, add_labels, |
| add_notify, warning, error) = self._DeserializeRuleConsequence( |
| consequence) |
| rule = filterrules_helpers.MakeRule( |
| predicate, default_status=default_status, |
| default_owner_id=default_owner_id, add_cc_ids=add_cc_ids, |
| add_labels=add_labels, add_notify=add_notify, warning=warning, |
| error=error) |
| result_dict[project_id].append(rule) |
| |
| return result_dict |
| |
| def _DeserializeRuleConsequence(self, consequence): |
| """Decode the THEN-part of a filter rule.""" |
| (default_status, default_owner_id, add_cc_ids, add_labels, |
| add_notify, warning, error) = None, None, [], [], [], None, None |
| for match in CONSEQUENCE_RE.finditer(consequence): |
| if match.group('default_status'): |
| default_status = match.group('default_status') |
| elif match.group('default_owner_id'): |
| default_owner_id = int(match.group('default_owner_id')) |
| elif match.group('add_cc_id'): |
| add_cc_ids.append(int(match.group('add_cc_id'))) |
| elif match.group('add_label'): |
| add_labels.append(match.group('add_label')) |
| elif match.group('add_notify'): |
| add_notify.append(match.group('add_notify')) |
| elif match.group('warning'): |
| warning = match.group('warning') |
| elif match.group('error'): |
| error = match.group('error') |
| |
| return (default_status, default_owner_id, add_cc_ids, add_labels, |
| add_notify, warning, error) |
| |
| def _GetFilterRulesByProjectIDs(self, cnxn, project_ids): |
| """Return {project_id: [FilterRule, ...]} for the specified projects.""" |
| # TODO(jrobbins): caching |
| filterrule_rows = self.filterrule_tbl.Select( |
| cnxn, cols=FILTERRULE_COLS, project_id=project_ids) |
| return self._DeserializeFilterRules(filterrule_rows) |
| |
| def GetFilterRules(self, cnxn, project_id): |
| """Return a list of FilterRule PBs for the specified project.""" |
| rules_by_project_id = self._GetFilterRulesByProjectIDs(cnxn, [project_id]) |
| return rules_by_project_id[project_id] |
| |
| def _SerializeRuleConsequence(self, rule): |
| """Put all actions of a filter rule into one string.""" |
| assignments = [] |
| for add_lab in rule.add_labels: |
| assignments.append('add_label:%s' % add_lab) |
| if rule.default_status: |
| assignments.append('default_status:%s' % rule.default_status) |
| if rule.default_owner_id: |
| assignments.append('default_owner_id:%d' % rule.default_owner_id) |
| for add_cc_id in rule.add_cc_ids: |
| assignments.append('add_cc_id:%d' % add_cc_id) |
| for add_notify in rule.add_notify_addrs: |
| assignments.append('add_notify:%s' % add_notify) |
| if rule.warning: |
| assignments.append('warning:%s' % rule.warning) |
| if rule.error: |
| assignments.append('error:%s' % rule.error) |
| |
| return ' '.join(assignments) |
| |
| def UpdateFilterRules(self, cnxn, project_id, rules): |
| """Update the filter rules part of a project's issue configuration. |
| |
| Args: |
| cnxn: connection to SQL database. |
| project_id: int ID of the current project. |
| rules: a list of FilterRule PBs. |
| """ |
| rows = [] |
| for rank, rule in enumerate(rules): |
| predicate = rule.predicate |
| consequence = self._SerializeRuleConsequence(rule) |
| if predicate and consequence: |
| rows.append((project_id, rank, predicate, consequence)) |
| |
| self.filterrule_tbl.Delete(cnxn, project_id=project_id) |
| self.filterrule_tbl.InsertRows(cnxn, FILTERRULE_COLS, rows) |
| |
| def ExpungeFilterRules(self, cnxn, project_id): |
| """Completely destroy filter rule info for the specified project.""" |
| self.filterrule_tbl.Delete(cnxn, project_id=project_id) |
| |
| def ExpungeFilterRulesByUser(self, cnxn, user_ids_by_email): |
| """Wipes any Filter Rules containing the given users. |
| |
| This method will not commit the operation. This method will not make |
| changes to in-memory data. |
| Args: |
| cnxn: connection to SQL database. |
| user_ids_by_email: dict of {email: user_id ..} of all users we want to |
| expunge |
| |
| Returns: |
| Dictionary of {project_id: [(predicate, consequence), ..]} for Filter |
| Rules that will be deleted for containing the given emails. |
| """ |
| deleted_project_rules_dict = collections.defaultdict(list) |
| if user_ids_by_email: |
| deleted_rows = [] |
| emails = user_ids_by_email.keys() |
| all_rules_rows = self.filterrule_tbl.Select(cnxn, FILTERRULE_COLS) |
| logging.info('Fetched all filter rules: %s' % (all_rules_rows,)) |
| for rule_row in all_rules_rows: |
| project_id, _rank, predicate, consequence = rule_row |
| if any(email in predicate for email in emails): |
| deleted_rows.append(rule_row) |
| continue |
| if any((('add_notify:%s' % email) in consequence or |
| ('add_cc_id:%s' % user_id) in consequence or |
| ('default_owner_id:%s' % user_id) in consequence) |
| for email, user_id in user_ids_by_email.items()): |
| deleted_rows.append(rule_row) |
| continue |
| |
| for deleted_row in deleted_rows: |
| project_id, rank, predicate, consequence = deleted_row |
| self.filterrule_tbl.Delete( |
| cnxn, project_id=project_id, rank=rank, predicate=predicate, |
| consequence=consequence, commit=False) |
| deleted_project_rules_dict = self._DeserializeFilterRules(deleted_rows) |
| |
| return deleted_project_rules_dict |
| |
| ### Creating hotlists |
| |
| def CreateHotlist( |
| self, cnxn, name, summary, description, owner_ids, editor_ids, |
| issue_ids=None, is_private=None, default_col_spec=None, ts=None): |
| # type: (MonorailConnection, string, string, string, Collection[int], |
| # Optional[Collection[int]], Optional[Boolean], Optional[string], |
| # Optional[int] -> int |
| """Create and store a Hotlist with the given attributes. |
| |
| Args: |
| cnxn: connection to SQL database. |
| name: a valid hotlist name. |
| summary: one-line explanation of the hotlist. |
| description: one-page explanation of the hotlist. |
| owner_ids: a list of user IDs for the hotlist owners. |
| editor_ids: a list of user IDs for the hotlist editors. |
| issue_ids: a list of issue IDs for the hotlist issues. |
| is_private: True if the hotlist can only be viewed by owners and editors. |
| default_col_spec: the default columns that show in list view. |
| ts: a timestamp for when this hotlist was created. |
| |
| Returns: |
| The int id of the new hotlist. |
| |
| Raises: |
| InputException: if the hotlist name is invalid. |
| HotlistAlreadyExists: if any of the owners already own a hotlist with |
| the same name. |
| UnownedHotlistException: if owner_ids is empty. |
| """ |
| # TODO(crbug.com/monorail/7677): These checks should be done in the |
| # the business layer. |
| # Remove when calls from non-business layer code are removed. |
| if not owner_ids: # Should never happen. |
| logging.error('Attempt to create unowned Hotlist: name:%r', name) |
| raise UnownedHotlistException() |
| if not framework_bizobj.IsValidHotlistName(name): |
| raise exceptions.InputException( |
| '%s is not a valid name for a Hotlist' % name) |
| if self.LookupHotlistIDs(cnxn, [name], owner_ids): |
| raise HotlistAlreadyExists() |
| # TODO(crbug.com/monorail/7677): We are not setting a |
| # default default_col_spec in v3. |
| if default_col_spec is None: |
| default_col_spec = features_constants.DEFAULT_COL_SPEC |
| |
| hotlist_item_fields = [ |
| (issue_id, rank*100, owner_ids[0], ts, '') for |
| rank, issue_id in enumerate(issue_ids or [])] |
| hotlist = features_pb2.MakeHotlist( |
| name, hotlist_item_fields=hotlist_item_fields, summary=summary, |
| description=description, is_private=is_private, owner_ids=owner_ids, |
| editor_ids=editor_ids, default_col_spec=default_col_spec) |
| hotlist.hotlist_id = self._InsertHotlist(cnxn, hotlist) |
| return hotlist |
| |
| def UpdateHotlist( |
| self, cnxn, hotlist_id, name=None, summary=None, description=None, |
| is_private=None, default_col_spec=None, owner_id=None, |
| add_editor_ids=None): |
| """Update the DB with the given hotlist information.""" |
| # Note: If something is None, it does not get changed to None, |
| # it just does not get updated. |
| hotlist = self.GetHotlist(cnxn, hotlist_id, use_cache=False) |
| if not hotlist: |
| raise NoSuchHotlistException() |
| |
| delta = {} |
| if name is not None: |
| delta['name'] = name |
| if summary is not None: |
| delta['summary'] = summary |
| if description is not None: |
| delta['description'] = description |
| if is_private is not None: |
| delta['is_private'] = is_private |
| if default_col_spec is not None: |
| delta['default_col_spec'] = default_col_spec |
| |
| self.hotlist_tbl.Update(cnxn, delta, id=hotlist_id, commit=False) |
| insert_rows = [] |
| if owner_id is not None: |
| insert_rows.append((hotlist_id, owner_id, 'owner')) |
| self.hotlist2user_tbl.Delete( |
| cnxn, hotlist_id=hotlist_id, role='owner', commit=False) |
| if add_editor_ids: |
| insert_rows.extend( |
| [(hotlist_id, user_id, 'editor') for user_id in add_editor_ids]) |
| if insert_rows: |
| self.hotlist2user_tbl.InsertRows( |
| cnxn, HOTLIST2USER_COLS, insert_rows, commit=False) |
| |
| cnxn.Commit() |
| |
| self.hotlist_2lc.InvalidateKeys(cnxn, [hotlist_id]) |
| if not hotlist.owner_ids: # Should never happen. |
| logging.warning( |
| 'Modifying unowned Hotlist: id:%r, name:%r', hotlist_id, hotlist.name) |
| elif hotlist.name: |
| self.hotlist_id_2lc.InvalidateKeys( |
| cnxn, [(hotlist.name.lower(), owner_id) for |
| owner_id in hotlist.owner_ids]) |
| |
| # Update the hotlist PB in RAM |
| if name is not None: |
| hotlist.name = name |
| if summary is not None: |
| hotlist.summary = summary |
| if description is not None: |
| hotlist.description = description |
| if is_private is not None: |
| hotlist.is_private = is_private |
| if default_col_spec is not None: |
| hotlist.default_col_spec = default_col_spec |
| if owner_id is not None: |
| hotlist.owner_ids = [owner_id] |
| if add_editor_ids: |
| hotlist.editor_ids.extend(add_editor_ids) |
| |
| def RemoveHotlistEditors(self, cnxn, hotlist_id, remove_editor_ids): |
| # type: MonorailConnection, int, Collection[int] |
| """Remove given editors from the specified hotlist. |
| |
| Args: |
| cnxn: MonorailConnection object. |
| hotlist_id: int ID of the Hotlist we want to update. |
| remove_editor_ids: collection of existing hotlist editor User IDs |
| that we want to remove from the hotlist. |
| |
| Raises: |
| NoSuchHotlistException: if the hotlist is not found. |
| InputException: if there are not editors to remove. |
| """ |
| if not remove_editor_ids: |
| raise exceptions.InputException |
| hotlist = self.GetHotlist(cnxn, hotlist_id, use_cache=False) |
| if not hotlist: |
| raise NoSuchHotlistException() |
| |
| self.hotlist2user_tbl.Delete( |
| cnxn, hotlist_id=hotlist_id, user_id=remove_editor_ids) |
| self.hotlist_2lc.InvalidateKeys(cnxn, [hotlist_id]) |
| |
| # Update in-memory data |
| for remove_id in remove_editor_ids: |
| hotlist.editor_ids.remove(remove_id) |
| |
| def UpdateHotlistIssues( |
| self, |
| cnxn, # type: sql.MonorailConnection |
| hotlist_id, # type: int |
| updated_items, # type: Collection[features_pb2.HotlistItem] |
| remove_issue_ids, # type: Collection[int] |
| issue_svc, # type: issue_svc.IssueService |
| chart_svc, # type: chart_svc.ChartService |
| commit=True # type: Optional[bool] |
| ): |
| # type: (...) -> None |
| """Update the Issues in a Hotlist. |
| This method removes the given remove_issue_ids from a Hotlist then |
| updates or adds the HotlistItems found in updated_items. HotlistItems |
| in updated_items may exist in the hotlist and just need to be updated |
| or they may be new items that should be added to the Hotlist. |
| |
| Args: |
| cnxn: MonorailConnection object. |
| hotlist_id: int ID of the Hotlist to update. |
| updated_items: Collection of HotlistItems that either already exist in |
| the hotlist and need to be updated or needed to be added to the hotlist. |
| remove_issue_ids: Collection of Issue IDs that should be removed from the |
| hotlist. |
| issue_svc: IssueService object. |
| chart_svc: ChartService object. |
| |
| Raises: |
| NoSuchHotlistException if a hotlist with the given ID is not found. |
| InputException if no changes were given. |
| """ |
| if not updated_items and not remove_issue_ids: |
| raise exceptions.InputException('No changes to make') |
| |
| hotlist = self.GetHotlist(cnxn, hotlist_id, use_cache=False) |
| if not hotlist: |
| raise NoSuchHotlistException() |
| |
| # Used to hold the updated Hotlist.items to use when updating |
| # the in-memory hotlist. |
| all_hotlist_items = list(hotlist.items) |
| |
| # Used to hold ids of issues affected by this change for storing |
| # Issue Snapshots. |
| affected_issue_ids = set() |
| |
| if remove_issue_ids: |
| affected_issue_ids.update(remove_issue_ids) |
| self.hotlist2issue_tbl.Delete( |
| cnxn, hotlist_id=hotlist_id, issue_id=remove_issue_ids, commit=False) |
| all_hotlist_items = list( |
| filter( |
| lambda item: item.issue_id not in remove_issue_ids, |
| all_hotlist_items)) |
| |
| if updated_items: |
| updated_issue_ids = [item.issue_id for item in updated_items] |
| affected_issue_ids.update(updated_issue_ids) |
| self.hotlist2issue_tbl.Delete( |
| cnxn, hotlist_id=hotlist_id, issue_id=updated_issue_ids, commit=False) |
| insert_rows = [] |
| for item in updated_items: |
| insert_rows.append( |
| ( |
| hotlist_id, item.issue_id, item.rank, item.adder_id, |
| item.date_added, item.note)) |
| self.hotlist2issue_tbl.InsertRows( |
| cnxn, cols=HOTLIST2ISSUE_COLS, row_values=insert_rows, commit=False) |
| all_hotlist_items = list( |
| filter( |
| lambda item: item.issue_id not in updated_issue_ids, |
| all_hotlist_items)) |
| all_hotlist_items.extend(updated_items) |
| |
| if commit: |
| cnxn.Commit() |
| self.hotlist_2lc.InvalidateKeys(cnxn, [hotlist_id]) |
| |
| # Update in-memory hotlist items. |
| hotlist.items = sorted(all_hotlist_items, key=lambda item: item.rank) |
| |
| issues = issue_svc.GetIssues(cnxn, list(affected_issue_ids)) |
| chart_svc.StoreIssueSnapshots(cnxn, issues, commit=commit) |
| |
| # TODO(crbug/monorail/7104): {Add|Remove}IssuesToHotlists both call |
| # UpdateHotlistItems to add/remove issues from a hotlist. |
| # UpdateHotlistItemsFields is called by methods for reranking existing issues |
| # and updating HotlistItem notes. |
| # (1) We are removing notes from HotlistItems. crbug/monorail/#### |
| # (2) our v3 AddHotlistItems will allow for inserting new issues to |
| # non-last ranks of a hotlist. So there could be some shared code |
| # for the reranking path and the adding issues path. |
| # UpdateHotlistIssues will be handling adding, removing, and reranking issues. |
| # {Add|Remove}IssueToHotlists, UpdateHotlistItems, UpdateHotlistItemFields |
| # should be removed, once all methods are updated to call UpdateHotlistIssues. |
| |
| def AddIssueToHotlists(self, cnxn, hotlist_ids, issue_tuple, issue_svc, |
| chart_svc, commit=True): |
| """Add a single issue, specified in the issue_tuple, to the given hotlists. |
| |
| Args: |
| cnxn: connection to SQL database. |
| hotlist_ids: a list of hotlist_ids to add the issues to. |
| issue_tuple: (issue_id, user_id, ts, note) of the issue to be added. |
| issue_svc: an instance of IssueService. |
| chart_svc: an instance of ChartService. |
| """ |
| self.AddIssuesToHotlists(cnxn, hotlist_ids, [issue_tuple], issue_svc, |
| chart_svc, commit=commit) |
| |
| def AddIssuesToHotlists(self, cnxn, hotlist_ids, added_tuples, issue_svc, |
| chart_svc, commit=True): |
| """Add the issues given in the added_tuples list to the given hotlists. |
| |
| Args: |
| cnxn: connection to SQL database. |
| hotlist_ids: a list of hotlist_ids to add the issues to. |
| added_tuples: a list of (issue_id, user_id, ts, note) |
| for issues to be added. |
| issue_svc: an instance of IssueService. |
| chart_svc: an instance of ChartService. |
| """ |
| for hotlist_id in hotlist_ids: |
| self.UpdateHotlistItems(cnxn, hotlist_id, [], added_tuples, commit=commit) |
| |
| issues = issue_svc.GetIssues(cnxn, |
| [added_tuple[0] for added_tuple in added_tuples]) |
| chart_svc.StoreIssueSnapshots(cnxn, issues, commit=commit) |
| |
| def RemoveIssuesFromHotlists(self, cnxn, hotlist_ids, issue_ids, issue_svc, |
| chart_svc, commit=True): |
| """Remove the issues given in issue_ids from the given hotlists. |
| |
| Args: |
| cnxn: connection to SQL database. |
| hotlist_ids: a list of hotlist ids to remove the issues from. |
| issue_ids: a list of issue_ids to be removed. |
| issue_svc: an instance of IssueService. |
| chart_svc: an instance of ChartService. |
| """ |
| for hotlist_id in hotlist_ids: |
| self.UpdateHotlistItems(cnxn, hotlist_id, issue_ids, [], commit=commit) |
| |
| issues = issue_svc.GetIssues(cnxn, issue_ids) |
| chart_svc.StoreIssueSnapshots(cnxn, issues, commit=commit) |
| |
| def UpdateHotlistItems( |
| self, cnxn, hotlist_id, remove, added_tuples, commit=True): |
| """Updates a hotlist's list of hotlistissues. |
| |
| Args: |
| cnxn: connection to SQL database. |
| hotlist_id: the ID of the hotlist to update. |
| remove: a list of issue_ids for be removed. |
| added_tuples: a list of (issue_id, user_id, ts, note) |
| for issues to be added. |
| """ |
| hotlist = self.GetHotlist(cnxn, hotlist_id, use_cache=False) |
| if not hotlist: |
| raise NoSuchHotlistException() |
| |
| # adding new Hotlistissues, ignoring pairs where issue_id is already in |
| # hotlist's iid_rank_pairs |
| current_issues_ids = { |
| item.issue_id for item in hotlist.items} |
| |
| self.hotlist2issue_tbl.Delete( |
| cnxn, hotlist_id=hotlist_id, |
| issue_id=[remove_id for remove_id in remove |
| if remove_id in current_issues_ids], |
| commit=False) |
| if hotlist.items: |
| items_sorted = sorted(hotlist.items, key=lambda item: item.rank) |
| rank_base = items_sorted[-1].rank + 10 |
| else: |
| rank_base = 1 |
| insert_rows = [ |
| (hotlist_id, issue_id, rank*10 + rank_base, user_id, ts, note) |
| for (rank, (issue_id, user_id, ts, note)) in enumerate(added_tuples) |
| if issue_id not in current_issues_ids] |
| self.hotlist2issue_tbl.InsertRows( |
| cnxn, cols=HOTLIST2ISSUE_COLS, row_values=insert_rows, commit=commit) |
| self.hotlist_2lc.InvalidateKeys(cnxn, [hotlist_id]) |
| |
| # removing an issue that was never in the hotlist would not cause any |
| # problems. |
| items = [ |
| item for item in hotlist.items if |
| item.issue_id not in remove] |
| |
| new_hotlist_items = [ |
| features_pb2.MakeHotlistItem(issue_id, rank, user_id, ts, note) |
| for (_hid, issue_id, rank, user_id, ts, note) in insert_rows] |
| items.extend(new_hotlist_items) |
| hotlist.items = items |
| |
| def UpdateHotlistItemsFields( |
| self, cnxn, hotlist_id, new_ranks=None, new_notes=None, commit=True): |
| """Updates rankings or notes of hotlistissues. |
| |
| Args: |
| cnxn: connection to SQL database. |
| hotlist_id: the ID of the hotlist to update. |
| new_ranks : This should be a dictionary of {issue_id: rank}. |
| new_notes: This should be a diciontary of {issue_id: note}. |
| commit: set to False to skip the DB commit and do it in the caller. |
| """ |
| hotlist = self.GetHotlist(cnxn, hotlist_id, use_cache=False) |
| if not hotlist: |
| raise NoSuchHotlistException() |
| if new_ranks is None: |
| new_ranks = {} |
| if new_notes is None: |
| new_notes = {} |
| issue_ids = [] |
| insert_rows = [] |
| |
| # Update the hotlist PB in RAM |
| for hotlist_item in hotlist.items: |
| item_updated = False |
| if hotlist_item.issue_id in new_ranks: |
| # Update rank before adding it to insert_rows |
| hotlist_item.rank = new_ranks[hotlist_item.issue_id] |
| item_updated = True |
| if hotlist_item.issue_id in new_notes: |
| # Update note before adding it to insert_rows |
| hotlist_item.note = new_notes[hotlist_item.issue_id] |
| item_updated = True |
| if item_updated: |
| issue_ids.append(hotlist_item.issue_id) |
| insert_rows.append(( |
| hotlist_id, hotlist_item.issue_id, hotlist_item.rank, |
| hotlist_item.adder_id, hotlist_item.date_added, hotlist_item.note)) |
| hotlist.items = sorted(hotlist.items, key=lambda item: item.rank) |
| self.hotlist2issue_tbl.Delete( |
| cnxn, hotlist_id=hotlist_id, issue_id=issue_ids, commit=False) |
| |
| self.hotlist2issue_tbl.InsertRows( |
| cnxn, cols=HOTLIST2ISSUE_COLS , row_values=insert_rows, commit=commit) |
| self.hotlist_2lc.InvalidateKeys(cnxn, [hotlist_id]) |
| |
| def _InsertHotlist(self, cnxn, hotlist): |
| """Insert the given hotlist into the database.""" |
| hotlist_id = self.hotlist_tbl.InsertRow( |
| cnxn, name=hotlist.name, summary=hotlist.summary, |
| description=hotlist.description, is_private=hotlist.is_private, |
| default_col_spec=hotlist.default_col_spec) |
| logging.info('stored hotlist was given id %d', hotlist_id) |
| |
| self.hotlist2issue_tbl.InsertRows( |
| cnxn, HOTLIST2ISSUE_COLS, |
| [(hotlist_id, issue.issue_id, issue.rank, |
| issue.adder_id, issue.date_added, issue.note) |
| for issue in hotlist.items], |
| commit=False) |
| self.hotlist2user_tbl.InsertRows( |
| cnxn, HOTLIST2USER_COLS, |
| [(hotlist_id, user_id, 'owner') |
| for user_id in hotlist.owner_ids] + |
| [(hotlist_id, user_id, 'editor') |
| for user_id in hotlist.editor_ids] + |
| [(hotlist_id, user_id, 'follower') |
| for user_id in hotlist.follower_ids]) |
| |
| self.hotlist_user_to_ids.InvalidateKeys(cnxn, hotlist.owner_ids) |
| |
| return hotlist_id |
| |
| def TransferHotlistOwnership( |
| self, cnxn, hotlist, new_owner_id, remain_editor, commit=True): |
| """Transfers ownership of a hotlist to a new owner.""" |
| new_editor_ids = hotlist.editor_ids |
| if remain_editor: |
| new_editor_ids.extend(hotlist.owner_ids) |
| if new_owner_id in new_editor_ids: |
| new_editor_ids.remove(new_owner_id) |
| new_follower_ids = hotlist.follower_ids |
| if new_owner_id in new_follower_ids: |
| new_follower_ids.remove(new_owner_id) |
| self.UpdateHotlistRoles( |
| cnxn, hotlist.hotlist_id, [new_owner_id], new_editor_ids, |
| new_follower_ids, commit=commit) |
| |
| ### Lookup hotlist IDs |
| |
| def LookupHotlistIDs(self, cnxn, hotlist_names, owner_ids): |
| """Return a dict of (name, owner_id) mapped to hotlist_id for all hotlists |
| with one of the given names and any of the given owners. Hotlists that |
| match multiple owners will be in the dict multiple times.""" |
| id_dict, _missed_keys = self.hotlist_id_2lc.GetAll( |
| cnxn, [(name.lower(), owner_id) |
| for name in hotlist_names for owner_id in owner_ids]) |
| return id_dict |
| |
| def LookupUserHotlists(self, cnxn, user_ids): |
| """Return a dict of {user_id: [hotlist_id,...]} for all user_ids.""" |
| id_dict, missed_ids = self.hotlist_user_to_ids.GetAll(user_ids) |
| if missed_ids: |
| retrieved_dict = {user_id: [] for user_id in missed_ids} |
| id_rows = self.hotlist2user_tbl.Select( |
| cnxn, cols=['user_id', 'hotlist_id'], user_id=user_ids, |
| left_joins=[('Hotlist ON hotlist_id = id', [])], |
| where=[('Hotlist.is_deleted = %s', [False])]) |
| for (user_id, hotlist_id) in id_rows: |
| retrieved_dict[user_id].append(hotlist_id) |
| self.hotlist_user_to_ids.CacheAll(retrieved_dict) |
| id_dict.update(retrieved_dict) |
| |
| return id_dict |
| |
| def LookupIssueHotlists(self, cnxn, issue_ids): |
| """Return a dict of {issue_id: [hotlist_id,...]} for all issue_ids.""" |
| # TODO(jojwang): create hotlist_issue_to_ids cache |
| retrieved_dict = {issue_id: [] for issue_id in issue_ids} |
| id_rows = self.hotlist2issue_tbl.Select( |
| cnxn, cols=['hotlist_id', 'issue_id'], issue_id=issue_ids, |
| left_joins=[('Hotlist ON hotlist_id = id', [])], |
| where=[('Hotlist.is_deleted = %s', [False])]) |
| for hotlist_id, issue_id in id_rows: |
| retrieved_dict[issue_id].append(hotlist_id) |
| return retrieved_dict |
| |
| def GetProjectIDsFromHotlist(self, cnxn, hotlist_id): |
| project_id_rows = self.hotlist2issue_tbl.Select(cnxn, |
| cols=['Issue.project_id'], hotlist_id=hotlist_id, distinct=True, |
| left_joins=[('Issue ON issue_id = id', [])]) |
| return [row[0] for row in project_id_rows] |
| |
| ### Get hotlists |
| def GetHotlists(self, cnxn, hotlist_ids, use_cache=True): |
| """Returns dict of {hotlist_id: hotlist PB}.""" |
| hotlists_dict, missed_ids = self.hotlist_2lc.GetAll( |
| cnxn, hotlist_ids, use_cache=use_cache) |
| |
| if missed_ids: |
| raise NoSuchHotlistException() |
| |
| return hotlists_dict |
| |
| def GetHotlistsByUserID(self, cnxn, user_id, use_cache=True): |
| """Get a list of hotlist PBs for a given user.""" |
| hotlist_id_dict = self.LookupUserHotlists(cnxn, [user_id]) |
| hotlists = self.GetHotlists( |
| cnxn, hotlist_id_dict.get(user_id, []), use_cache=use_cache) |
| return list(hotlists.values()) |
| |
| def GetHotlistsByIssueID(self, cnxn, issue_id, use_cache=True): |
| """Get a list of hotlist PBs for a given issue.""" |
| hotlist_id_dict = self.LookupIssueHotlists(cnxn, [issue_id]) |
| hotlists = self.GetHotlists( |
| cnxn, hotlist_id_dict.get(issue_id, []), use_cache=use_cache) |
| return list(hotlists.values()) |
| |
| def GetHotlist(self, cnxn, hotlist_id, use_cache=True): |
| """Returns hotlist PB.""" |
| hotlist_dict = self.GetHotlists(cnxn, [hotlist_id], use_cache=use_cache) |
| return hotlist_dict[hotlist_id] |
| |
| def GetHotlistsByID(self, cnxn, hotlist_ids, use_cache=True): |
| """Load all the Hotlist PBs for the given hotlists. |
| |
| Args: |
| cnxn: connection to SQL database. |
| hotlist_ids: list of hotlist ids. |
| use_cache: specifiy False to force database query. |
| |
| Returns: |
| A dict mapping ids to the corresponding Hotlist protocol buffers and |
| a list of any hotlist_ids that were not found. |
| """ |
| hotlists_dict, missed_ids = self.hotlist_2lc.GetAll( |
| cnxn, hotlist_ids, use_cache=use_cache) |
| return hotlists_dict, missed_ids |
| |
| def GetHotlistByID(self, cnxn, hotlist_id, use_cache=True): |
| """Load the specified hotlist from the database, None if does not exist.""" |
| hotlist_dict, _ = self.GetHotlistsByID( |
| cnxn, [hotlist_id], use_cache=use_cache) |
| return hotlist_dict.get(hotlist_id) |
| |
| def UpdateHotlistRoles( |
| self, cnxn, hotlist_id, owner_ids, editor_ids, follower_ids, commit=True): |
| """"Store the hotlist's roles in the DB.""" |
| # This will be a newly contructed object, not from the cache and not |
| # shared with any other thread. |
| hotlist = self.GetHotlist(cnxn, hotlist_id, use_cache=False) |
| if not hotlist: |
| raise NoSuchHotlistException() |
| |
| self.hotlist2user_tbl.Delete( |
| cnxn, hotlist_id=hotlist_id, commit=False) |
| |
| insert_rows = [(hotlist_id, user_id, 'owner') for user_id in owner_ids] |
| insert_rows.extend( |
| [(hotlist_id, user_id, 'editor') for user_id in editor_ids]) |
| insert_rows.extend( |
| [(hotlist_id, user_id, 'follower') for user_id in follower_ids]) |
| self.hotlist2user_tbl.InsertRows( |
| cnxn, HOTLIST2USER_COLS, insert_rows, commit=False) |
| |
| if commit: |
| cnxn.Commit() |
| self.hotlist_2lc.InvalidateKeys(cnxn, [hotlist_id]) |
| self.hotlist_user_to_ids.InvalidateKeys(cnxn, hotlist.owner_ids) |
| hotlist.owner_ids = owner_ids |
| hotlist.editor_ids = editor_ids |
| hotlist.follower_ids = follower_ids |
| |
| def DeleteHotlist(self, cnxn, hotlist_id, commit=True): |
| hotlist = self.GetHotlist(cnxn, hotlist_id, use_cache=False) |
| if not hotlist: |
| raise NoSuchHotlistException() |
| |
| # Fetch all associated project IDs in order to invalidate their cache. |
| project_ids = self.GetProjectIDsFromHotlist(cnxn, hotlist_id) |
| |
| delta = {'is_deleted': True} |
| self.hotlist_tbl.Update(cnxn, delta, id=hotlist_id, commit=commit) |
| |
| self.hotlist_2lc.InvalidateKeys(cnxn, [hotlist_id]) |
| self.hotlist_user_to_ids.InvalidateKeys(cnxn, hotlist.owner_ids) |
| self.hotlist_user_to_ids.InvalidateKeys(cnxn, hotlist.editor_ids) |
| if not hotlist.owner_ids: # Should never happen. |
| logging.warning( |
| 'Soft-deleting unowned Hotlist: id:%r, name:%r', hotlist_id, |
| hotlist.name) |
| elif hotlist.name: |
| self.hotlist_id_2lc.InvalidateKeys( |
| cnxn, [(hotlist.name.lower(), owner_id) for |
| owner_id in hotlist.owner_ids]) |
| |
| for project_id in project_ids: |
| self.config_service.InvalidateMemcacheForEntireProject(project_id) |
| |
| def ExpungeHotlists( |
| self, cnxn, hotlist_ids, star_svc, user_svc, chart_svc, commit=True): |
| """Wipes the given hotlists from the DB tables. |
| |
| This method will only do cache invalidation if commit is set to True. |
| |
| Args: |
| cnxn: connection to SQL database. |
| hotlist_ids: the ID of the hotlists to Expunge. |
| star_svc: an instance of a HotlistStarService. |
| user_svc: an instance of a UserService. |
| chart_svc: an instance of a ChartService. |
| commit: set to False to skip the DB commit and do it in the caller. |
| """ |
| |
| hotlists_by_id = self.GetHotlists(cnxn, hotlist_ids) |
| |
| for hotlist_id in hotlist_ids: |
| star_svc.ExpungeStars(cnxn, hotlist_id, commit=commit) |
| chart_svc.ExpungeHotlistsFromIssueSnapshots( |
| cnxn, hotlist_ids, commit=commit) |
| user_svc.ExpungeHotlistsFromHistory(cnxn, hotlist_ids, commit=commit) |
| self.hotlist2user_tbl.Delete(cnxn, hotlist_id=hotlist_ids, commit=commit) |
| self.hotlist2issue_tbl.Delete(cnxn, hotlist_id=hotlist_ids, commit=commit) |
| self.hotlist_tbl.Delete(cnxn, id=hotlist_ids, commit=commit) |
| |
| # Invalidate cache for deleted hotlists. |
| self.hotlist_2lc.InvalidateKeys(cnxn, hotlist_ids) |
| users_to_invalidate = set() |
| for hotlist in hotlists_by_id.values(): |
| users_to_invalidate.update( |
| hotlist.owner_ids + hotlist.editor_ids + hotlist.follower_ids) |
| self.hotlist_id_2lc.InvalidateKeys( |
| cnxn, [(hotlist.name, owner_id) for owner_id in hotlist.owner_ids]) |
| self.hotlist_user_to_ids.InvalidateKeys(cnxn, list(users_to_invalidate)) |
| hotlist_project_ids = set() |
| for hotlist_id in hotlist_ids: |
| hotlist_project_ids.update(self.GetProjectIDsFromHotlist( |
| cnxn, hotlist_id)) |
| for project_id in hotlist_project_ids: |
| self.config_service.InvalidateMemcacheForEntireProject(project_id) |
| |
| def ExpungeUsersInHotlists( |
| self, cnxn, user_ids, star_svc, user_svc, chart_svc): |
| """Wipes the given users and any hotlists they owned from the |
| hotlists system. |
| |
| This method will not commit the operation. This method will not make |
| changes to in-memory data. |
| """ |
| # Transfer hotlist ownership to editors, if possible. |
| hotlist_ids_by_user_id = self.LookupUserHotlists(cnxn, user_ids) |
| hotlist_ids = [hotlist_id for hotlist_ids in hotlist_ids_by_user_id.values() |
| for hotlist_id in hotlist_ids] |
| hotlists_by_id, missed = self.GetHotlistsByID( |
| cnxn, list(set(hotlist_ids)), use_cache=False) |
| logging.info('Missed hotlists: %s', missed) |
| |
| hotlists_to_delete = [] |
| for hotlist_id, hotlist in hotlists_by_id.items(): |
| # One of the users to be deleted is an owner of hotlist. |
| if not set(hotlist.owner_ids).isdisjoint(user_ids): |
| hotlists_to_delete.append(hotlist_id) |
| candidate_new_owners = [user_id for user_id in hotlist.editor_ids |
| if user_id not in user_ids] |
| for candidate_id in candidate_new_owners: |
| if not self.LookupHotlistIDs(cnxn, [hotlist.name], [candidate_id]): |
| self.TransferHotlistOwnership( |
| cnxn, hotlist, candidate_id, False, commit=False) |
| # Hotlist transferred successfully. No need to delete it. |
| hotlists_to_delete.remove(hotlist_id) |
| break |
| |
| # Delete users |
| self.hotlist2user_tbl.Delete(cnxn, user_id=user_ids, commit=False) |
| self.hotlist2issue_tbl.Update( |
| cnxn, {'adder_id': framework_constants.DELETED_USER_ID}, |
| adder_id=user_ids, commit=False) |
| user_svc.ExpungeUsersHotlistsHistory(cnxn, user_ids, commit=False) |
| # Delete hotlists |
| if hotlists_to_delete: |
| self.ExpungeHotlists( |
| cnxn, hotlists_to_delete, star_svc, user_svc, chart_svc, commit=False) |
| |
| |
| class HotlistAlreadyExists(Exception): |
| """Tried to create a hotlist with the same name as another hotlist |
| with the same owner.""" |
| pass |
| |
| |
| class NoSuchHotlistException(Exception): |
| """The requested hotlist was not found.""" |
| pass |
| |
| |
| class UnownedHotlistException(Exception): |
| """Tried to create a hotlist with no owner.""" |
| pass |