Project import generated by Copybara.
GitOrigin-RevId: d9e9e3fb4e31372ec1fb43b178994ca78fa8fe70
diff --git a/tracker/issueimport.py b/tracker/issueimport.py
new file mode 100644
index 0000000..1e0289b
--- /dev/null
+++ b/tracker/issueimport.py
@@ -0,0 +1,310 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+"""Servlet to import a file of issues in JSON format.
+"""
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+
+import collections
+import json
+import logging
+import time
+
+import ezt
+
+from features import filterrules_helpers
+from framework import framework_helpers
+from framework import jsonfeed
+from framework import permissions
+from framework import servlet
+from framework import urls
+from proto import tracker_pb2
+
+
+ParserState = collections.namedtuple(
+ 'ParserState',
+ 'user_id_dict, nonexist_emails, issue_list, comments_dict, starrers_dict, '
+ 'relations_dict')
+
+
+class IssueImport(servlet.Servlet):
+ """IssueImport loads a file of issues in JSON format."""
+
+ _PAGE_TEMPLATE = 'tracker/issue-import-page.ezt'
+ _MAIN_TAB_MODE = servlet.Servlet.MAIN_TAB_ISSUES
+
+ def AssertBasePermission(self, mr):
+ """Make sure that the logged in user has permission to view this page."""
+ super(IssueImport, self).AssertBasePermission(mr)
+ if not mr.auth.user_pb.is_site_admin:
+ raise permissions.PermissionException(
+ 'Only site admins may import issues')
+
+ def GatherPageData(self, mr):
+ """Build up a dictionary of data values to use when rendering the page."""
+ return {
+ 'issue_tab_mode': None,
+ 'page_perms': self.MakePagePerms(mr, None, permissions.CREATE_ISSUE),
+ 'import_errors': [],
+ }
+
+ def ProcessFormData(self, mr, post_data):
+ """Process the issue entry form.
+
+ Args:
+ mr: commonly used info parsed from the request.
+ post_data: The post_data dict for the current request.
+
+ Returns:
+ String URL to redirect the user to after processing.
+ """
+ import_errors = []
+ json_data = None
+
+ pre_check_only = 'pre_check_only' in post_data
+
+ uploaded_file = post_data.get('jsonfile')
+ if uploaded_file is None:
+ import_errors.append('No file uploaded')
+ else:
+ try:
+ json_str = uploaded_file.value
+ if json_str.startswith(jsonfeed.XSSI_PREFIX):
+ json_str = json_str[len(jsonfeed.XSSI_PREFIX):]
+ json_data = json.loads(json_str)
+ except ValueError:
+ import_errors.append('error parsing JSON in file')
+
+ if uploaded_file and not json_data:
+ import_errors.append('JSON file was empty')
+
+ # Note that the project must already exist in order to even reach
+ # this servlet because it is hosted in the context of a project.
+ if json_data and mr.project_name != json_data['metadata']['project']:
+ import_errors.append(
+ 'Project name does not match. '
+ 'Edit the file if you want to import into this project anyway.')
+
+ if import_errors:
+ return self.PleaseCorrect(mr, import_errors=import_errors)
+
+ event_log = [] # We accumulate a list of messages to display to the user.
+
+ try:
+ # First we parse the JSON into objects, but we don't have DB IDs yet.
+ state = self._ParseObjects(mr.cnxn, mr.project_id, json_data, event_log)
+ # If that worked, go ahead and start saving the data to the DB.
+ if not pre_check_only:
+ self._SaveObjects(mr.cnxn, mr.project_id, state, event_log)
+ except JSONImportError:
+ # just report it to the user by displaying event_log
+ event_log.append('Aborted import processing')
+
+ # This is a little bit of a hack because it always uses the form validation
+ # error message display logic to show the results of this import run,
+ # which may include errors or not.
+ return self.PleaseCorrect(mr, import_errors=event_log)
+
+ def _ParseObjects(self, cnxn, project_id, json_data, event_log):
+ """Examine JSON data and return a parser state for further processing."""
+ # Decide which users need to be created.
+ needed_emails = json_data['emails']
+ user_id_dict = self.services.user.LookupExistingUserIDs(cnxn, needed_emails)
+ nonexist_emails = [email for email in needed_emails
+ if email not in user_id_dict]
+
+ event_log.append('Need to create %d users: %r' %
+ (len(nonexist_emails), nonexist_emails))
+ user_id_dict.update({
+ email.lower(): framework_helpers.MurmurHash3_x86_32(email.lower())
+ for email in nonexist_emails})
+
+ num_comments = 0
+ num_stars = 0
+ issue_list = []
+ comments_dict = collections.defaultdict(list)
+ starrers_dict = collections.defaultdict(list)
+ relations_dict = collections.defaultdict(list)
+ for issue_json in json_data.get('issues', []):
+ issue, comment_list, starrer_list, relation_list = self._ParseIssue(
+ cnxn, project_id, user_id_dict, issue_json, event_log)
+ issue_list.append(issue)
+ comments_dict[issue.local_id] = comment_list
+ starrers_dict[issue.local_id] = starrer_list
+ relations_dict[issue.local_id] = relation_list
+ num_comments += len(comment_list)
+ num_stars += len(starrer_list)
+
+ event_log.append(
+ 'Found info for %d issues: %r' %
+ (len(issue_list), sorted([issue.local_id for issue in issue_list])))
+
+ event_log.append(
+ 'Found %d total comments for %d issues' %
+ (num_comments, len(comments_dict)))
+
+ event_log.append(
+ 'Found %d total stars for %d issues' %
+ (num_stars, len(starrers_dict)))
+
+ event_log.append(
+ 'Found %d total relationships.' %
+ sum((len(dsts) for dsts in relations_dict.values())))
+
+ event_log.append('Parsing phase finished OK')
+ return ParserState(
+ user_id_dict, nonexist_emails, issue_list,
+ comments_dict, starrers_dict, relations_dict)
+
+ def _ParseIssue(self, cnxn, project_id, user_id_dict, issue_json, event_log):
+ issue = tracker_pb2.Issue(
+ project_id=project_id,
+ local_id=issue_json['local_id'],
+ reporter_id=user_id_dict[issue_json['reporter']],
+ summary=issue_json['summary'],
+ opened_timestamp=issue_json['opened'],
+ modified_timestamp=issue_json['modified'],
+ cc_ids=[user_id_dict[cc_email]
+ for cc_email in issue_json.get('cc', [])
+ if cc_email in user_id_dict],
+ status=issue_json.get('status', ''),
+ labels=issue_json.get('labels', []),
+ field_values=[self._ParseFieldValue(cnxn, project_id, user_id_dict, field)
+ for field in issue_json.get('fields', [])])
+ if issue_json.get('owner'):
+ issue.owner_id = user_id_dict[issue_json['owner']]
+ if issue_json.get('closed'):
+ issue.closed_timestamp = issue_json['closed']
+ comments = [self._ParseComment(
+ project_id, user_id_dict, comment_json, event_log)
+ for comment_json in issue_json.get('comments', [])]
+
+ starrers = [user_id_dict[starrer] for starrer in issue_json['starrers']]
+
+ relations = []
+ relations.extend(
+ [(i, 'blockedon') for i in issue_json.get('blocked_on', [])])
+ relations.extend(
+ [(i, 'blocking') for i in issue_json.get('blocking', [])])
+ if 'merged_into' in issue_json:
+ relations.append((issue_json['merged_into'], 'mergedinto'))
+
+ return issue, comments, starrers, relations
+
+ def _ParseFieldValue(self, cnxn, project_id, user_id_dict, field_json):
+ field = tracker_pb2.FieldValue(
+ field_id=self.services.config.LookupFieldID(cnxn, project_id,
+ field_json['field']))
+ if 'int_value' in field_json:
+ field.int_value = field_json['int_value']
+ if 'str_value' in field_json:
+ field.str_value = field_json['str_value']
+ if 'user_value' in field_json:
+ field.user_value = user_id_dict.get(field_json['user_value'])
+
+ return field
+
+ def _ParseComment(self, project_id, user_id_dict, comment_json, event_log):
+ comment = tracker_pb2.IssueComment(
+ # Note: issue_id is filled in after the issue is saved.
+ project_id=project_id,
+ timestamp=comment_json['timestamp'],
+ user_id=user_id_dict[comment_json['commenter']],
+ content=comment_json.get('content'))
+
+ for amendment in comment_json['amendments']:
+ comment.amendments.append(
+ self._ParseAmendment(amendment, user_id_dict, event_log))
+
+ for attachment in comment_json['attachments']:
+ comment.attachments.append(
+ self._ParseAttachment(attachment, event_log))
+
+ if comment_json['description_num']:
+ comment.is_description = True
+
+ return comment
+
+ def _ParseAmendment(self, amendment_json, user_id_dict, _event_log):
+ amendment = tracker_pb2.Amendment(
+ field=tracker_pb2.FieldID(amendment_json['field']))
+
+ if 'new_value' in amendment_json:
+ amendment.newvalue = amendment_json['new_value']
+ if 'custom_field_name' in amendment_json:
+ amendment.custom_field_name = amendment_json['custom_field_name']
+ if 'added_users' in amendment_json:
+ amendment.added_user_ids.extend(
+ [user_id_dict[email] for email in amendment_json['added_users']])
+ if 'removed_users' in amendment_json:
+ amendment.removed_user_ids.extend(
+ [user_id_dict[email] for email in amendment_json['removed_users']])
+
+ return amendment
+
+ def _ParseAttachment(self, attachment_json, _event_log):
+ attachment = tracker_pb2.Attachment(
+ filename=attachment_json['name'],
+ filesize=attachment_json['size'],
+ mimetype=attachment_json['mimetype'],
+ gcs_object_id=attachment_json['gcs_object_id']
+ )
+ return attachment
+
+ def _SaveObjects(self, cnxn, project_id, state, event_log):
+ """Examine JSON data and create users, issues, and comments."""
+
+ created_user_ids = self.services.user.LookupUserIDs(
+ cnxn, state.nonexist_emails, autocreate=True)
+ for created_email, created_id in created_user_ids.items():
+ if created_id != state.user_id_dict[created_email]:
+ event_log.append('Mismatched user_id for %r' % created_email)
+ raise JSONImportError()
+ event_log.append('Created %d users' % len(state.nonexist_emails))
+
+ total_comments = 0
+ total_stars = 0
+ config = self.services.config.GetProjectConfig(cnxn, project_id)
+ for issue in state.issue_list:
+ # TODO(jrobbins): renumber issues if there is a local_id conflict.
+ if issue.local_id not in state.starrers_dict:
+ # Issues with stars will have filter rules applied in SetStar().
+ filterrules_helpers.ApplyFilterRules(
+ cnxn, self.services, issue, config)
+ issue_id = self.services.issue.InsertIssue(cnxn, issue)
+ for comment in state.comments_dict[issue.local_id]:
+ total_comments += 1
+ comment.issue_id = issue_id
+ self.services.issue.InsertComment(cnxn, comment)
+ self.services.issue_star.SetStarsBatch(
+ cnxn, self.services, config, issue_id,
+ state.starrers_dict[issue.local_id], True)
+ total_stars += len(state.starrers_dict[issue.local_id])
+
+ event_log.append('Created %d issues' % len(state.issue_list))
+ event_log.append('Created %d comments for %d issues' % (
+ total_comments, len(state.comments_dict)))
+ event_log.append('Set %d stars on %d issues' % (
+ total_stars, len(state.starrers_dict)))
+
+ global_relations_dict = collections.defaultdict(list)
+ for issue, rels in state.relations_dict.items():
+ src_iid = self.services.issue.GetIssueByLocalID(
+ cnxn, project_id, issue).issue_id
+ dst_iids = [i.issue_id for i in self.services.issue.GetIssuesByLocalIDs(
+ cnxn, project_id, [rel[0] for rel in rels])]
+ kinds = [rel[1] for rel in rels]
+ global_relations_dict[src_iid] = list(zip(dst_iids, kinds))
+ self.services.issue.RelateIssues(cnxn, global_relations_dict)
+
+ self.services.issue.SetUsedLocalID(cnxn, project_id)
+ event_log.append('Finished import')
+
+
+class JSONImportError(Exception):
+ """Exception to raise if imported JSON is invalid."""
+ pass