| # Copyright 2016 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style |
| # license that can be found in the LICENSE file or at |
| # https://developers.google.com/open-source/licenses/bsd |
| |
| """Servlet to import a file of issues in JSON format. |
| """ |
| from __future__ import print_function |
| from __future__ import division |
| from __future__ import absolute_import |
| |
| import collections |
| import json |
| import logging |
| import time |
| |
| import ezt |
| |
| from features import filterrules_helpers |
| from framework import flaskservlet |
| from framework import framework_helpers |
| from framework import jsonfeed |
| from framework import permissions |
| from framework import servlet |
| from framework import urls |
| from proto import tracker_pb2 |
| |
| |
| ParserState = collections.namedtuple( |
| 'ParserState', |
| 'user_id_dict, nonexist_emails, issue_list, comments_dict, starrers_dict, ' |
| 'relations_dict') |
| |
| |
| class IssueImport(servlet.Servlet): |
| """IssueImport loads a file of issues in JSON format.""" |
| |
| _PAGE_TEMPLATE = 'tracker/issue-import-page.ezt' |
| _MAIN_TAB_MODE = flaskservlet.FlaskServlet.MAIN_TAB_ISSUES |
| |
| def AssertBasePermission(self, mr): |
| """Make sure that the logged in user has permission to view this page.""" |
| super(IssueImport, self).AssertBasePermission(mr) |
| if not mr.auth.user_pb.is_site_admin: |
| raise permissions.PermissionException( |
| 'Only site admins may import issues') |
| |
| def GatherPageData(self, mr): |
| """Build up a dictionary of data values to use when rendering the page.""" |
| return { |
| 'issue_tab_mode': None, |
| 'page_perms': self.MakePagePerms(mr, None, permissions.CREATE_ISSUE), |
| 'import_errors': [], |
| } |
| |
| def ProcessFormData(self, mr, post_data): |
| """Process the issue entry form. |
| |
| Args: |
| mr: commonly used info parsed from the request. |
| post_data: The post_data dict for the current request. |
| |
| Returns: |
| String URL to redirect the user to after processing. |
| """ |
| import_errors = [] |
| json_data = None |
| |
| pre_check_only = 'pre_check_only' in post_data |
| |
| uploaded_file = post_data.get('jsonfile') |
| if uploaded_file is None: |
| import_errors.append('No file uploaded') |
| else: |
| try: |
| json_str = uploaded_file.value |
| if json_str.startswith(jsonfeed.XSSI_PREFIX): |
| json_str = json_str[len(jsonfeed.XSSI_PREFIX):] |
| json_data = json.loads(json_str) |
| except ValueError: |
| import_errors.append('error parsing JSON in file') |
| |
| if uploaded_file and not json_data: |
| import_errors.append('JSON file was empty') |
| |
| # Note that the project must already exist in order to even reach |
| # this servlet because it is hosted in the context of a project. |
| if json_data and mr.project_name != json_data['metadata']['project']: |
| import_errors.append( |
| 'Project name does not match. ' |
| 'Edit the file if you want to import into this project anyway.') |
| |
| if import_errors: |
| return self.PleaseCorrect(mr, import_errors=import_errors) |
| |
| event_log = [] # We accumulate a list of messages to display to the user. |
| |
| try: |
| # First we parse the JSON into objects, but we don't have DB IDs yet. |
| state = self._ParseObjects(mr.cnxn, mr.project_id, json_data, event_log) |
| # If that worked, go ahead and start saving the data to the DB. |
| if not pre_check_only: |
| self._SaveObjects(mr.cnxn, mr.project_id, state, event_log) |
| except JSONImportError: |
| # just report it to the user by displaying event_log |
| event_log.append('Aborted import processing') |
| |
| # This is a little bit of a hack because it always uses the form validation |
| # error message display logic to show the results of this import run, |
| # which may include errors or not. |
| return self.PleaseCorrect(mr, import_errors=event_log) |
| |
| def _ParseObjects(self, cnxn, project_id, json_data, event_log): |
| """Examine JSON data and return a parser state for further processing.""" |
| # Decide which users need to be created. |
| needed_emails = json_data['emails'] |
| user_id_dict = self.services.user.LookupExistingUserIDs(cnxn, needed_emails) |
| nonexist_emails = [email for email in needed_emails |
| if email not in user_id_dict] |
| |
| event_log.append('Need to create %d users: %r' % |
| (len(nonexist_emails), nonexist_emails)) |
| user_id_dict.update({ |
| email.lower(): framework_helpers.MurmurHash3_x86_32(email.lower()) |
| for email in nonexist_emails}) |
| |
| num_comments = 0 |
| num_stars = 0 |
| issue_list = [] |
| comments_dict = collections.defaultdict(list) |
| starrers_dict = collections.defaultdict(list) |
| relations_dict = collections.defaultdict(list) |
| for issue_json in json_data.get('issues', []): |
| issue, comment_list, starrer_list, relation_list = self._ParseIssue( |
| cnxn, project_id, user_id_dict, issue_json, event_log) |
| issue_list.append(issue) |
| comments_dict[issue.local_id] = comment_list |
| starrers_dict[issue.local_id] = starrer_list |
| relations_dict[issue.local_id] = relation_list |
| num_comments += len(comment_list) |
| num_stars += len(starrer_list) |
| |
| event_log.append( |
| 'Found info for %d issues: %r' % |
| (len(issue_list), sorted([issue.local_id for issue in issue_list]))) |
| |
| event_log.append( |
| 'Found %d total comments for %d issues' % |
| (num_comments, len(comments_dict))) |
| |
| event_log.append( |
| 'Found %d total stars for %d issues' % |
| (num_stars, len(starrers_dict))) |
| |
| event_log.append( |
| 'Found %d total relationships.' % |
| sum((len(dsts) for dsts in relations_dict.values()))) |
| |
| event_log.append('Parsing phase finished OK') |
| return ParserState( |
| user_id_dict, nonexist_emails, issue_list, |
| comments_dict, starrers_dict, relations_dict) |
| |
| def _ParseIssue(self, cnxn, project_id, user_id_dict, issue_json, event_log): |
| issue = tracker_pb2.Issue( |
| project_id=project_id, |
| local_id=issue_json['local_id'], |
| reporter_id=user_id_dict[issue_json['reporter']], |
| summary=issue_json['summary'], |
| opened_timestamp=issue_json['opened'], |
| modified_timestamp=issue_json['modified'], |
| cc_ids=[user_id_dict[cc_email] |
| for cc_email in issue_json.get('cc', []) |
| if cc_email in user_id_dict], |
| status=issue_json.get('status', ''), |
| labels=issue_json.get('labels', []), |
| field_values=[self._ParseFieldValue(cnxn, project_id, user_id_dict, field) |
| for field in issue_json.get('fields', [])]) |
| if issue_json.get('owner'): |
| issue.owner_id = user_id_dict[issue_json['owner']] |
| if issue_json.get('closed'): |
| issue.closed_timestamp = issue_json['closed'] |
| comments = [self._ParseComment( |
| project_id, user_id_dict, comment_json, event_log) |
| for comment_json in issue_json.get('comments', [])] |
| |
| starrers = [user_id_dict[starrer] for starrer in issue_json['starrers']] |
| |
| relations = [] |
| relations.extend( |
| [(i, 'blockedon') for i in issue_json.get('blocked_on', [])]) |
| relations.extend( |
| [(i, 'blocking') for i in issue_json.get('blocking', [])]) |
| if 'merged_into' in issue_json: |
| relations.append((issue_json['merged_into'], 'mergedinto')) |
| |
| return issue, comments, starrers, relations |
| |
| def _ParseFieldValue(self, cnxn, project_id, user_id_dict, field_json): |
| field = tracker_pb2.FieldValue( |
| field_id=self.services.config.LookupFieldID(cnxn, project_id, |
| field_json['field'])) |
| if 'int_value' in field_json: |
| field.int_value = field_json['int_value'] |
| if 'str_value' in field_json: |
| field.str_value = field_json['str_value'] |
| if 'user_value' in field_json: |
| field.user_value = user_id_dict.get(field_json['user_value']) |
| |
| return field |
| |
| def _ParseComment(self, project_id, user_id_dict, comment_json, event_log): |
| comment = tracker_pb2.IssueComment( |
| # Note: issue_id is filled in after the issue is saved. |
| project_id=project_id, |
| timestamp=comment_json['timestamp'], |
| user_id=user_id_dict[comment_json['commenter']], |
| content=comment_json.get('content')) |
| |
| for amendment in comment_json['amendments']: |
| comment.amendments.append( |
| self._ParseAmendment(amendment, user_id_dict, event_log)) |
| |
| for attachment in comment_json['attachments']: |
| comment.attachments.append( |
| self._ParseAttachment(attachment, event_log)) |
| |
| if comment_json['description_num']: |
| comment.is_description = True |
| |
| return comment |
| |
| def _ParseAmendment(self, amendment_json, user_id_dict, _event_log): |
| amendment = tracker_pb2.Amendment( |
| field=tracker_pb2.FieldID(amendment_json['field'])) |
| |
| if 'new_value' in amendment_json: |
| amendment.newvalue = amendment_json['new_value'] |
| if 'custom_field_name' in amendment_json: |
| amendment.custom_field_name = amendment_json['custom_field_name'] |
| if 'added_users' in amendment_json: |
| amendment.added_user_ids.extend( |
| [user_id_dict[email] for email in amendment_json['added_users']]) |
| if 'removed_users' in amendment_json: |
| amendment.removed_user_ids.extend( |
| [user_id_dict[email] for email in amendment_json['removed_users']]) |
| |
| return amendment |
| |
| def _ParseAttachment(self, attachment_json, _event_log): |
| attachment = tracker_pb2.Attachment( |
| filename=attachment_json['name'], |
| filesize=attachment_json['size'], |
| mimetype=attachment_json['mimetype'], |
| gcs_object_id=attachment_json['gcs_object_id'] |
| ) |
| return attachment |
| |
| def _SaveObjects(self, cnxn, project_id, state, event_log): |
| """Examine JSON data and create users, issues, and comments.""" |
| |
| created_user_ids = self.services.user.LookupUserIDs( |
| cnxn, state.nonexist_emails, autocreate=True) |
| for created_email, created_id in created_user_ids.items(): |
| if created_id != state.user_id_dict[created_email]: |
| event_log.append('Mismatched user_id for %r' % created_email) |
| raise JSONImportError() |
| event_log.append('Created %d users' % len(state.nonexist_emails)) |
| |
| total_comments = 0 |
| total_stars = 0 |
| config = self.services.config.GetProjectConfig(cnxn, project_id) |
| for issue in state.issue_list: |
| # TODO(jrobbins): renumber issues if there is a local_id conflict. |
| if issue.local_id not in state.starrers_dict: |
| # Issues with stars will have filter rules applied in SetStar(). |
| filterrules_helpers.ApplyFilterRules( |
| cnxn, self.services, issue, config) |
| issue_id = self.services.issue.InsertIssue(cnxn, issue) |
| for comment in state.comments_dict[issue.local_id]: |
| total_comments += 1 |
| comment.issue_id = issue_id |
| self.services.issue.InsertComment(cnxn, comment) |
| self.services.issue_star.SetStarsBatch( |
| cnxn, self.services, config, issue_id, |
| state.starrers_dict[issue.local_id], True) |
| total_stars += len(state.starrers_dict[issue.local_id]) |
| |
| event_log.append('Created %d issues' % len(state.issue_list)) |
| event_log.append('Created %d comments for %d issues' % ( |
| total_comments, len(state.comments_dict))) |
| event_log.append('Set %d stars on %d issues' % ( |
| total_stars, len(state.starrers_dict))) |
| |
| global_relations_dict = collections.defaultdict(list) |
| for issue, rels in state.relations_dict.items(): |
| src_iid = self.services.issue.GetIssueByLocalID( |
| cnxn, project_id, issue).issue_id |
| dst_iids = [i.issue_id for i in self.services.issue.GetIssuesByLocalIDs( |
| cnxn, project_id, [rel[0] for rel in rels])] |
| kinds = [rel[1] for rel in rels] |
| global_relations_dict[src_iid] = list(zip(dst_iids, kinds)) |
| self.services.issue.RelateIssues(cnxn, global_relations_dict) |
| |
| self.services.issue.SetUsedLocalID(cnxn, project_id) |
| event_log.append('Finished import') |
| |
| # def GetIssueImport(self, **kwargs): |
| # return self.handler(**kwargs) |
| |
| # def PostIssueImport(self, **kwargs): |
| # return self.handler(**kwargs) |
| |
| |
| class JSONImportError(Exception): |
| """Exception to raise if imported JSON is invalid.""" |
| pass |