| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style |
| # license that can be found in the LICENSE file or at |
| # https://developers.google.com/open-source/licenses/bsd |
| |
| """Functions that convert protorpc business objects into protoc objects. |
| |
| Monorail uses protorpc objects internally, whereas the API uses protoc |
| objects. The difference is not just the choice of protobuf library, there |
| will always be a need for conversion because out internal objects may have |
| field that we do not want to expose externally, or the format of some fields |
| may be different than how we want to expose them. |
| """ |
| from __future__ import print_function |
| from __future__ import division |
| from __future__ import absolute_import |
| |
| import logging |
| |
| from six import string_types |
| |
| import settings |
| from api.api_proto import common_pb2 |
| from api.api_proto import features_objects_pb2 |
| from api.api_proto import issue_objects_pb2 |
| from api.api_proto import project_objects_pb2 |
| from api.api_proto import user_objects_pb2 |
| from features import federated |
| from framework import exceptions |
| from framework import filecontent |
| from framework import framework_constants |
| from framework import framework_helpers |
| from framework import permissions |
| from framework import validate |
| from services import features_svc |
| from tracker import attachment_helpers |
| from tracker import field_helpers |
| from tracker import tracker_bizobj |
| from tracker import tracker_helpers |
| from proto import tracker_pb2 |
| from proto import user_pb2 |
| |
| |
| # Convert and ingest objects in issue_objects.proto. |
| |
| |
| def ConvertApprovalValues(approval_values, phases, users_by_id, config): |
| """Convert a list of ApprovalValue into protoc Approvals.""" |
| phases_by_id = { |
| phase.phase_id: phase |
| for phase in phases} |
| result = [ |
| ConvertApproval( |
| av, users_by_id, config, phase=phases_by_id.get(av.phase_id)) |
| for av in approval_values] |
| result = [av for av in result if av] |
| return result |
| |
| |
| def ConvertApproval(approval_value, users_by_id, config, phase=None): |
| """Use the given ApprovalValue to create a protoc Approval.""" |
| approval_name = '' |
| fd = tracker_bizobj.FindFieldDefByID(approval_value.approval_id, config) |
| if fd: |
| approval_name = fd.field_name |
| else: |
| logging.info( |
| 'Ignoring approval value referencing a non-existing field: %r', |
| approval_value) |
| return None |
| |
| field_ref = ConvertFieldRef( |
| approval_value.approval_id, approval_name, |
| tracker_pb2.FieldTypes.APPROVAL_TYPE, None) |
| approver_refs = ConvertUserRefs(approval_value.approver_ids, [], users_by_id, |
| False) |
| setter_ref = ConvertUserRef(approval_value.setter_id, None, users_by_id) |
| |
| status = ConvertApprovalStatus(approval_value.status) |
| set_on = approval_value.set_on |
| |
| phase_ref = issue_objects_pb2.PhaseRef() |
| if phase: |
| phase_ref.phase_name = phase.name |
| |
| result = issue_objects_pb2.Approval( |
| field_ref=field_ref, approver_refs=approver_refs, |
| status=status, set_on=set_on, setter_ref=setter_ref, |
| phase_ref=phase_ref) |
| return result |
| |
| |
| def ConvertStatusRef(explicit_status, derived_status, config): |
| """Use the given status strings to create a StatusRef.""" |
| status = explicit_status or derived_status |
| is_derived = not explicit_status |
| if not status: |
| return common_pb2.StatusRef( |
| status=framework_constants.NO_VALUES, is_derived=False, means_open=True) |
| |
| return common_pb2.StatusRef( |
| status=status, |
| is_derived=is_derived, |
| means_open=tracker_helpers.MeansOpenInProject(status, config)) |
| |
| |
| def ConvertApprovalStatus(status): |
| """Use the given protorpc ApprovalStatus to create a protoc ApprovalStatus""" |
| return issue_objects_pb2.ApprovalStatus.Value(status.name) |
| |
| |
| def ConvertUserRef(explicit_user_id, derived_user_id, users_by_id): |
| """Use the given user IDs to create a UserRef.""" |
| user_id = explicit_user_id or derived_user_id |
| is_derived = not explicit_user_id |
| if not user_id: |
| return None; |
| |
| return common_pb2.UserRef( |
| user_id=user_id, |
| is_derived=is_derived, |
| display_name=users_by_id[user_id].display_name) |
| |
| # TODO(jojwang): Rewrite method, ConvertUserRefs should be able to |
| # call ConvertUserRef |
| def ConvertUserRefs(explicit_user_ids, derived_user_ids, users_by_id, |
| use_email): |
| # (List(int), List(int), Dict(int: UserView), bool) -> List(UserRef) |
| """Use the given user ID lists to create a list of UserRef. |
| |
| Args: |
| explicit_user_ids: list of user_ids for users that are not derived. |
| derived_user_ids: list of user_ids for users derived from FilterRules. |
| users_by_id: dict of {user_id: UserView, ...} for all users in |
| explicit_user_ids and derived_user_ids. |
| use_email: boolean true if the UserView.email should be used as |
| the display_name instead of UserView.display_name, which may be obscured. |
| |
| Returns: |
| A single list of UserRefs. |
| """ |
| result = [] |
| for user_id in explicit_user_ids: |
| result.append(common_pb2.UserRef( |
| user_id=user_id, |
| is_derived=False, |
| display_name=( |
| users_by_id[user_id].email |
| if use_email |
| else users_by_id[user_id].display_name))) |
| for user_id in derived_user_ids: |
| result.append(common_pb2.UserRef( |
| user_id=user_id, |
| is_derived=True, |
| display_name=( |
| users_by_id[user_id].email |
| if use_email |
| else users_by_id[user_id].display_name))) |
| return result |
| |
| |
| def ConvertUsers(users, users_by_id): |
| """Use the given protorpc Users to create protoc Users. |
| |
| Args: |
| users: list of protorpc Users to convert. |
| users_by_id: dict {user_id: UserView} of all Users linked |
| from the users list. |
| |
| Returns: |
| A list of protoc Users. |
| """ |
| result = [] |
| for user in users: |
| linked_parent_ref = None |
| if user.linked_parent_id: |
| linked_parent_ref = ConvertUserRefs( |
| [user.linked_parent_id], [], users_by_id, False)[0] |
| linked_child_refs = ConvertUserRefs( |
| user.linked_child_ids, [], users_by_id, False) |
| converted_user = user_objects_pb2.User( |
| user_id=user.user_id, |
| display_name=user.email, |
| is_site_admin=user.is_site_admin, |
| availability=framework_helpers.GetUserAvailability(user)[0], |
| linked_parent_ref=linked_parent_ref, |
| linked_child_refs=linked_child_refs) |
| result.append(converted_user) |
| return result |
| |
| |
| def ConvertPrefValues(userprefvalues): |
| """Convert a list of protorpc UserPrefValue to protoc UserPrefValues.""" |
| return [ |
| user_objects_pb2.UserPrefValue(name=upv.name, value=upv.value) |
| for upv in userprefvalues] |
| |
| |
| def ConvertLabels(explicit_labels, derived_labels): |
| """Combine the given explicit and derived lists into LabelRefs.""" |
| explicit_refs = [common_pb2.LabelRef(label=lab, is_derived=False) |
| for lab in explicit_labels] |
| derived_refs = [common_pb2.LabelRef(label=lab, is_derived=True) |
| for lab in derived_labels] |
| return explicit_refs + derived_refs |
| |
| |
| def ConvertComponentRef(component_id, config, is_derived=False): |
| """Make a ComponentRef from the component_id and project config.""" |
| component_def = tracker_bizobj.FindComponentDefByID(component_id, config) |
| if not component_def: |
| logging.info('Ignoring non-existing component id %s', component_id) |
| return None |
| result = common_pb2.ComponentRef( |
| path=component_def.path, |
| is_derived=is_derived) |
| return result |
| |
| # TODO(jojwang): rename to ConvertComponentRefs |
| def ConvertComponents(explicit_component_ids, derived_component_ids, config): |
| """Make a ComponentRef for each component_id.""" |
| result = [ConvertComponentRef(cid, config) for cid in explicit_component_ids] |
| result += [ |
| ConvertComponentRef(cid, config, is_derived=True) |
| for cid in derived_component_ids] |
| result = [cr for cr in result if cr] |
| return result |
| |
| |
| def ConvertIssueRef(issue_ref_pair, ext_id=''): |
| """Convert (project_name, local_id) to an IssueRef protoc object. |
| |
| With optional external ref in ext_id. |
| """ |
| project_name, local_id = issue_ref_pair |
| ref = common_pb2.IssueRef(project_name=project_name, local_id=local_id, |
| ext_identifier=ext_id) |
| return ref |
| |
| |
| def ConvertIssueRefs(issue_ids, related_refs_dict): |
| """Convert a list of iids to IssueRef protoc objects.""" |
| return [ConvertIssueRef(related_refs_dict[iid]) for iid in issue_ids] |
| |
| |
| def ConvertFieldValue(field_id, field_name, value, field_type, |
| approval_name=None, phase_name=None, is_derived=False): |
| """Convert one field value view item into a protoc FieldValue.""" |
| if not isinstance(value, string_types): |
| value = str(value) |
| fv = issue_objects_pb2.FieldValue( |
| field_ref=ConvertFieldRef(field_id, field_name, field_type, |
| approval_name), |
| value=value, |
| is_derived=is_derived) |
| if phase_name: |
| fv.phase_ref.phase_name = phase_name |
| |
| return fv |
| |
| |
| def ConvertFieldType(field_type): |
| """Use the given protorpc FieldTypes enum to create a protoc FieldType.""" |
| return common_pb2.FieldType.Value(field_type.name) |
| |
| |
| def ConvertFieldRef(field_id, field_name, field_type, approval_name): |
| """Convert a field name and protorpc FieldType into a protoc FieldRef.""" |
| return common_pb2.FieldRef(field_id=field_id, |
| field_name=field_name, |
| type=ConvertFieldType(field_type), |
| approval_name=approval_name) |
| |
| |
| def ConvertFieldValues( |
| config, labels, derived_labels, field_values, users_by_id, phases=None): |
| """Convert lists of labels and field_values to protoc FieldValues.""" |
| fvs = [] |
| phase_names_by_id = {phase.phase_id: phase.name for phase in phases or []} |
| fds_by_id = {fd.field_id:fd for fd in config.field_defs} |
| fids_by_name = {fd.field_name:fd.field_id for fd in config.field_defs} |
| enum_names_by_lower = { |
| fd.field_name.lower(): fd.field_name for fd in config.field_defs |
| if fd.field_type == tracker_pb2.FieldTypes.ENUM_TYPE} |
| |
| labels_by_prefix = tracker_bizobj.LabelsByPrefix( |
| labels, list(enum_names_by_lower.keys())) |
| der_labels_by_prefix = tracker_bizobj.LabelsByPrefix( |
| derived_labels, list(enum_names_by_lower.keys())) |
| |
| for lower_field_name, values in labels_by_prefix.items(): |
| field_name = enum_names_by_lower.get(lower_field_name) |
| if not field_name: |
| continue |
| fvs.extend( |
| [ConvertFieldValue( |
| fids_by_name.get(field_name), field_name, value, |
| tracker_pb2.FieldTypes.ENUM_TYPE) |
| for value in values]) |
| |
| for lower_field_name, values in der_labels_by_prefix.items(): |
| field_name = enum_names_by_lower.get(lower_field_name) |
| if not field_name: |
| continue |
| fvs.extend( |
| [ConvertFieldValue( |
| fids_by_name.get(field_name), field_name, value, |
| tracker_pb2.FieldTypes.ENUM_TYPE, is_derived=True) |
| for value in values]) |
| |
| for fv in field_values: |
| field_def = fds_by_id.get(fv.field_id) |
| if not field_def: |
| logging.info( |
| 'Ignoring field value referencing a non-existent field: %r', fv) |
| continue |
| |
| value = tracker_bizobj.GetFieldValue(fv, users_by_id) |
| field_name = field_def.field_name |
| field_type = field_def.field_type |
| approval_name = None |
| |
| if field_def.approval_id is not None: |
| approval_def = fds_by_id.get(field_def.approval_id) |
| if approval_def: |
| approval_name = approval_def.field_name |
| |
| fvs.append(ConvertFieldValue( |
| fv.field_id, field_name, value, field_type, approval_name=approval_name, |
| phase_name=phase_names_by_id.get(fv.phase_id), is_derived=fv.derived)) |
| |
| return fvs |
| |
| |
| def ConvertIssue(issue, users_by_id, related_refs, config): |
| """Convert our protorpc Issue to a protoc Issue. |
| |
| Args: |
| issue: protorpc issue used by monorail internally. |
| users_by_id: dict {user_id: UserViews} for all users mentioned in issue. |
| related_refs: dict {issue_id: (project_name, local_id)} of all blocked-on, |
| blocking, or merged-into issues referenced from this issue, regardless |
| of perms. |
| config: ProjectIssueConfig for this issue. |
| |
| Returns: A protoc Issue object. |
| """ |
| status_ref = ConvertStatusRef(issue.status, issue.derived_status, config) |
| owner_ref = ConvertUserRef( |
| issue.owner_id, issue.derived_owner_id, users_by_id) |
| cc_refs = ConvertUserRefs( |
| issue.cc_ids, issue.derived_cc_ids, users_by_id, False) |
| labels, derived_labels = tracker_bizobj.ExplicitAndDerivedNonMaskedLabels( |
| issue.labels, issue.derived_labels, config) |
| label_refs = ConvertLabels(labels, derived_labels) |
| component_refs = ConvertComponents( |
| issue.component_ids, issue.derived_component_ids, config) |
| blocked_on_issue_refs = ConvertIssueRefs( |
| issue.blocked_on_iids, related_refs) |
| dangling_blocked_on_refs = [ |
| ConvertIssueRef((dangling_issue.project, dangling_issue.issue_id), |
| ext_id=dangling_issue.ext_issue_identifier) |
| for dangling_issue in issue.dangling_blocked_on_refs] |
| blocking_issue_refs = ConvertIssueRefs( |
| issue.blocking_iids, related_refs) |
| dangling_blocking_refs = [ |
| ConvertIssueRef((dangling_issue.project, dangling_issue.issue_id), |
| ext_id=dangling_issue.ext_issue_identifier) |
| for dangling_issue in issue.dangling_blocking_refs] |
| merged_into_issue_ref = None |
| if issue.merged_into: |
| merged_into_issue_ref = ConvertIssueRef(related_refs[issue.merged_into]) |
| if issue.merged_into_external: |
| merged_into_issue_ref = ConvertIssueRef((None, None), |
| ext_id=issue.merged_into_external) |
| |
| field_values = ConvertFieldValues( |
| config, issue.labels, issue.derived_labels, |
| issue.field_values, users_by_id, phases=issue.phases) |
| approval_values = ConvertApprovalValues( |
| issue.approval_values, issue.phases, users_by_id, config) |
| reporter_ref = ConvertUserRef(issue.reporter_id, None, users_by_id) |
| phases = [ConvertPhaseDef(phase) for phase in issue.phases] |
| result = issue_objects_pb2.Issue( |
| project_name=issue.project_name, local_id=issue.local_id, |
| summary=issue.summary, status_ref=status_ref, owner_ref=owner_ref, |
| cc_refs=cc_refs, label_refs=label_refs, component_refs=component_refs, |
| blocked_on_issue_refs=blocked_on_issue_refs, |
| dangling_blocked_on_refs=dangling_blocked_on_refs, |
| blocking_issue_refs=blocking_issue_refs, |
| dangling_blocking_refs=dangling_blocking_refs, |
| merged_into_issue_ref=merged_into_issue_ref, field_values=field_values, |
| is_deleted=issue.deleted, reporter_ref=reporter_ref, |
| opened_timestamp=issue.opened_timestamp, |
| closed_timestamp=issue.closed_timestamp, |
| modified_timestamp=issue.modified_timestamp, |
| component_modified_timestamp=issue.component_modified_timestamp, |
| status_modified_timestamp=issue.status_modified_timestamp, |
| owner_modified_timestamp=issue.owner_modified_timestamp, |
| star_count=issue.star_count, is_spam=issue.is_spam, |
| approval_values=approval_values, phases=phases) |
| |
| # TODO(crbug.com/monorail/5857): Set attachment_count unconditionally |
| # after the underlying source of negative attachment counts has been |
| # resolved and database has been repaired. |
| if issue.attachment_count >= 0: |
| result.attachment_count = issue.attachment_count |
| |
| return result |
| |
| |
| def ConvertPhaseDef(phase): |
| """Convert a protorpc Phase to a protoc PhaseDef.""" |
| phase_def = issue_objects_pb2.PhaseDef( |
| phase_ref=issue_objects_pb2.PhaseRef(phase_name=phase.name), |
| rank=phase.rank) |
| return phase_def |
| |
| |
| def ConvertAmendment(amendment, users_by_id): |
| """Convert a protorpc Amendment to a protoc Amendment.""" |
| field_name = tracker_bizobj.GetAmendmentFieldName(amendment) |
| new_value = tracker_bizobj.AmendmentString(amendment, users_by_id) |
| result = issue_objects_pb2.Amendment( |
| field_name=field_name, new_or_delta_value=new_value, |
| old_value=amendment.oldvalue) |
| return result |
| |
| |
| def ConvertAttachment(attach, project_name): |
| """Convert a protorpc Attachment to a protoc Attachment.""" |
| size, thumbnail_url, view_url, download_url = None, None, None, None |
| if not attach.deleted: |
| size = attach.filesize |
| download_url = attachment_helpers.GetDownloadURL(attach.attachment_id) |
| view_url = attachment_helpers.GetViewURL(attach, download_url, project_name) |
| thumbnail_url = attachment_helpers.GetThumbnailURL(attach, download_url) |
| |
| result = issue_objects_pb2.Attachment( |
| attachment_id=attach.attachment_id, filename=attach.filename, |
| size=size, content_type=attach.mimetype, |
| is_deleted=attach.deleted, thumbnail_url=thumbnail_url, |
| view_url=view_url, download_url=download_url) |
| return result |
| |
| |
| def ConvertComment( |
| issue, comment, config, users_by_id, comment_reporters, description_nums, |
| user_id, perms): |
| """Convert a protorpc IssueComment to a protoc Comment.""" |
| commenter = users_by_id[comment.user_id] |
| |
| can_delete = permissions.CanDeleteComment( |
| comment, commenter, user_id, perms) |
| can_flag, is_flagged = permissions.CanFlagComment( |
| comment, commenter, comment_reporters, user_id, perms) |
| can_view = permissions.CanViewComment( |
| comment, commenter, user_id, perms) |
| can_view_inbound_message = permissions.CanViewInboundMessage( |
| comment, user_id, perms) |
| |
| is_deleted = bool(comment.deleted_by or is_flagged or commenter.banned) |
| |
| result = issue_objects_pb2.Comment( |
| project_name=issue.project_name, |
| local_id=issue.local_id, |
| sequence_num=comment.sequence, |
| is_deleted=is_deleted, |
| can_delete=can_delete, |
| is_spam=is_flagged, |
| can_flag=can_flag, |
| timestamp=comment.timestamp) |
| |
| if can_view: |
| result.commenter.CopyFrom( |
| ConvertUserRef(comment.user_id, None, users_by_id)) |
| result.content = comment.content |
| if comment.inbound_message and can_view_inbound_message: |
| result.inbound_message = comment.inbound_message |
| result.amendments.extend([ |
| ConvertAmendment(amend, users_by_id) |
| for amend in comment.amendments]) |
| result.attachments.extend([ |
| ConvertAttachment(attach, issue.project_name) |
| for attach in comment.attachments]) |
| |
| if comment.id in description_nums: |
| result.description_num = description_nums[comment.id] |
| |
| fd = tracker_bizobj.FindFieldDefByID(comment.approval_id, config) |
| if fd: |
| result.approval_ref.field_name = fd.field_name |
| |
| return result |
| |
| |
| def ConvertCommentList( |
| issue, comments, config, users_by_id, comment_reporters, user_id, perms): |
| """Convert a list of protorpc IssueComments to protoc Comments.""" |
| description_nums = {} |
| for comment in comments: |
| if (comment.is_description and not users_by_id[comment.user_id].banned and |
| not comment.deleted_by and not comment.is_spam): |
| description_nums[comment.id] = len(description_nums) + 1 |
| |
| result = [ |
| ConvertComment( |
| issue, c, config, users_by_id, comment_reporters.get(c.id, []), |
| description_nums, user_id, perms) |
| for c in comments] |
| return result |
| |
| |
| def IngestUserRef(cnxn, user_ref, user_service, autocreate=False): |
| """Return ID of specified user or raise NoSuchUserException.""" |
| try: |
| return IngestUserRefs( |
| cnxn, [user_ref], user_service, autocreate=autocreate)[0] |
| except IndexError: |
| # user_ref.display_name was not a valid email. |
| raise exceptions.NoSuchUserException |
| |
| |
| def IngestUserRefs(cnxn, user_refs, user_service, autocreate=False): |
| """Return IDs of specified users or raise NoSuchUserException.""" |
| |
| # Filter out user_refs with invalid display_names. |
| # Invalid emails won't get auto-created in LookupUserIds, but un-specified |
| # user_ref.user_id values have the zero-value 0. So invalid user_ref's |
| # need to be filtered out here to prevent these resulting in '0's in |
| # the 'result' array. |
| if autocreate: |
| user_refs = [user_ref for user_ref in user_refs |
| if (not user_ref.display_name) or |
| validate.IsValidEmail(user_ref.display_name)] |
| |
| # 1. Verify that all specified user IDs actually match existing users. |
| user_ids_to_verify = [user_ref.user_id for user_ref in user_refs |
| if user_ref.user_id] |
| user_service.LookupUserEmails(cnxn, user_ids_to_verify) |
| |
| # 2. Lookup or create any users that are specified by email address. |
| needed_emails = [user_ref.display_name for user_ref in user_refs |
| if not user_ref.user_id and user_ref.display_name] |
| emails_to_ids = user_service.LookupUserIDs( |
| cnxn, needed_emails, autocreate=autocreate) |
| |
| # 3. Build the result from emails_to_ids or straight from user_ref's |
| # user_id. |
| # Note: user_id can be specified as 0 to clear the issue owner. |
| result = [ |
| emails_to_ids.get(user_ref.display_name.lower(), user_ref.user_id) |
| for user_ref in user_refs |
| ] |
| return result |
| |
| |
| def IngestPrefValues(pref_values): |
| """Return protorpc UserPrefValues for the given values.""" |
| return [user_pb2.UserPrefValue(name=upv.name, value=upv.value) |
| for upv in pref_values] |
| |
| |
| def IngestComponentRefs(comp_refs, config, ignore_missing_objects=False): |
| """Return IDs of specified components or raise NoSuchComponentException.""" |
| cids_by_path = {cd.path.lower(): cd.component_id |
| for cd in config.component_defs} |
| result = [] |
| for comp_ref in comp_refs: |
| cid = cids_by_path.get(comp_ref.path.lower()) |
| if cid: |
| result.append(cid) |
| else: |
| if not ignore_missing_objects: |
| raise exceptions.NoSuchComponentException() |
| return result |
| |
| |
| def IngestFieldRefs(field_refs, config): |
| """Return IDs of specified fields or raise NoSuchFieldDefException.""" |
| fids_by_name = {fd.field_name.lower(): fd.field_id |
| for fd in config.field_defs} |
| result = [] |
| for field_ref in field_refs: |
| fid = fids_by_name.get(field_ref.field_name.lower()) |
| if fid: |
| result.append(fid) |
| else: |
| raise exceptions.NoSuchFieldDefException() |
| return result |
| |
| |
| def IngestIssueRefs(cnxn, issue_refs, services): |
| """Look up issue IDs for the specified issues.""" |
| project_names = set(ref.project_name for ref in issue_refs) |
| project_names_to_id = services.project.LookupProjectIDs(cnxn, project_names) |
| project_local_id_pairs = [] |
| for ref in issue_refs: |
| if ref.ext_identifier: |
| # TODO(jeffcarp): For external tracker refs, once we have the classes |
| # set up, validate that the tracker for this specific ref is supported |
| # and store the external ref in the issue properly. |
| if '/' not in ref.ext_identifier: |
| raise exceptions.InvalidExternalIssueReference() |
| continue |
| if ref.project_name in project_names_to_id: |
| pair = (project_names_to_id[ref.project_name], ref.local_id) |
| project_local_id_pairs.append(pair) |
| else: |
| raise exceptions.NoSuchProjectException() |
| issue_ids, misses = services.issue.LookupIssueIDs( |
| cnxn, project_local_id_pairs) |
| if misses: |
| raise exceptions.NoSuchIssueException() |
| return issue_ids |
| |
| |
| def IngestExtIssueRefs(issue_refs): |
| """Validate and return external issue refs.""" |
| return [ |
| ref.ext_identifier |
| for ref in issue_refs |
| if ref.ext_identifier |
| and federated.IsShortlinkValid(ref.ext_identifier)] |
| |
| |
| def IngestIssueDelta( |
| cnxn, services, delta, config, phases, ignore_missing_objects=False): |
| """Ingest a protoc IssueDelta and create a protorpc IssueDelta.""" |
| status = None |
| if delta.HasField('status'): |
| status = delta.status.value |
| owner_id = None |
| if delta.HasField('owner_ref'): |
| try: |
| owner_id = IngestUserRef(cnxn, delta.owner_ref, services.user) |
| except exceptions.NoSuchUserException as e: |
| if not ignore_missing_objects: |
| raise e |
| summary = None |
| if delta.HasField('summary'): |
| summary = delta.summary.value |
| |
| cc_ids_add = IngestUserRefs( |
| cnxn, delta.cc_refs_add, services.user, autocreate=True) |
| cc_ids_remove = IngestUserRefs(cnxn, delta.cc_refs_remove, services.user) |
| |
| comp_ids_add = IngestComponentRefs( |
| delta.comp_refs_add, config, |
| ignore_missing_objects=ignore_missing_objects) |
| comp_ids_remove = IngestComponentRefs( |
| delta.comp_refs_remove, config, |
| ignore_missing_objects=ignore_missing_objects) |
| |
| labels_add = [lab_ref.label for lab_ref in delta.label_refs_add] |
| labels_remove = [lab_ref.label for lab_ref in delta.label_refs_remove] |
| |
| field_vals_add, field_vals_remove = _RedistributeEnumFieldsIntoLabels( |
| labels_add, labels_remove, |
| delta.field_vals_add, delta.field_vals_remove, |
| config) |
| |
| field_vals_add = IngestFieldValues( |
| cnxn, services.user, field_vals_add, config, phases=phases) |
| field_vals_remove = IngestFieldValues( |
| cnxn, services.user, field_vals_remove, config, phases=phases) |
| fields_clear = IngestFieldRefs(delta.fields_clear, config) |
| |
| # Ingest intra-tracker issue refs. |
| blocked_on_add = IngestIssueRefs( |
| cnxn, delta.blocked_on_refs_add, services) |
| blocked_on_remove = IngestIssueRefs( |
| cnxn, delta.blocked_on_refs_remove, services) |
| blocking_add = IngestIssueRefs( |
| cnxn, delta.blocking_refs_add, services) |
| blocking_remove = IngestIssueRefs( |
| cnxn, delta.blocking_refs_remove, services) |
| |
| # Ingest inter-tracker issue refs. |
| ext_blocked_on_add = IngestExtIssueRefs(delta.blocked_on_refs_add) |
| ext_blocked_on_remove = IngestExtIssueRefs(delta.blocked_on_refs_remove) |
| ext_blocking_add = IngestExtIssueRefs(delta.blocking_refs_add) |
| ext_blocking_remove = IngestExtIssueRefs(delta.blocking_refs_remove) |
| |
| merged_into = None |
| merged_into_external = None |
| if delta.HasField('merged_into_ref'): |
| if delta.merged_into_ref.ext_identifier: # Adding an external merged. |
| merged_into_external = delta.merged_into_ref.ext_identifier |
| elif not delta.merged_into_ref.local_id: # Clearing an internal merged. |
| merged_into = 0 |
| else: # Adding an internal merged. |
| merged_into = IngestIssueRefs(cnxn, [delta.merged_into_ref], services)[0] |
| |
| result = tracker_bizobj.MakeIssueDelta( |
| status, owner_id, cc_ids_add, cc_ids_remove, comp_ids_add, |
| comp_ids_remove, labels_add, labels_remove, field_vals_add, |
| field_vals_remove, fields_clear, blocked_on_add, blocked_on_remove, |
| blocking_add, blocking_remove, merged_into, summary, |
| ext_blocked_on_add=ext_blocked_on_add, |
| ext_blocked_on_remove=ext_blocked_on_remove, |
| ext_blocking_add=ext_blocking_add, |
| ext_blocking_remove=ext_blocking_remove, |
| merged_into_external=merged_into_external) |
| return result |
| |
| def IngestAttachmentUploads(attachment_uploads): |
| """Ingest protoc AttachmentUpload objects as tuples.""" |
| result = [] |
| for up in attachment_uploads: |
| if not up.filename: |
| raise exceptions.InputException('Missing attachment name') |
| if not up.content: |
| raise exceptions.InputException('Missing attachment content') |
| mimetype = filecontent.GuessContentTypeFromFilename(up.filename) |
| attachment_tuple = (up.filename, up.content, mimetype) |
| result.append(attachment_tuple) |
| return result |
| |
| |
| def IngestApprovalDelta(cnxn, user_service, approval_delta, setter_id, config): |
| """Ingest a protoc ApprovalDelta and create a protorpc ApprovalDelta.""" |
| fids_by_name = {fd.field_name.lower(): fd.field_id for |
| fd in config.field_defs} |
| |
| approver_ids_add = IngestUserRefs( |
| cnxn, approval_delta.approver_refs_add, user_service, autocreate=True) |
| approver_ids_remove = IngestUserRefs( |
| cnxn, approval_delta.approver_refs_remove, user_service, autocreate=True) |
| |
| labels_add, labels_remove = [], [] |
| # TODO(jojwang): monorail:4673, validate enum values all belong to approval. |
| field_vals_add, field_vals_remove = _RedistributeEnumFieldsIntoLabels( |
| labels_add, labels_remove, |
| approval_delta.field_vals_add, approval_delta.field_vals_remove, |
| config) |
| |
| sub_fvs_add = IngestFieldValues(cnxn, user_service, field_vals_add, config) |
| sub_fvs_remove = IngestFieldValues( |
| cnxn, user_service, field_vals_remove, config) |
| sub_fields_clear = [fids_by_name.get(clear.field_name.lower()) for |
| clear in approval_delta.fields_clear |
| if clear.field_name.lower() in fids_by_name] |
| |
| # protoc ENUMs default to the zero value (in this case: NOT_SET). |
| # NOT_SET should only be allowed when an issue is first created. |
| # Once a user changes it to something else, no one should be allowed |
| # to set it back. |
| status = None |
| if approval_delta.status != issue_objects_pb2.NOT_SET: |
| status = IngestApprovalStatus(approval_delta.status) |
| |
| return tracker_bizobj.MakeApprovalDelta( |
| status, setter_id, approver_ids_add, approver_ids_remove, |
| sub_fvs_add, sub_fvs_remove, sub_fields_clear, labels_add, labels_remove) |
| |
| |
| def IngestApprovalStatus(approval_status): |
| """Ingest a protoc ApprovalStatus and create a protorpc ApprovalStatus. """ |
| if approval_status == issue_objects_pb2.NOT_SET: |
| return tracker_pb2.ApprovalStatus.NOT_SET |
| return tracker_pb2.ApprovalStatus(approval_status) |
| |
| |
| def IngestFieldValues(cnxn, user_service, field_values, config, phases=None): |
| """Ingest a list of protoc FieldValues and create protorpc FieldValues. |
| |
| Args: |
| cnxn: connection to the DB. |
| user_service: interface to user data storage. |
| field_values: a list of protoc FieldValue used by the API. |
| config: ProjectIssueConfig for this field_value's project. |
| phases: a list of the issue's protorpc Phases. |
| |
| |
| Returns: A protorpc FieldValue object. |
| """ |
| fds_by_name = {fd.field_name.lower(): fd for fd in config.field_defs} |
| phases_by_name = {phase.name: phase.phase_id for phase in phases or []} |
| |
| ingested_fvs = [] |
| for fv in field_values: |
| fd = fds_by_name.get(fv.field_ref.field_name.lower()) |
| if fd: |
| if not fv.value: |
| logging.info('Ignoring blank field value: %r', fv) |
| continue |
| ingested_fv = field_helpers.ParseOneFieldValue( |
| cnxn, user_service, fd, fv.value) |
| if not ingested_fv: |
| raise exceptions.InputException( |
| 'Unparsable value for field %s' % fv.field_ref.field_name) |
| if ingested_fv.user_id == field_helpers.INVALID_USER_ID: |
| raise exceptions.NoSuchUserException() |
| if fd.is_phase_field: |
| ingested_fv.phase_id = phases_by_name.get(fv.phase_ref.phase_name) |
| ingested_fvs.append(ingested_fv) |
| |
| return ingested_fvs |
| |
| |
| def IngestSavedQueries(cnxn, project_service, saved_queries): |
| """Ingest a list of protoc SavedQuery and create protorpc SavedQuery. |
| |
| Args: |
| cnxn: connection to the DB. |
| project_service: interface to project data storage. |
| saved_queries: a list of protoc Savedquery. |
| |
| Returns: A protorpc SavedQuery object. |
| """ |
| if not saved_queries: |
| return [] |
| |
| project_ids = set() |
| for sq in saved_queries: |
| project_ids.update(sq.executes_in_project_ids) |
| |
| project_name_dict = project_service.LookupProjectNames(cnxn, |
| project_ids) |
| return [ |
| common_pb2.SavedQuery( |
| query_id=sq.query_id, |
| name=sq.name, |
| query=sq.query, |
| project_names=[project_name_dict[project_id] |
| for project_id in sq.executes_in_project_ids] |
| ) |
| for sq in saved_queries] |
| |
| |
| def IngestHotlistRefs(cnxn, user_service, features_service, hotlist_refs): |
| return [IngestHotlistRef(cnxn, user_service, features_service, hotlist_ref) |
| for hotlist_ref in hotlist_refs] |
| |
| |
| def IngestHotlistRef(cnxn, user_service, features_service, hotlist_ref): |
| hotlist_id = None |
| |
| if hotlist_ref.hotlist_id: |
| # If a hotlist ID was specified, verify it actually match existing hotlists. |
| features_service.GetHotlist(cnxn, hotlist_ref.hotlist_id) |
| hotlist_id = hotlist_ref.hotlist_id |
| |
| if hotlist_ref.name and hotlist_ref.owner: |
| name = hotlist_ref.name |
| owner_id = IngestUserRef(cnxn, hotlist_ref.owner, user_service) |
| hotlists = features_service.LookupHotlistIDs(cnxn, [name], [owner_id]) |
| # Verify there is a hotlist with that name and owner. |
| if (name.lower(), owner_id) not in hotlists: |
| raise features_svc.NoSuchHotlistException() |
| found_id = hotlists[name.lower(), owner_id] |
| # If a hotlist_id was also given, verify it correspond to the name and |
| # owner. |
| if hotlist_id is not None and found_id != hotlist_id: |
| raise features_svc.NoSuchHotlistException() |
| hotlist_id = found_id |
| |
| # Neither an ID, nor a name-owner ref were given. |
| if hotlist_id is None: |
| raise features_svc.NoSuchHotlistException() |
| |
| return hotlist_id |
| |
| |
| def IngestPagination(pagination): |
| max_items = settings.max_artifact_search_results_per_page |
| if pagination.max_items: |
| max_items = min(max_items, pagination.max_items) |
| return pagination.start, max_items |
| |
| # Convert and ingest objects in project_objects.proto. |
| |
| def ConvertStatusDef(status_def): |
| """Convert a protorpc StatusDef into a protoc StatusDef.""" |
| result = project_objects_pb2.StatusDef( |
| status=status_def.status, |
| means_open=status_def.means_open, |
| docstring=status_def.status_docstring, |
| deprecated=status_def.deprecated) |
| return result |
| |
| |
| def ConvertLabelDef(label_def): |
| """Convert a protorpc LabelDef into a protoc LabelDef.""" |
| result = project_objects_pb2.LabelDef( |
| label=label_def.label, |
| docstring=label_def.label_docstring, |
| deprecated=label_def.deprecated) |
| return result |
| |
| |
| def ConvertComponentDef( |
| component_def, users_by_id, labels_by_id, include_admin_info): |
| """Convert a protorpc ComponentDef into a protoc ComponentDef.""" |
| if not include_admin_info: |
| return project_objects_pb2.ComponentDef( |
| path=component_def.path, |
| docstring=component_def.docstring, |
| deprecated=component_def.deprecated) |
| |
| admin_refs = ConvertUserRefs(component_def.admin_ids, [], users_by_id, False) |
| cc_refs = ConvertUserRefs(component_def.cc_ids, [], users_by_id, False) |
| labels = [labels_by_id[lid] for lid in component_def.label_ids] |
| label_refs = ConvertLabels(labels, []) |
| creator_ref = ConvertUserRef(component_def.creator_id, None, users_by_id) |
| modifier_ref = ConvertUserRef(component_def.modifier_id, None, users_by_id) |
| return project_objects_pb2.ComponentDef( |
| path=component_def.path, |
| docstring=component_def.docstring, |
| admin_refs=admin_refs, |
| cc_refs=cc_refs, |
| deprecated=component_def.deprecated, |
| created=component_def.created, |
| creator_ref=creator_ref, |
| modified=component_def.modified, |
| modifier_ref=modifier_ref, |
| label_refs=label_refs) |
| |
| |
| def ConvertFieldDef(field_def, user_choices, users_by_id, config, |
| include_admin_info): |
| """Convert a protorpc FieldDef into a protoc FieldDef.""" |
| parent_approval_name = None |
| if field_def.approval_id: |
| parent_fd = tracker_bizobj.FindFieldDefByID(field_def.approval_id, config) |
| if parent_fd: |
| parent_approval_name = parent_fd.field_name |
| field_ref = ConvertFieldRef( |
| field_def.field_id, field_def.field_name, field_def.field_type, |
| parent_approval_name) |
| |
| enum_choices = [] |
| if field_def.field_type == tracker_pb2.FieldTypes.ENUM_TYPE: |
| masked_labels = tracker_helpers.LabelsMaskedByFields( |
| config, [field_def.field_name], True) |
| enum_choices = [ |
| project_objects_pb2.LabelDef( |
| label=label.name, |
| docstring=label.docstring, |
| deprecated=(label.commented == '#')) |
| for label in masked_labels] |
| |
| if not include_admin_info: |
| return project_objects_pb2.FieldDef( |
| field_ref=field_ref, |
| docstring=field_def.docstring, |
| # Display full email address for user choices. |
| user_choices=ConvertUserRefs(user_choices, [], users_by_id, True), |
| enum_choices=enum_choices) |
| |
| admin_refs = ConvertUserRefs(field_def.admin_ids, [], users_by_id, False) |
| # TODO(jrobbins): validation, permission granting, and notification options. |
| |
| return project_objects_pb2.FieldDef( |
| field_ref=field_ref, |
| applicable_type=field_def.applicable_type, |
| is_required=field_def.is_required, |
| is_niche=field_def.is_niche, |
| is_multivalued=field_def.is_multivalued, |
| docstring=field_def.docstring, |
| admin_refs=admin_refs, |
| is_phase_field=field_def.is_phase_field, |
| enum_choices=enum_choices) |
| |
| |
| def ConvertApprovalDef(approval_def, users_by_id, config, include_admin_info): |
| """Convert a protorpc ApprovalDef into a protoc ApprovalDef.""" |
| field_def = tracker_bizobj.FindFieldDefByID(approval_def.approval_id, config) |
| field_ref = ConvertFieldRef(field_def.field_id, field_def.field_name, |
| field_def.field_type, None) |
| if not include_admin_info: |
| return project_objects_pb2.ApprovalDef(field_ref=field_ref) |
| |
| approver_refs = ConvertUserRefs(approval_def.approver_ids, [], users_by_id, |
| False) |
| return project_objects_pb2.ApprovalDef( |
| field_ref=field_ref, |
| approver_refs=approver_refs, |
| survey=approval_def.survey) |
| |
| |
| def ConvertConfig( |
| project, config, users_by_id, labels_by_id): |
| """Convert a protorpc ProjectIssueConfig into a protoc Config.""" |
| status_defs = [ |
| ConvertStatusDef(sd) |
| for sd in config.well_known_statuses] |
| statuses_offer_merge = [ |
| ConvertStatusRef(sd.status, None, config) |
| for sd in config.well_known_statuses |
| if sd.status in config.statuses_offer_merge] |
| label_defs = [ |
| ConvertLabelDef(ld) |
| for ld in config.well_known_labels] |
| component_defs = [ |
| ConvertComponentDef( |
| cd, users_by_id, labels_by_id, True) |
| for cd in config.component_defs] |
| field_defs = [ |
| ConvertFieldDef(fd, [], users_by_id, config, True) |
| for fd in config.field_defs |
| if not fd.is_deleted] |
| approval_defs = [ |
| ConvertApprovalDef(ad, users_by_id, config, True) |
| for ad in config.approval_defs] |
| result = project_objects_pb2.Config( |
| project_name=project.project_name, |
| status_defs=status_defs, |
| statuses_offer_merge=statuses_offer_merge, |
| label_defs=label_defs, |
| exclusive_label_prefixes=config.exclusive_label_prefixes, |
| component_defs=component_defs, |
| field_defs=field_defs, |
| approval_defs=approval_defs, |
| restrict_to_known=config.restrict_to_known) |
| return result |
| |
| |
| def ConvertProjectTemplateDefs(templates, users_by_id, config): |
| """Convert a project's protorpc TemplateDefs into protoc TemplateDefs.""" |
| converted_templates = [] |
| for template in templates: |
| owner_ref = ConvertUserRef(template.owner_id, None, users_by_id) |
| status_ref = ConvertStatusRef(template.status, None, config) |
| labels, _derived_labels = tracker_bizobj.ExplicitAndDerivedNonMaskedLabels( |
| template.labels, [], config) |
| label_refs = ConvertLabels(labels, []) |
| admin_refs = ConvertUserRefs(template.admin_ids, [], users_by_id, False) |
| field_values = ConvertFieldValues( |
| config, template.labels, [], template.field_values, users_by_id, |
| phases=template.phases) |
| component_refs = ConvertComponents(template.component_ids, [], config) |
| approval_values = ConvertApprovalValues( |
| template.approval_values, template.phases, users_by_id, config) |
| phases = [ConvertPhaseDef(phase) for phase in template.phases] |
| |
| converted_templates.append( |
| project_objects_pb2.TemplateDef( |
| template_name=template.name, content=template.content, |
| summary=template.summary, |
| summary_must_be_edited=template.summary_must_be_edited, |
| owner_ref=owner_ref, status_ref=status_ref, label_refs=label_refs, |
| members_only=template.members_only, |
| owner_defaults_to_member=template.owner_defaults_to_member, |
| admin_refs=admin_refs, field_values=field_values, |
| component_refs=component_refs, |
| component_required=template.component_required, |
| approval_values=approval_values, phases=phases) |
| ) |
| return converted_templates |
| |
| |
| def ConvertHotlist(hotlist, users_by_id): |
| """Convert a protorpc Hotlist into a protoc Hotlist.""" |
| owner_ref = ConvertUserRef( |
| hotlist.owner_ids[0], None, users_by_id) |
| editor_refs = ConvertUserRefs(hotlist.editor_ids, [], users_by_id, False) |
| follower_refs = ConvertUserRefs( |
| hotlist.follower_ids, [], users_by_id, False) |
| result = features_objects_pb2.Hotlist( |
| owner_ref=owner_ref, |
| editor_refs=editor_refs, |
| follower_refs=follower_refs, |
| name=hotlist.name, |
| summary=hotlist.summary, |
| description=hotlist.description, |
| default_col_spec=hotlist.default_col_spec, |
| is_private=hotlist.is_private, |
| ) |
| return result |
| |
| |
| def ConvertHotlistItems(hotlist_items, issues_by_id, users_by_id, related_refs, |
| harmonized_config): |
| # Note: hotlist_items are not always sorted by 'rank' |
| sorted_ranks = sorted(item.rank for item in hotlist_items) |
| friendly_ranks_dict = { |
| rank: friendly_rank for friendly_rank, rank in |
| enumerate(sorted_ranks, 1)} |
| converted_items = [] |
| for item in hotlist_items: |
| issue_pb = issues_by_id[item.issue_id] |
| issue = ConvertIssue( |
| issue_pb, users_by_id, related_refs, harmonized_config) |
| adder_ref = ConvertUserRef(item.adder_id, None, users_by_id) |
| converted_items.append(features_objects_pb2.HotlistItem( |
| issue=issue, |
| rank=friendly_ranks_dict[item.rank], |
| adder_ref=adder_ref, |
| added_timestamp=item.date_added, |
| note=item.note)) |
| return converted_items |
| |
| |
| def ConvertValueAndWhy(value_and_why): |
| return common_pb2.ValueAndWhy( |
| value=value_and_why.get('value'), |
| why=value_and_why.get('why')) |
| |
| |
| def ConvertValueAndWhyList(value_and_why_list): |
| return [ConvertValueAndWhy(vnw) for vnw in value_and_why_list] |
| |
| |
| def _RedistributeEnumFieldsIntoLabels( |
| labels_add, labels_remove, field_vals_add, field_vals_remove, config): |
| """Look at the custom field values and treat enum fields as labels. |
| |
| Args: |
| labels_add: list of labels to add/set on the issue. |
| labels_remove: list of labels to remove from the issue. |
| field_val_add: list of protoc FieldValues to be added. |
| field_val_remove: list of protoc FieldValues to be removed. |
| remove. |
| config: ProjectIssueConfig PB including custom field definitions. |
| |
| Returns: |
| Two revised lists of protoc FieldValues to be added and removed, |
| without enum_types. |
| |
| SIDE-EFFECT: the labels and labels_remove lists will be extended with |
| key-value labels corresponding to the enum field values. |
| """ |
| field_val_strs_add = {} |
| for field_val in field_vals_add: |
| field_val_strs_add.setdefault(field_val.field_ref.field_id, []).append( |
| field_val.value) |
| |
| field_val_strs_remove = {} |
| for field_val in field_vals_remove: |
| field_val_strs_remove.setdefault(field_val.field_ref.field_id, []).append( |
| field_val.value) |
| |
| field_helpers.ShiftEnumFieldsIntoLabels( |
| labels_add, labels_remove, field_val_strs_add, field_val_strs_remove, |
| config) |
| |
| # Filter out the fields that were shifted into labels |
| updated_field_vals_add = [ |
| fv for fv in field_vals_add |
| if fv.field_ref.field_id in field_val_strs_add] |
| updated_field_vals_remove = [ |
| fv for fv in field_vals_remove |
| if fv.field_ref.field_id in field_val_strs_remove] |
| |
| return updated_field_vals_add, updated_field_vals_remove |