blob: 4f01a8bb5923d8d7b4629ce37f909ac91cc0b8e6 [file] [log] [blame]
Copybara854996b2021-09-07 19:36:02 +00001# Copyright 2018 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style
3# license that can be found in the LICENSE file or at
4# https://developers.google.com/open-source/licenses/bsd
5
6"""Functions that convert protorpc business objects into protoc objects.
7
8Monorail uses protorpc objects internally, whereas the API uses protoc
9objects. The difference is not just the choice of protobuf library, there
10will always be a need for conversion because out internal objects may have
11field that we do not want to expose externally, or the format of some fields
12may be different than how we want to expose them.
13"""
14from __future__ import print_function
15from __future__ import division
16from __future__ import absolute_import
17
18import logging
19
20from six import string_types
21
22import settings
23from api.api_proto import common_pb2
24from api.api_proto import features_objects_pb2
25from api.api_proto import issue_objects_pb2
26from api.api_proto import project_objects_pb2
27from api.api_proto import user_objects_pb2
28from features import federated
29from framework import exceptions
30from framework import filecontent
31from framework import framework_constants
32from framework import framework_helpers
33from framework import permissions
34from framework import validate
35from services import features_svc
36from tracker import attachment_helpers
37from tracker import field_helpers
38from tracker import tracker_bizobj
39from tracker import tracker_helpers
40from proto import tracker_pb2
41from proto import user_pb2
42
43
44# Convert and ingest objects in issue_objects.proto.
45
46
47def ConvertApprovalValues(approval_values, phases, users_by_id, config):
48 """Convert a list of ApprovalValue into protoc Approvals."""
49 phases_by_id = {
50 phase.phase_id: phase
51 for phase in phases}
52 result = [
53 ConvertApproval(
54 av, users_by_id, config, phase=phases_by_id.get(av.phase_id))
55 for av in approval_values]
56 result = [av for av in result if av]
57 return result
58
59
60def ConvertApproval(approval_value, users_by_id, config, phase=None):
61 """Use the given ApprovalValue to create a protoc Approval."""
62 approval_name = ''
63 fd = tracker_bizobj.FindFieldDefByID(approval_value.approval_id, config)
64 if fd:
65 approval_name = fd.field_name
66 else:
67 logging.info(
68 'Ignoring approval value referencing a non-existing field: %r',
69 approval_value)
70 return None
71
72 field_ref = ConvertFieldRef(
73 approval_value.approval_id, approval_name,
74 tracker_pb2.FieldTypes.APPROVAL_TYPE, None)
75 approver_refs = ConvertUserRefs(approval_value.approver_ids, [], users_by_id,
76 False)
77 setter_ref = ConvertUserRef(approval_value.setter_id, None, users_by_id)
78
79 status = ConvertApprovalStatus(approval_value.status)
80 set_on = approval_value.set_on
81
82 phase_ref = issue_objects_pb2.PhaseRef()
83 if phase:
84 phase_ref.phase_name = phase.name
85
86 result = issue_objects_pb2.Approval(
87 field_ref=field_ref, approver_refs=approver_refs,
88 status=status, set_on=set_on, setter_ref=setter_ref,
89 phase_ref=phase_ref)
90 return result
91
92
93def ConvertStatusRef(explicit_status, derived_status, config):
94 """Use the given status strings to create a StatusRef."""
95 status = explicit_status or derived_status
96 is_derived = not explicit_status
97 if not status:
98 return common_pb2.StatusRef(
99 status=framework_constants.NO_VALUES, is_derived=False, means_open=True)
100
101 return common_pb2.StatusRef(
102 status=status,
103 is_derived=is_derived,
104 means_open=tracker_helpers.MeansOpenInProject(status, config))
105
106
107def ConvertApprovalStatus(status):
108 """Use the given protorpc ApprovalStatus to create a protoc ApprovalStatus"""
109 return issue_objects_pb2.ApprovalStatus.Value(status.name)
110
111
112def ConvertUserRef(explicit_user_id, derived_user_id, users_by_id):
113 """Use the given user IDs to create a UserRef."""
114 user_id = explicit_user_id or derived_user_id
115 is_derived = not explicit_user_id
116 if not user_id:
117 return None;
118
119 return common_pb2.UserRef(
120 user_id=user_id,
121 is_derived=is_derived,
122 display_name=users_by_id[user_id].display_name)
123
124# TODO(jojwang): Rewrite method, ConvertUserRefs should be able to
125# call ConvertUserRef
126def ConvertUserRefs(explicit_user_ids, derived_user_ids, users_by_id,
127 use_email):
128 # (List(int), List(int), Dict(int: UserView), bool) -> List(UserRef)
129 """Use the given user ID lists to create a list of UserRef.
130
131 Args:
132 explicit_user_ids: list of user_ids for users that are not derived.
133 derived_user_ids: list of user_ids for users derived from FilterRules.
134 users_by_id: dict of {user_id: UserView, ...} for all users in
135 explicit_user_ids and derived_user_ids.
136 use_email: boolean true if the UserView.email should be used as
137 the display_name instead of UserView.display_name, which may be obscured.
138
139 Returns:
140 A single list of UserRefs.
141 """
142 result = []
143 for user_id in explicit_user_ids:
144 result.append(common_pb2.UserRef(
145 user_id=user_id,
146 is_derived=False,
147 display_name=(
148 users_by_id[user_id].email
149 if use_email
150 else users_by_id[user_id].display_name)))
151 for user_id in derived_user_ids:
152 result.append(common_pb2.UserRef(
153 user_id=user_id,
154 is_derived=True,
155 display_name=(
156 users_by_id[user_id].email
157 if use_email
158 else users_by_id[user_id].display_name)))
159 return result
160
161
162def ConvertUsers(users, users_by_id):
163 """Use the given protorpc Users to create protoc Users.
164
165 Args:
166 users: list of protorpc Users to convert.
167 users_by_id: dict {user_id: UserView} of all Users linked
168 from the users list.
169
170 Returns:
171 A list of protoc Users.
172 """
173 result = []
174 for user in users:
175 linked_parent_ref = None
176 if user.linked_parent_id:
177 linked_parent_ref = ConvertUserRefs(
178 [user.linked_parent_id], [], users_by_id, False)[0]
179 linked_child_refs = ConvertUserRefs(
180 user.linked_child_ids, [], users_by_id, False)
181 converted_user = user_objects_pb2.User(
182 user_id=user.user_id,
183 display_name=user.email,
184 is_site_admin=user.is_site_admin,
185 availability=framework_helpers.GetUserAvailability(user)[0],
186 linked_parent_ref=linked_parent_ref,
187 linked_child_refs=linked_child_refs)
188 result.append(converted_user)
189 return result
190
191
192def ConvertPrefValues(userprefvalues):
193 """Convert a list of protorpc UserPrefValue to protoc UserPrefValues."""
194 return [
195 user_objects_pb2.UserPrefValue(name=upv.name, value=upv.value)
196 for upv in userprefvalues]
197
198
199def ConvertLabels(explicit_labels, derived_labels):
200 """Combine the given explicit and derived lists into LabelRefs."""
201 explicit_refs = [common_pb2.LabelRef(label=lab, is_derived=False)
202 for lab in explicit_labels]
203 derived_refs = [common_pb2.LabelRef(label=lab, is_derived=True)
204 for lab in derived_labels]
205 return explicit_refs + derived_refs
206
207
208def ConvertComponentRef(component_id, config, is_derived=False):
209 """Make a ComponentRef from the component_id and project config."""
210 component_def = tracker_bizobj.FindComponentDefByID(component_id, config)
211 if not component_def:
212 logging.info('Ignoring non-existing component id %s', component_id)
213 return None
214 result = common_pb2.ComponentRef(
215 path=component_def.path,
216 is_derived=is_derived)
217 return result
218
219# TODO(jojwang): rename to ConvertComponentRefs
220def ConvertComponents(explicit_component_ids, derived_component_ids, config):
221 """Make a ComponentRef for each component_id."""
222 result = [ConvertComponentRef(cid, config) for cid in explicit_component_ids]
223 result += [
224 ConvertComponentRef(cid, config, is_derived=True)
225 for cid in derived_component_ids]
226 result = [cr for cr in result if cr]
227 return result
228
229
230def ConvertIssueRef(issue_ref_pair, ext_id=''):
231 """Convert (project_name, local_id) to an IssueRef protoc object.
232
233 With optional external ref in ext_id.
234 """
235 project_name, local_id = issue_ref_pair
236 ref = common_pb2.IssueRef(project_name=project_name, local_id=local_id,
237 ext_identifier=ext_id)
238 return ref
239
240
241def ConvertIssueRefs(issue_ids, related_refs_dict):
242 """Convert a list of iids to IssueRef protoc objects."""
243 return [ConvertIssueRef(related_refs_dict[iid]) for iid in issue_ids]
244
245
246def ConvertFieldValue(field_id, field_name, value, field_type,
247 approval_name=None, phase_name=None, is_derived=False):
248 """Convert one field value view item into a protoc FieldValue."""
249 if not isinstance(value, string_types):
250 value = str(value)
251 fv = issue_objects_pb2.FieldValue(
252 field_ref=ConvertFieldRef(field_id, field_name, field_type,
253 approval_name),
254 value=value,
255 is_derived=is_derived)
256 if phase_name:
257 fv.phase_ref.phase_name = phase_name
258
259 return fv
260
261
262def ConvertFieldType(field_type):
263 """Use the given protorpc FieldTypes enum to create a protoc FieldType."""
264 return common_pb2.FieldType.Value(field_type.name)
265
266
267def ConvertFieldRef(field_id, field_name, field_type, approval_name):
268 """Convert a field name and protorpc FieldType into a protoc FieldRef."""
269 return common_pb2.FieldRef(field_id=field_id,
270 field_name=field_name,
271 type=ConvertFieldType(field_type),
272 approval_name=approval_name)
273
274
275def ConvertFieldValues(
276 config, labels, derived_labels, field_values, users_by_id, phases=None):
277 """Convert lists of labels and field_values to protoc FieldValues."""
278 fvs = []
279 phase_names_by_id = {phase.phase_id: phase.name for phase in phases or []}
280 fds_by_id = {fd.field_id:fd for fd in config.field_defs}
281 fids_by_name = {fd.field_name:fd.field_id for fd in config.field_defs}
282 enum_names_by_lower = {
283 fd.field_name.lower(): fd.field_name for fd in config.field_defs
284 if fd.field_type == tracker_pb2.FieldTypes.ENUM_TYPE}
285
286 labels_by_prefix = tracker_bizobj.LabelsByPrefix(
287 labels, list(enum_names_by_lower.keys()))
288 der_labels_by_prefix = tracker_bizobj.LabelsByPrefix(
289 derived_labels, list(enum_names_by_lower.keys()))
290
291 for lower_field_name, values in labels_by_prefix.items():
292 field_name = enum_names_by_lower.get(lower_field_name)
293 if not field_name:
294 continue
295 fvs.extend(
296 [ConvertFieldValue(
297 fids_by_name.get(field_name), field_name, value,
298 tracker_pb2.FieldTypes.ENUM_TYPE)
299 for value in values])
300
301 for lower_field_name, values in der_labels_by_prefix.items():
302 field_name = enum_names_by_lower.get(lower_field_name)
303 if not field_name:
304 continue
305 fvs.extend(
306 [ConvertFieldValue(
307 fids_by_name.get(field_name), field_name, value,
308 tracker_pb2.FieldTypes.ENUM_TYPE, is_derived=True)
309 for value in values])
310
311 for fv in field_values:
312 field_def = fds_by_id.get(fv.field_id)
313 if not field_def:
314 logging.info(
315 'Ignoring field value referencing a non-existent field: %r', fv)
316 continue
317
318 value = tracker_bizobj.GetFieldValue(fv, users_by_id)
319 field_name = field_def.field_name
320 field_type = field_def.field_type
321 approval_name = None
322
323 if field_def.approval_id is not None:
324 approval_def = fds_by_id.get(field_def.approval_id)
325 if approval_def:
326 approval_name = approval_def.field_name
327
328 fvs.append(ConvertFieldValue(
329 fv.field_id, field_name, value, field_type, approval_name=approval_name,
330 phase_name=phase_names_by_id.get(fv.phase_id), is_derived=fv.derived))
331
332 return fvs
333
334
335def ConvertIssue(issue, users_by_id, related_refs, config):
336 """Convert our protorpc Issue to a protoc Issue.
337
338 Args:
339 issue: protorpc issue used by monorail internally.
340 users_by_id: dict {user_id: UserViews} for all users mentioned in issue.
341 related_refs: dict {issue_id: (project_name, local_id)} of all blocked-on,
342 blocking, or merged-into issues referenced from this issue, regardless
343 of perms.
344 config: ProjectIssueConfig for this issue.
345
346 Returns: A protoc Issue object.
347 """
348 status_ref = ConvertStatusRef(issue.status, issue.derived_status, config)
349 owner_ref = ConvertUserRef(
350 issue.owner_id, issue.derived_owner_id, users_by_id)
351 cc_refs = ConvertUserRefs(
352 issue.cc_ids, issue.derived_cc_ids, users_by_id, False)
353 labels, derived_labels = tracker_bizobj.ExplicitAndDerivedNonMaskedLabels(
354 issue.labels, issue.derived_labels, config)
355 label_refs = ConvertLabels(labels, derived_labels)
356 component_refs = ConvertComponents(
357 issue.component_ids, issue.derived_component_ids, config)
358 blocked_on_issue_refs = ConvertIssueRefs(
359 issue.blocked_on_iids, related_refs)
360 dangling_blocked_on_refs = [
361 ConvertIssueRef((dangling_issue.project, dangling_issue.issue_id),
362 ext_id=dangling_issue.ext_issue_identifier)
363 for dangling_issue in issue.dangling_blocked_on_refs]
364 blocking_issue_refs = ConvertIssueRefs(
365 issue.blocking_iids, related_refs)
366 dangling_blocking_refs = [
367 ConvertIssueRef((dangling_issue.project, dangling_issue.issue_id),
368 ext_id=dangling_issue.ext_issue_identifier)
369 for dangling_issue in issue.dangling_blocking_refs]
370 merged_into_issue_ref = None
371 if issue.merged_into:
372 merged_into_issue_ref = ConvertIssueRef(related_refs[issue.merged_into])
373 if issue.merged_into_external:
374 merged_into_issue_ref = ConvertIssueRef((None, None),
375 ext_id=issue.merged_into_external)
376
377 field_values = ConvertFieldValues(
378 config, issue.labels, issue.derived_labels,
379 issue.field_values, users_by_id, phases=issue.phases)
380 approval_values = ConvertApprovalValues(
381 issue.approval_values, issue.phases, users_by_id, config)
382 reporter_ref = ConvertUserRef(issue.reporter_id, None, users_by_id)
383 phases = [ConvertPhaseDef(phase) for phase in issue.phases]
384 result = issue_objects_pb2.Issue(
385 project_name=issue.project_name, local_id=issue.local_id,
386 summary=issue.summary, status_ref=status_ref, owner_ref=owner_ref,
387 cc_refs=cc_refs, label_refs=label_refs, component_refs=component_refs,
388 blocked_on_issue_refs=blocked_on_issue_refs,
389 dangling_blocked_on_refs=dangling_blocked_on_refs,
390 blocking_issue_refs=blocking_issue_refs,
391 dangling_blocking_refs=dangling_blocking_refs,
392 merged_into_issue_ref=merged_into_issue_ref, field_values=field_values,
393 is_deleted=issue.deleted, reporter_ref=reporter_ref,
394 opened_timestamp=issue.opened_timestamp,
395 closed_timestamp=issue.closed_timestamp,
396 modified_timestamp=issue.modified_timestamp,
397 component_modified_timestamp=issue.component_modified_timestamp,
398 status_modified_timestamp=issue.status_modified_timestamp,
399 owner_modified_timestamp=issue.owner_modified_timestamp,
400 star_count=issue.star_count, is_spam=issue.is_spam,
401 approval_values=approval_values, phases=phases)
402
403 # TODO(crbug.com/monorail/5857): Set attachment_count unconditionally
404 # after the underlying source of negative attachment counts has been
405 # resolved and database has been repaired.
406 if issue.attachment_count >= 0:
407 result.attachment_count = issue.attachment_count
408
409 return result
410
411
412def ConvertPhaseDef(phase):
413 """Convert a protorpc Phase to a protoc PhaseDef."""
414 phase_def = issue_objects_pb2.PhaseDef(
415 phase_ref=issue_objects_pb2.PhaseRef(phase_name=phase.name),
416 rank=phase.rank)
417 return phase_def
418
419
420def ConvertAmendment(amendment, users_by_id):
421 """Convert a protorpc Amendment to a protoc Amendment."""
422 field_name = tracker_bizobj.GetAmendmentFieldName(amendment)
423 new_value = tracker_bizobj.AmendmentString(amendment, users_by_id)
424 result = issue_objects_pb2.Amendment(
425 field_name=field_name, new_or_delta_value=new_value,
426 old_value=amendment.oldvalue)
427 return result
428
429
430def ConvertAttachment(attach, project_name):
431 """Convert a protorpc Attachment to a protoc Attachment."""
432 size, thumbnail_url, view_url, download_url = None, None, None, None
433 if not attach.deleted:
434 size = attach.filesize
435 download_url = attachment_helpers.GetDownloadURL(attach.attachment_id)
436 view_url = attachment_helpers.GetViewURL(attach, download_url, project_name)
437 thumbnail_url = attachment_helpers.GetThumbnailURL(attach, download_url)
438
439 result = issue_objects_pb2.Attachment(
440 attachment_id=attach.attachment_id, filename=attach.filename,
441 size=size, content_type=attach.mimetype,
442 is_deleted=attach.deleted, thumbnail_url=thumbnail_url,
443 view_url=view_url, download_url=download_url)
444 return result
445
446
447def ConvertComment(
448 issue, comment, config, users_by_id, comment_reporters, description_nums,
449 user_id, perms):
450 """Convert a protorpc IssueComment to a protoc Comment."""
451 commenter = users_by_id[comment.user_id]
452
453 can_delete = permissions.CanDeleteComment(
454 comment, commenter, user_id, perms)
455 can_flag, is_flagged = permissions.CanFlagComment(
456 comment, commenter, comment_reporters, user_id, perms)
457 can_view = permissions.CanViewComment(
458 comment, commenter, user_id, perms)
459 can_view_inbound_message = permissions.CanViewInboundMessage(
460 comment, user_id, perms)
461
462 is_deleted = bool(comment.deleted_by or is_flagged or commenter.banned)
463
464 result = issue_objects_pb2.Comment(
465 project_name=issue.project_name,
466 local_id=issue.local_id,
467 sequence_num=comment.sequence,
468 is_deleted=is_deleted,
469 can_delete=can_delete,
470 is_spam=is_flagged,
471 can_flag=can_flag,
472 timestamp=comment.timestamp)
473
474 if can_view:
475 result.commenter.CopyFrom(
476 ConvertUserRef(comment.user_id, None, users_by_id))
477 result.content = comment.content
478 if comment.inbound_message and can_view_inbound_message:
479 result.inbound_message = comment.inbound_message
480 result.amendments.extend([
481 ConvertAmendment(amend, users_by_id)
482 for amend in comment.amendments])
483 result.attachments.extend([
484 ConvertAttachment(attach, issue.project_name)
485 for attach in comment.attachments])
486
487 if comment.id in description_nums:
488 result.description_num = description_nums[comment.id]
489
490 fd = tracker_bizobj.FindFieldDefByID(comment.approval_id, config)
491 if fd:
492 result.approval_ref.field_name = fd.field_name
493
494 return result
495
496
497def ConvertCommentList(
498 issue, comments, config, users_by_id, comment_reporters, user_id, perms):
499 """Convert a list of protorpc IssueComments to protoc Comments."""
500 description_nums = {}
501 for comment in comments:
502 if (comment.is_description and not users_by_id[comment.user_id].banned and
503 not comment.deleted_by and not comment.is_spam):
504 description_nums[comment.id] = len(description_nums) + 1
505
506 result = [
507 ConvertComment(
508 issue, c, config, users_by_id, comment_reporters.get(c.id, []),
509 description_nums, user_id, perms)
510 for c in comments]
511 return result
512
513
514def IngestUserRef(cnxn, user_ref, user_service, autocreate=False):
515 """Return ID of specified user or raise NoSuchUserException."""
516 try:
517 return IngestUserRefs(
518 cnxn, [user_ref], user_service, autocreate=autocreate)[0]
519 except IndexError:
520 # user_ref.display_name was not a valid email.
521 raise exceptions.NoSuchUserException
522
523
524def IngestUserRefs(cnxn, user_refs, user_service, autocreate=False):
525 """Return IDs of specified users or raise NoSuchUserException."""
526
527 # Filter out user_refs with invalid display_names.
528 # Invalid emails won't get auto-created in LookupUserIds, but un-specified
529 # user_ref.user_id values have the zero-value 0. So invalid user_ref's
530 # need to be filtered out here to prevent these resulting in '0's in
531 # the 'result' array.
532 if autocreate:
533 user_refs = [user_ref for user_ref in user_refs
534 if (not user_ref.display_name) or
535 validate.IsValidEmail(user_ref.display_name)]
536
537 # 1. Verify that all specified user IDs actually match existing users.
538 user_ids_to_verify = [user_ref.user_id for user_ref in user_refs
539 if user_ref.user_id]
540 user_service.LookupUserEmails(cnxn, user_ids_to_verify)
541
542 # 2. Lookup or create any users that are specified by email address.
543 needed_emails = [user_ref.display_name for user_ref in user_refs
544 if not user_ref.user_id and user_ref.display_name]
545 emails_to_ids = user_service.LookupUserIDs(
546 cnxn, needed_emails, autocreate=autocreate)
547
548 # 3. Build the result from emails_to_ids or straight from user_ref's
549 # user_id.
550 # Note: user_id can be specified as 0 to clear the issue owner.
551 result = [
552 emails_to_ids.get(user_ref.display_name.lower(), user_ref.user_id)
553 for user_ref in user_refs
554 ]
555 return result
556
557
558def IngestPrefValues(pref_values):
559 """Return protorpc UserPrefValues for the given values."""
560 return [user_pb2.UserPrefValue(name=upv.name, value=upv.value)
561 for upv in pref_values]
562
563
564def IngestComponentRefs(comp_refs, config, ignore_missing_objects=False):
565 """Return IDs of specified components or raise NoSuchComponentException."""
566 cids_by_path = {cd.path.lower(): cd.component_id
567 for cd in config.component_defs}
568 result = []
569 for comp_ref in comp_refs:
570 cid = cids_by_path.get(comp_ref.path.lower())
571 if cid:
572 result.append(cid)
573 else:
574 if not ignore_missing_objects:
575 raise exceptions.NoSuchComponentException()
576 return result
577
578
579def IngestFieldRefs(field_refs, config):
580 """Return IDs of specified fields or raise NoSuchFieldDefException."""
581 fids_by_name = {fd.field_name.lower(): fd.field_id
582 for fd in config.field_defs}
583 result = []
584 for field_ref in field_refs:
585 fid = fids_by_name.get(field_ref.field_name.lower())
586 if fid:
587 result.append(fid)
588 else:
589 raise exceptions.NoSuchFieldDefException()
590 return result
591
592
593def IngestIssueRefs(cnxn, issue_refs, services):
594 """Look up issue IDs for the specified issues."""
595 project_names = set(ref.project_name for ref in issue_refs)
596 project_names_to_id = services.project.LookupProjectIDs(cnxn, project_names)
597 project_local_id_pairs = []
598 for ref in issue_refs:
599 if ref.ext_identifier:
600 # TODO(jeffcarp): For external tracker refs, once we have the classes
601 # set up, validate that the tracker for this specific ref is supported
602 # and store the external ref in the issue properly.
603 if '/' not in ref.ext_identifier:
604 raise exceptions.InvalidExternalIssueReference()
605 continue
606 if ref.project_name in project_names_to_id:
607 pair = (project_names_to_id[ref.project_name], ref.local_id)
608 project_local_id_pairs.append(pair)
609 else:
610 raise exceptions.NoSuchProjectException()
611 issue_ids, misses = services.issue.LookupIssueIDs(
612 cnxn, project_local_id_pairs)
613 if misses:
614 raise exceptions.NoSuchIssueException()
615 return issue_ids
616
617
618def IngestExtIssueRefs(issue_refs):
619 """Validate and return external issue refs."""
620 return [
621 ref.ext_identifier
622 for ref in issue_refs
623 if ref.ext_identifier
624 and federated.IsShortlinkValid(ref.ext_identifier)]
625
626
627def IngestIssueDelta(
628 cnxn, services, delta, config, phases, ignore_missing_objects=False):
629 """Ingest a protoc IssueDelta and create a protorpc IssueDelta."""
630 status = None
631 if delta.HasField('status'):
632 status = delta.status.value
633 owner_id = None
634 if delta.HasField('owner_ref'):
635 try:
636 owner_id = IngestUserRef(cnxn, delta.owner_ref, services.user)
637 except exceptions.NoSuchUserException as e:
638 if not ignore_missing_objects:
639 raise e
640 summary = None
641 if delta.HasField('summary'):
642 summary = delta.summary.value
643
644 cc_ids_add = IngestUserRefs(
645 cnxn, delta.cc_refs_add, services.user, autocreate=True)
646 cc_ids_remove = IngestUserRefs(cnxn, delta.cc_refs_remove, services.user)
647
648 comp_ids_add = IngestComponentRefs(
649 delta.comp_refs_add, config,
650 ignore_missing_objects=ignore_missing_objects)
651 comp_ids_remove = IngestComponentRefs(
652 delta.comp_refs_remove, config,
653 ignore_missing_objects=ignore_missing_objects)
654
655 labels_add = [lab_ref.label for lab_ref in delta.label_refs_add]
656 labels_remove = [lab_ref.label for lab_ref in delta.label_refs_remove]
657
658 field_vals_add, field_vals_remove = _RedistributeEnumFieldsIntoLabels(
659 labels_add, labels_remove,
660 delta.field_vals_add, delta.field_vals_remove,
661 config)
662
663 field_vals_add = IngestFieldValues(
664 cnxn, services.user, field_vals_add, config, phases=phases)
665 field_vals_remove = IngestFieldValues(
666 cnxn, services.user, field_vals_remove, config, phases=phases)
667 fields_clear = IngestFieldRefs(delta.fields_clear, config)
668
669 # Ingest intra-tracker issue refs.
670 blocked_on_add = IngestIssueRefs(
671 cnxn, delta.blocked_on_refs_add, services)
672 blocked_on_remove = IngestIssueRefs(
673 cnxn, delta.blocked_on_refs_remove, services)
674 blocking_add = IngestIssueRefs(
675 cnxn, delta.blocking_refs_add, services)
676 blocking_remove = IngestIssueRefs(
677 cnxn, delta.blocking_refs_remove, services)
678
679 # Ingest inter-tracker issue refs.
680 ext_blocked_on_add = IngestExtIssueRefs(delta.blocked_on_refs_add)
681 ext_blocked_on_remove = IngestExtIssueRefs(delta.blocked_on_refs_remove)
682 ext_blocking_add = IngestExtIssueRefs(delta.blocking_refs_add)
683 ext_blocking_remove = IngestExtIssueRefs(delta.blocking_refs_remove)
684
685 merged_into = None
686 merged_into_external = None
687 if delta.HasField('merged_into_ref'):
688 if delta.merged_into_ref.ext_identifier: # Adding an external merged.
689 merged_into_external = delta.merged_into_ref.ext_identifier
690 elif not delta.merged_into_ref.local_id: # Clearing an internal merged.
691 merged_into = 0
692 else: # Adding an internal merged.
693 merged_into = IngestIssueRefs(cnxn, [delta.merged_into_ref], services)[0]
694
695 result = tracker_bizobj.MakeIssueDelta(
696 status, owner_id, cc_ids_add, cc_ids_remove, comp_ids_add,
697 comp_ids_remove, labels_add, labels_remove, field_vals_add,
698 field_vals_remove, fields_clear, blocked_on_add, blocked_on_remove,
699 blocking_add, blocking_remove, merged_into, summary,
700 ext_blocked_on_add=ext_blocked_on_add,
701 ext_blocked_on_remove=ext_blocked_on_remove,
702 ext_blocking_add=ext_blocking_add,
703 ext_blocking_remove=ext_blocking_remove,
704 merged_into_external=merged_into_external)
705 return result
706
707def IngestAttachmentUploads(attachment_uploads):
708 """Ingest protoc AttachmentUpload objects as tuples."""
709 result = []
710 for up in attachment_uploads:
711 if not up.filename:
712 raise exceptions.InputException('Missing attachment name')
713 if not up.content:
714 raise exceptions.InputException('Missing attachment content')
715 mimetype = filecontent.GuessContentTypeFromFilename(up.filename)
716 attachment_tuple = (up.filename, up.content, mimetype)
717 result.append(attachment_tuple)
718 return result
719
720
721def IngestApprovalDelta(cnxn, user_service, approval_delta, setter_id, config):
722 """Ingest a protoc ApprovalDelta and create a protorpc ApprovalDelta."""
723 fids_by_name = {fd.field_name.lower(): fd.field_id for
724 fd in config.field_defs}
725
726 approver_ids_add = IngestUserRefs(
727 cnxn, approval_delta.approver_refs_add, user_service, autocreate=True)
728 approver_ids_remove = IngestUserRefs(
729 cnxn, approval_delta.approver_refs_remove, user_service, autocreate=True)
730
731 labels_add, labels_remove = [], []
732 # TODO(jojwang): monorail:4673, validate enum values all belong to approval.
733 field_vals_add, field_vals_remove = _RedistributeEnumFieldsIntoLabels(
734 labels_add, labels_remove,
735 approval_delta.field_vals_add, approval_delta.field_vals_remove,
736 config)
737
738 sub_fvs_add = IngestFieldValues(cnxn, user_service, field_vals_add, config)
739 sub_fvs_remove = IngestFieldValues(
740 cnxn, user_service, field_vals_remove, config)
741 sub_fields_clear = [fids_by_name.get(clear.field_name.lower()) for
742 clear in approval_delta.fields_clear
743 if clear.field_name.lower() in fids_by_name]
744
745 # protoc ENUMs default to the zero value (in this case: NOT_SET).
746 # NOT_SET should only be allowed when an issue is first created.
747 # Once a user changes it to something else, no one should be allowed
748 # to set it back.
749 status = None
750 if approval_delta.status != issue_objects_pb2.NOT_SET:
751 status = IngestApprovalStatus(approval_delta.status)
752
753 return tracker_bizobj.MakeApprovalDelta(
754 status, setter_id, approver_ids_add, approver_ids_remove,
755 sub_fvs_add, sub_fvs_remove, sub_fields_clear, labels_add, labels_remove)
756
757
758def IngestApprovalStatus(approval_status):
759 """Ingest a protoc ApprovalStatus and create a protorpc ApprovalStatus. """
760 if approval_status == issue_objects_pb2.NOT_SET:
761 return tracker_pb2.ApprovalStatus.NOT_SET
762 return tracker_pb2.ApprovalStatus(approval_status)
763
764
765def IngestFieldValues(cnxn, user_service, field_values, config, phases=None):
766 """Ingest a list of protoc FieldValues and create protorpc FieldValues.
767
768 Args:
769 cnxn: connection to the DB.
770 user_service: interface to user data storage.
771 field_values: a list of protoc FieldValue used by the API.
772 config: ProjectIssueConfig for this field_value's project.
773 phases: a list of the issue's protorpc Phases.
774
775
776 Returns: A protorpc FieldValue object.
777 """
778 fds_by_name = {fd.field_name.lower(): fd for fd in config.field_defs}
779 phases_by_name = {phase.name: phase.phase_id for phase in phases or []}
780
781 ingested_fvs = []
782 for fv in field_values:
783 fd = fds_by_name.get(fv.field_ref.field_name.lower())
784 if fd:
785 if not fv.value:
786 logging.info('Ignoring blank field value: %r', fv)
787 continue
788 ingested_fv = field_helpers.ParseOneFieldValue(
789 cnxn, user_service, fd, fv.value)
790 if not ingested_fv:
791 raise exceptions.InputException(
792 'Unparsable value for field %s' % fv.field_ref.field_name)
793 if ingested_fv.user_id == field_helpers.INVALID_USER_ID:
794 raise exceptions.NoSuchUserException()
795 if fd.is_phase_field:
796 ingested_fv.phase_id = phases_by_name.get(fv.phase_ref.phase_name)
797 ingested_fvs.append(ingested_fv)
798
799 return ingested_fvs
800
801
802def IngestSavedQueries(cnxn, project_service, saved_queries):
803 """Ingest a list of protoc SavedQuery and create protorpc SavedQuery.
804
805 Args:
806 cnxn: connection to the DB.
807 project_service: interface to project data storage.
808 saved_queries: a list of protoc Savedquery.
809
810 Returns: A protorpc SavedQuery object.
811 """
812 if not saved_queries:
813 return []
814
815 project_ids = set()
816 for sq in saved_queries:
817 project_ids.update(sq.executes_in_project_ids)
818
819 project_name_dict = project_service.LookupProjectNames(cnxn,
820 project_ids)
821 return [
822 common_pb2.SavedQuery(
823 query_id=sq.query_id,
824 name=sq.name,
825 query=sq.query,
826 project_names=[project_name_dict[project_id]
827 for project_id in sq.executes_in_project_ids]
828 )
829 for sq in saved_queries]
830
831
832def IngestHotlistRefs(cnxn, user_service, features_service, hotlist_refs):
833 return [IngestHotlistRef(cnxn, user_service, features_service, hotlist_ref)
834 for hotlist_ref in hotlist_refs]
835
836
837def IngestHotlistRef(cnxn, user_service, features_service, hotlist_ref):
838 hotlist_id = None
839
840 if hotlist_ref.hotlist_id:
841 # If a hotlist ID was specified, verify it actually match existing hotlists.
842 features_service.GetHotlist(cnxn, hotlist_ref.hotlist_id)
843 hotlist_id = hotlist_ref.hotlist_id
844
845 if hotlist_ref.name and hotlist_ref.owner:
846 name = hotlist_ref.name
847 owner_id = IngestUserRef(cnxn, hotlist_ref.owner, user_service)
848 hotlists = features_service.LookupHotlistIDs(cnxn, [name], [owner_id])
849 # Verify there is a hotlist with that name and owner.
850 if (name.lower(), owner_id) not in hotlists:
851 raise features_svc.NoSuchHotlistException()
852 found_id = hotlists[name.lower(), owner_id]
853 # If a hotlist_id was also given, verify it correspond to the name and
854 # owner.
855 if hotlist_id is not None and found_id != hotlist_id:
856 raise features_svc.NoSuchHotlistException()
857 hotlist_id = found_id
858
859 # Neither an ID, nor a name-owner ref were given.
860 if hotlist_id is None:
861 raise features_svc.NoSuchHotlistException()
862
863 return hotlist_id
864
865
866def IngestPagination(pagination):
867 max_items = settings.max_artifact_search_results_per_page
868 if pagination.max_items:
869 max_items = min(max_items, pagination.max_items)
870 return pagination.start, max_items
871
872# Convert and ingest objects in project_objects.proto.
873
874def ConvertStatusDef(status_def):
875 """Convert a protorpc StatusDef into a protoc StatusDef."""
876 result = project_objects_pb2.StatusDef(
877 status=status_def.status,
878 means_open=status_def.means_open,
879 docstring=status_def.status_docstring,
880 deprecated=status_def.deprecated)
881 return result
882
883
884def ConvertLabelDef(label_def):
885 """Convert a protorpc LabelDef into a protoc LabelDef."""
886 result = project_objects_pb2.LabelDef(
887 label=label_def.label,
888 docstring=label_def.label_docstring,
889 deprecated=label_def.deprecated)
890 return result
891
892
893def ConvertComponentDef(
894 component_def, users_by_id, labels_by_id, include_admin_info):
895 """Convert a protorpc ComponentDef into a protoc ComponentDef."""
896 if not include_admin_info:
897 return project_objects_pb2.ComponentDef(
898 path=component_def.path,
899 docstring=component_def.docstring,
900 deprecated=component_def.deprecated)
901
902 admin_refs = ConvertUserRefs(component_def.admin_ids, [], users_by_id, False)
903 cc_refs = ConvertUserRefs(component_def.cc_ids, [], users_by_id, False)
904 labels = [labels_by_id[lid] for lid in component_def.label_ids]
905 label_refs = ConvertLabels(labels, [])
906 creator_ref = ConvertUserRef(component_def.creator_id, None, users_by_id)
907 modifier_ref = ConvertUserRef(component_def.modifier_id, None, users_by_id)
908 return project_objects_pb2.ComponentDef(
909 path=component_def.path,
910 docstring=component_def.docstring,
911 admin_refs=admin_refs,
912 cc_refs=cc_refs,
913 deprecated=component_def.deprecated,
914 created=component_def.created,
915 creator_ref=creator_ref,
916 modified=component_def.modified,
917 modifier_ref=modifier_ref,
918 label_refs=label_refs)
919
920
921def ConvertFieldDef(field_def, user_choices, users_by_id, config,
922 include_admin_info):
923 """Convert a protorpc FieldDef into a protoc FieldDef."""
924 parent_approval_name = None
925 if field_def.approval_id:
926 parent_fd = tracker_bizobj.FindFieldDefByID(field_def.approval_id, config)
927 if parent_fd:
928 parent_approval_name = parent_fd.field_name
929 field_ref = ConvertFieldRef(
930 field_def.field_id, field_def.field_name, field_def.field_type,
931 parent_approval_name)
932
933 enum_choices = []
934 if field_def.field_type == tracker_pb2.FieldTypes.ENUM_TYPE:
935 masked_labels = tracker_helpers.LabelsMaskedByFields(
936 config, [field_def.field_name], True)
937 enum_choices = [
938 project_objects_pb2.LabelDef(
939 label=label.name,
940 docstring=label.docstring,
941 deprecated=(label.commented == '#'))
942 for label in masked_labels]
943
944 if not include_admin_info:
945 return project_objects_pb2.FieldDef(
946 field_ref=field_ref,
947 docstring=field_def.docstring,
948 # Display full email address for user choices.
949 user_choices=ConvertUserRefs(user_choices, [], users_by_id, True),
950 enum_choices=enum_choices)
951
952 admin_refs = ConvertUserRefs(field_def.admin_ids, [], users_by_id, False)
953 # TODO(jrobbins): validation, permission granting, and notification options.
954
955 return project_objects_pb2.FieldDef(
956 field_ref=field_ref,
957 applicable_type=field_def.applicable_type,
958 is_required=field_def.is_required,
959 is_niche=field_def.is_niche,
960 is_multivalued=field_def.is_multivalued,
961 docstring=field_def.docstring,
962 admin_refs=admin_refs,
963 is_phase_field=field_def.is_phase_field,
964 enum_choices=enum_choices)
965
966
967def ConvertApprovalDef(approval_def, users_by_id, config, include_admin_info):
968 """Convert a protorpc ApprovalDef into a protoc ApprovalDef."""
969 field_def = tracker_bizobj.FindFieldDefByID(approval_def.approval_id, config)
970 field_ref = ConvertFieldRef(field_def.field_id, field_def.field_name,
971 field_def.field_type, None)
972 if not include_admin_info:
973 return project_objects_pb2.ApprovalDef(field_ref=field_ref)
974
975 approver_refs = ConvertUserRefs(approval_def.approver_ids, [], users_by_id,
976 False)
977 return project_objects_pb2.ApprovalDef(
978 field_ref=field_ref,
979 approver_refs=approver_refs,
980 survey=approval_def.survey)
981
982
983def ConvertConfig(
984 project, config, users_by_id, labels_by_id):
985 """Convert a protorpc ProjectIssueConfig into a protoc Config."""
986 status_defs = [
987 ConvertStatusDef(sd)
988 for sd in config.well_known_statuses]
989 statuses_offer_merge = [
990 ConvertStatusRef(sd.status, None, config)
991 for sd in config.well_known_statuses
992 if sd.status in config.statuses_offer_merge]
993 label_defs = [
994 ConvertLabelDef(ld)
995 for ld in config.well_known_labels]
996 component_defs = [
997 ConvertComponentDef(
998 cd, users_by_id, labels_by_id, True)
999 for cd in config.component_defs]
1000 field_defs = [
1001 ConvertFieldDef(fd, [], users_by_id, config, True)
1002 for fd in config.field_defs
1003 if not fd.is_deleted]
1004 approval_defs = [
1005 ConvertApprovalDef(ad, users_by_id, config, True)
1006 for ad in config.approval_defs]
1007 result = project_objects_pb2.Config(
1008 project_name=project.project_name,
1009 status_defs=status_defs,
1010 statuses_offer_merge=statuses_offer_merge,
1011 label_defs=label_defs,
1012 exclusive_label_prefixes=config.exclusive_label_prefixes,
1013 component_defs=component_defs,
1014 field_defs=field_defs,
1015 approval_defs=approval_defs,
1016 restrict_to_known=config.restrict_to_known)
1017 return result
1018
1019
1020def ConvertProjectTemplateDefs(templates, users_by_id, config):
1021 """Convert a project's protorpc TemplateDefs into protoc TemplateDefs."""
1022 converted_templates = []
1023 for template in templates:
1024 owner_ref = ConvertUserRef(template.owner_id, None, users_by_id)
1025 status_ref = ConvertStatusRef(template.status, None, config)
1026 labels, _derived_labels = tracker_bizobj.ExplicitAndDerivedNonMaskedLabels(
1027 template.labels, [], config)
1028 label_refs = ConvertLabels(labels, [])
1029 admin_refs = ConvertUserRefs(template.admin_ids, [], users_by_id, False)
1030 field_values = ConvertFieldValues(
1031 config, template.labels, [], template.field_values, users_by_id,
1032 phases=template.phases)
1033 component_refs = ConvertComponents(template.component_ids, [], config)
1034 approval_values = ConvertApprovalValues(
1035 template.approval_values, template.phases, users_by_id, config)
1036 phases = [ConvertPhaseDef(phase) for phase in template.phases]
1037
1038 converted_templates.append(
1039 project_objects_pb2.TemplateDef(
1040 template_name=template.name, content=template.content,
1041 summary=template.summary,
1042 summary_must_be_edited=template.summary_must_be_edited,
1043 owner_ref=owner_ref, status_ref=status_ref, label_refs=label_refs,
1044 members_only=template.members_only,
1045 owner_defaults_to_member=template.owner_defaults_to_member,
1046 admin_refs=admin_refs, field_values=field_values,
1047 component_refs=component_refs,
1048 component_required=template.component_required,
1049 approval_values=approval_values, phases=phases)
1050 )
1051 return converted_templates
1052
1053
1054def ConvertHotlist(hotlist, users_by_id):
1055 """Convert a protorpc Hotlist into a protoc Hotlist."""
1056 owner_ref = ConvertUserRef(
1057 hotlist.owner_ids[0], None, users_by_id)
1058 editor_refs = ConvertUserRefs(hotlist.editor_ids, [], users_by_id, False)
1059 follower_refs = ConvertUserRefs(
1060 hotlist.follower_ids, [], users_by_id, False)
1061 result = features_objects_pb2.Hotlist(
1062 owner_ref=owner_ref,
1063 editor_refs=editor_refs,
1064 follower_refs=follower_refs,
1065 name=hotlist.name,
1066 summary=hotlist.summary,
1067 description=hotlist.description,
1068 default_col_spec=hotlist.default_col_spec,
1069 is_private=hotlist.is_private,
1070 )
1071 return result
1072
1073
1074def ConvertHotlistItems(hotlist_items, issues_by_id, users_by_id, related_refs,
1075 harmonized_config):
1076 # Note: hotlist_items are not always sorted by 'rank'
1077 sorted_ranks = sorted(item.rank for item in hotlist_items)
1078 friendly_ranks_dict = {
1079 rank: friendly_rank for friendly_rank, rank in
1080 enumerate(sorted_ranks, 1)}
1081 converted_items = []
1082 for item in hotlist_items:
1083 issue_pb = issues_by_id[item.issue_id]
1084 issue = ConvertIssue(
1085 issue_pb, users_by_id, related_refs, harmonized_config)
1086 adder_ref = ConvertUserRef(item.adder_id, None, users_by_id)
1087 converted_items.append(features_objects_pb2.HotlistItem(
1088 issue=issue,
1089 rank=friendly_ranks_dict[item.rank],
1090 adder_ref=adder_ref,
1091 added_timestamp=item.date_added,
1092 note=item.note))
1093 return converted_items
1094
1095
1096def ConvertValueAndWhy(value_and_why):
1097 return common_pb2.ValueAndWhy(
1098 value=value_and_why.get('value'),
1099 why=value_and_why.get('why'))
1100
1101
1102def ConvertValueAndWhyList(value_and_why_list):
1103 return [ConvertValueAndWhy(vnw) for vnw in value_and_why_list]
1104
1105
1106def _RedistributeEnumFieldsIntoLabels(
1107 labels_add, labels_remove, field_vals_add, field_vals_remove, config):
1108 """Look at the custom field values and treat enum fields as labels.
1109
1110 Args:
1111 labels_add: list of labels to add/set on the issue.
1112 labels_remove: list of labels to remove from the issue.
1113 field_val_add: list of protoc FieldValues to be added.
1114 field_val_remove: list of protoc FieldValues to be removed.
1115 remove.
1116 config: ProjectIssueConfig PB including custom field definitions.
1117
1118 Returns:
1119 Two revised lists of protoc FieldValues to be added and removed,
1120 without enum_types.
1121
1122 SIDE-EFFECT: the labels and labels_remove lists will be extended with
1123 key-value labels corresponding to the enum field values.
1124 """
1125 field_val_strs_add = {}
1126 for field_val in field_vals_add:
1127 field_val_strs_add.setdefault(field_val.field_ref.field_id, []).append(
1128 field_val.value)
1129
1130 field_val_strs_remove = {}
1131 for field_val in field_vals_remove:
1132 field_val_strs_remove.setdefault(field_val.field_ref.field_id, []).append(
1133 field_val.value)
1134
1135 field_helpers.ShiftEnumFieldsIntoLabels(
1136 labels_add, labels_remove, field_val_strs_add, field_val_strs_remove,
1137 config)
1138
1139 # Filter out the fields that were shifted into labels
1140 updated_field_vals_add = [
1141 fv for fv in field_vals_add
1142 if fv.field_ref.field_id in field_val_strs_add]
1143 updated_field_vals_remove = [
1144 fv for fv in field_vals_remove
1145 if fv.field_ref.field_id in field_val_strs_remove]
1146
1147 return updated_field_vals_add, updated_field_vals_remove