Project import generated by Copybara.
GitOrigin-RevId: d9e9e3fb4e31372ec1fb43b178994ca78fa8fe70
diff --git a/search/test/frontendsearchpipeline_test.py b/search/test/frontendsearchpipeline_test.py
new file mode 100644
index 0000000..b2e7fb3
--- /dev/null
+++ b/search/test/frontendsearchpipeline_test.py
@@ -0,0 +1,1339 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+"""Tests for the frontendsearchpipeline module."""
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+
+import mox
+import unittest
+
+from google.appengine.api import memcache
+from google.appengine.api import modules
+from google.appengine.ext import testbed
+from google.appengine.api import urlfetch
+
+import settings
+from framework import framework_helpers
+from framework import sorting
+from framework import urls
+from proto import ast_pb2
+from proto import project_pb2
+from proto import tracker_pb2
+from search import frontendsearchpipeline
+from search import searchpipeline
+from search import query2ast
+from services import service_manager
+from testing import fake
+from testing import testing_helpers
+from tracker import tracker_bizobj
+
+
+# Just an example timestamp. The value does not matter.
+NOW = 2444950132
+
+
+class FrontendSearchPipelineTest(unittest.TestCase):
+
+ def setUp(self):
+ self.config = tracker_bizobj.MakeDefaultProjectIssueConfig(789)
+ self.services = service_manager.Services(
+ user=fake.UserService(),
+ project=fake.ProjectService(),
+ issue=fake.IssueService(),
+ config=fake.ConfigService(),
+ cache_manager=fake.CacheManager())
+ self.services.user.TestAddUser('a@example.com', 111)
+ self.project = self.services.project.TestAddProject('proj', project_id=789)
+ self.mr = testing_helpers.MakeMonorailRequest(
+ path='/p/proj/issues/list', project=self.project)
+ self.mr.me_user_id = 111
+
+ self.issue_1 = fake.MakeTestIssue(
+ 789, 1, 'one', 'New', 111, labels=['Priority-High'])
+ self.services.issue.TestAddIssue(self.issue_1)
+ self.issue_2 = fake.MakeTestIssue(
+ 789, 2, 'two', 'New', 111, labels=['Priority-Low'])
+ self.services.issue.TestAddIssue(self.issue_2)
+ self.issue_3 = fake.MakeTestIssue(
+ 789, 3, 'three', 'New', 111, labels=['Priority-Medium'])
+ self.services.issue.TestAddIssue(self.issue_3)
+ self.mr.sort_spec = 'Priority'
+
+ self.cnxn = self.mr.cnxn
+ self.project = self.mr.project
+ self.auth = self.mr.auth
+ self.me_user_id = self.mr.me_user_id
+ self.query = self.mr.query
+ self.query_project_names = self.mr.query_project_names
+ self.items_per_page = self.mr.num # defaults to 100
+ self.paginate_start = self.mr.start
+ self.paginate_end = self.paginate_start + self.items_per_page
+ self.can = self.mr.can
+ self.group_by_spec = self.mr.group_by_spec
+ self.sort_spec = self.mr.sort_spec
+ self.warnings = self.mr.warnings
+ self.errors = self.mr.errors
+ self.use_cached_searches = self.mr.use_cached_searches
+ self.profiler = self.mr.profiler
+
+ self.mox = mox.Mox()
+ self.testbed = testbed.Testbed()
+ self.testbed.activate()
+ self.testbed.init_user_stub()
+ self.testbed.init_memcache_stub()
+ sorting.InitializeArtValues(self.services)
+
+ def tearDown(self):
+ self.testbed.deactivate()
+ self.mox.UnsetStubs()
+ self.mox.ResetAll()
+
+ def testSearchForIIDs_AllResultsCached_AllAtRiskCached(self):
+ unfiltered_iids = {(1, 'p:v'): [1001, 1011]}
+ nonviewable_iids = {1: set()}
+ self.mox.StubOutWithMock(frontendsearchpipeline, '_StartBackendSearch')
+ frontendsearchpipeline._StartBackendSearch(
+ self.cnxn, ['proj'], [789], mox.IsA(tracker_pb2.ProjectIssueConfig),
+ unfiltered_iids, {}, nonviewable_iids, set(), self.services,
+ self.me_user_id, self.auth.user_id or 0, self.paginate_end,
+ self.query.split(' OR '), self.can, self.group_by_spec, self.sort_spec,
+ self.warnings, self.use_cached_searches).AndReturn([])
+ self.mox.StubOutWithMock(frontendsearchpipeline, '_FinishBackendSearch')
+ frontendsearchpipeline._FinishBackendSearch([])
+ self.mox.ReplayAll()
+
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ pipeline.unfiltered_iids = unfiltered_iids
+ pipeline.nonviewable_iids = nonviewable_iids
+ pipeline.SearchForIIDs()
+ self.mox.VerifyAll()
+ self.assertEqual(2, pipeline.total_count)
+ self.assertEqual([1001, 1011], pipeline.filtered_iids[(1, 'p:v')])
+
+ def testSearchForIIDs_CrossProject_AllViewable(self):
+ self.services.project.TestAddProject('other', project_id=790)
+ unfiltered_iids = {(1, 'p:v'): [1001, 1011, 2001]}
+ nonviewable_iids = {1: set()}
+ self.query_project_names = ['other']
+ self.mox.StubOutWithMock(frontendsearchpipeline, '_StartBackendSearch')
+ frontendsearchpipeline._StartBackendSearch(
+ self.cnxn, ['other', 'proj'], [789, 790],
+ mox.IsA(tracker_pb2.ProjectIssueConfig), unfiltered_iids, {},
+ nonviewable_iids, set(), self.services,
+ self.me_user_id, self.auth.user_id or 0, self.paginate_end,
+ self.query.split(' OR '), self.can, self.group_by_spec, self.sort_spec,
+ self.warnings, self.use_cached_searches).AndReturn([])
+ self.mox.StubOutWithMock(frontendsearchpipeline, '_FinishBackendSearch')
+ frontendsearchpipeline._FinishBackendSearch([])
+ self.mox.ReplayAll()
+
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+
+ pipeline.unfiltered_iids = unfiltered_iids
+ pipeline.nonviewable_iids = nonviewable_iids
+ pipeline.SearchForIIDs()
+ self.mox.VerifyAll()
+ self.assertEqual(3, pipeline.total_count)
+ self.assertEqual([1001, 1011, 2001], pipeline.filtered_iids[(1, 'p:v')])
+
+ def testSearchForIIDs_CrossProject_MembersOnlyOmitted(self):
+ self.services.project.TestAddProject(
+ 'other', project_id=790, access=project_pb2.ProjectAccess.MEMBERS_ONLY)
+ unfiltered_iids = {(1, 'p:v'): [1001, 1011]}
+ nonviewable_iids = {1: set()}
+ # project 'other' gets filtered out before the backend call.
+ self.mr.query_project_names = ['other']
+ self.mox.StubOutWithMock(frontendsearchpipeline, '_StartBackendSearch')
+ frontendsearchpipeline._StartBackendSearch(
+ self.cnxn, ['proj'], [789], mox.IsA(tracker_pb2.ProjectIssueConfig),
+ unfiltered_iids, {}, nonviewable_iids, set(), self.services,
+ self.me_user_id, self.auth.user_id or 0, self.paginate_end,
+ self.query.split(' OR '), self.can, self.group_by_spec, self.sort_spec,
+ self.warnings, self.use_cached_searches).AndReturn([])
+ self.mox.StubOutWithMock(frontendsearchpipeline, '_FinishBackendSearch')
+ frontendsearchpipeline._FinishBackendSearch([])
+ self.mox.ReplayAll()
+
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ pipeline.unfiltered_iids = unfiltered_iids
+ pipeline.nonviewable_iids = nonviewable_iids
+ pipeline.SearchForIIDs()
+ self.mox.VerifyAll()
+ self.assertEqual(2, pipeline.total_count)
+ self.assertEqual([1001, 1011], pipeline.filtered_iids[(1, 'p:v')])
+
+ def testMergeAndSortIssues_EmptyResult(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ pipeline.filtered_iids = {0: [], 1: [], 2: []}
+
+ pipeline.MergeAndSortIssues()
+ self.assertEqual([], pipeline.allowed_iids)
+ self.assertEqual([], pipeline.allowed_results)
+ self.assertEqual({}, pipeline.users_by_id)
+
+ def testMergeAndSortIssues_Normal(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ # In this unit test case we are not calling SearchForIIDs(), instead just
+ # set pipeline.filtered_iids directly.
+ pipeline.filtered_iids = {
+ 0: [],
+ 1: [self.issue_1.issue_id],
+ 2: [self.issue_2.issue_id],
+ 3: [self.issue_3.issue_id]
+ }
+
+ pipeline.MergeAndSortIssues()
+ self.assertEqual(
+ [self.issue_1.issue_id, self.issue_2.issue_id, self.issue_3.issue_id],
+ pipeline.allowed_iids)
+ self.assertEqual(
+ [self.issue_1, self.issue_3, self.issue_2], # high, medium, low.
+ pipeline.allowed_results)
+ self.assertEqual([0, 111], list(pipeline.users_by_id.keys()))
+
+ def testDetermineIssuePosition_Normal(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ # In this unit test case we are not calling SearchForIIDs(), instead just
+ # set pipeline.filtered_iids directly.
+ pipeline.filtered_iids = {
+ 0: [],
+ 1: [self.issue_1.issue_id],
+ 2: [self.issue_2.issue_id],
+ 3: [self.issue_3.issue_id]
+ }
+
+ prev_iid, index, next_iid = pipeline.DetermineIssuePosition(self.issue_3)
+ # The total ordering is issue_1, issue_3, issue_2 for high, med, low.
+ self.assertEqual(self.issue_1.issue_id, prev_iid)
+ self.assertEqual(1, index)
+ self.assertEqual(self.issue_2.issue_id, next_iid)
+
+ def testDetermineIssuePosition_NotInResults(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ # In this unit test case we are not calling SearchForIIDs(), instead just
+ # set pipeline.filtered_iids directly.
+ pipeline.filtered_iids = {
+ 0: [],
+ 1: [self.issue_1.issue_id],
+ 2: [self.issue_2.issue_id],
+ 3: []
+ }
+
+ prev_iid, index, next_iid = pipeline.DetermineIssuePosition(self.issue_3)
+ # The total ordering is issue_1, issue_3, issue_2 for high, med, low.
+ self.assertEqual(None, prev_iid)
+ self.assertEqual(None, index)
+ self.assertEqual(None, next_iid)
+
+ def testDetermineIssuePositionInShard_IssueIsInShard(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ # Let's assume issues 1, 2, and 3 are all in the same shard.
+ pipeline.filtered_iids = {
+ 0: [self.issue_1.issue_id, self.issue_2.issue_id, self.issue_3.issue_id],
+ }
+
+ # The total ordering is issue_1, issue_3, issue_2 for high, med, low.
+ prev_cand, index, next_cand = pipeline._DetermineIssuePositionInShard(
+ 0, self.issue_1, {})
+ self.assertEqual(None, prev_cand)
+ self.assertEqual(0, index)
+ self.assertEqual(self.issue_3, next_cand)
+
+ prev_cand, index, next_cand = pipeline._DetermineIssuePositionInShard(
+ 0, self.issue_3, {})
+ self.assertEqual(self.issue_1, prev_cand)
+ self.assertEqual(1, index)
+ self.assertEqual(self.issue_2, next_cand)
+
+ prev_cand, index, next_cand = pipeline._DetermineIssuePositionInShard(
+ 0, self.issue_2, {})
+ self.assertEqual(self.issue_3, prev_cand)
+ self.assertEqual(2, index)
+ self.assertEqual(None, next_cand)
+
+ def testDetermineIssuePositionInShard_IssueIsNotInShard(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+
+ # The total ordering is issue_1, issue_3, issue_2 for high, med, low.
+ pipeline.filtered_iids = {
+ 0: [self.issue_2.issue_id, self.issue_3.issue_id],
+ }
+ prev_cand, index, next_cand = pipeline._DetermineIssuePositionInShard(
+ 0, self.issue_1, {})
+ self.assertEqual(None, prev_cand)
+ self.assertEqual(0, index)
+ self.assertEqual(self.issue_3, next_cand)
+
+ pipeline.filtered_iids = {
+ 0: [self.issue_1.issue_id, self.issue_2.issue_id],
+ }
+ prev_cand, index, next_cand = pipeline._DetermineIssuePositionInShard(
+ 0, self.issue_3, {})
+ self.assertEqual(self.issue_1, prev_cand)
+ self.assertEqual(1, index)
+ self.assertEqual(self.issue_2, next_cand)
+
+ pipeline.filtered_iids = {
+ 0: [self.issue_1.issue_id, self.issue_3.issue_id],
+ }
+ prev_cand, index, next_cand = pipeline._DetermineIssuePositionInShard(
+ 0, self.issue_2, {})
+ self.assertEqual(self.issue_3, prev_cand)
+ self.assertEqual(2, index)
+ self.assertEqual(None, next_cand)
+
+ def testFetchAllSamples_Empty(self):
+ filtered_iids = {}
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ samples_by_shard, sample_iids_to_shard = pipeline._FetchAllSamples(
+ filtered_iids)
+ self.assertEqual({}, samples_by_shard)
+ self.assertEqual({}, sample_iids_to_shard)
+
+ def testFetchAllSamples_SmallResultsPerShard(self):
+ filtered_iids = {
+ 0: [100, 110, 120],
+ 1: [101, 111, 121],
+ }
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+
+ samples_by_shard, sample_iids_to_shard = pipeline._FetchAllSamples(
+ filtered_iids)
+ self.assertEqual(2, len(samples_by_shard))
+ self.assertEqual(0, len(sample_iids_to_shard))
+
+ def testFetchAllSamples_Normal(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ issues = self.MakeIssues(23)
+ filtered_iids = {
+ 0: [issue.issue_id for issue in issues],
+ }
+
+ samples_by_shard, sample_iids_to_shard = pipeline._FetchAllSamples(
+ filtered_iids)
+ self.assertEqual(1, len(samples_by_shard))
+ self.assertEqual(2, len(samples_by_shard[0]))
+ self.assertEqual(2, len(sample_iids_to_shard))
+ for sample_iid in sample_iids_to_shard:
+ shard_key = sample_iids_to_shard[sample_iid]
+ self.assertIn(sample_iid, filtered_iids[shard_key])
+
+ def testChooseSampleIssues_Empty(self):
+ """When the search gave no results, there cannot be any samples."""
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ issue_ids = []
+ on_hand_issues, needed_iids = pipeline._ChooseSampleIssues(issue_ids)
+ self.assertEqual({}, on_hand_issues)
+ self.assertEqual([], needed_iids)
+
+ def testChooseSampleIssues_Small(self):
+ """When the search gave few results, don't bother with samples."""
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ issue_ids = [78901, 78902]
+ on_hand_issues, needed_iids = pipeline._ChooseSampleIssues(issue_ids)
+ self.assertEqual({}, on_hand_issues)
+ self.assertEqual([], needed_iids)
+
+ def MakeIssues(self, num_issues):
+ issues = []
+ for i in range(num_issues):
+ issue = fake.MakeTestIssue(789, 100 + i, 'samp test', 'New', 111)
+ issues.append(issue)
+ self.services.issue.TestAddIssue(issue)
+ return issues
+
+ def testChooseSampleIssues_Normal(self):
+ """We will choose at least one sample for every 10 results in a shard."""
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ issues = self.MakeIssues(23)
+ issue_ids = [issue.issue_id for issue in issues]
+ on_hand_issues, needed_iids = pipeline._ChooseSampleIssues(issue_ids)
+ self.assertEqual({}, on_hand_issues)
+ self.assertEqual(2, len(needed_iids))
+ for sample_iid in needed_iids:
+ self.assertIn(sample_iid, issue_ids)
+
+ def testLookupNeededUsers(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+
+ pipeline._LookupNeededUsers([])
+ self.assertEqual([], list(pipeline.users_by_id.keys()))
+
+ pipeline._LookupNeededUsers([self.issue_1, self.issue_2, self.issue_3])
+ self.assertEqual([0, 111], list(pipeline.users_by_id.keys()))
+
+ def testPaginate_List(self):
+ pipeline = frontendsearchpipeline.FrontendSearchPipeline(
+ self.cnxn,
+ self.services,
+ self.auth,
+ self.me_user_id,
+ self.query,
+ self.query_project_names,
+ self.items_per_page,
+ self.paginate_start,
+ self.can,
+ self.group_by_spec,
+ self.sort_spec,
+ self.warnings,
+ self.errors,
+ self.use_cached_searches,
+ self.profiler,
+ project=self.project)
+ pipeline.allowed_iids = [
+ self.issue_1.issue_id, self.issue_2.issue_id, self.issue_3.issue_id]
+ pipeline.allowed_results = [self.issue_1, self.issue_2, self.issue_3]
+ pipeline.total_count = len(pipeline.allowed_results)
+ pipeline.Paginate()
+ self.assertEqual(
+ [self.issue_1, self.issue_2, self.issue_3],
+ pipeline.visible_results)
+ self.assertFalse(pipeline.pagination.limit_reached)
+
+
+class FrontendSearchPipelineMethodsTest(unittest.TestCase):
+
+ def setUp(self):
+ self.mox = mox.Mox()
+ self.testbed = testbed.Testbed()
+ self.testbed.activate()
+ self.testbed.init_user_stub()
+ self.testbed.init_memcache_stub()
+
+ self.project_id = 789
+ self.default_config = tracker_bizobj.MakeDefaultProjectIssueConfig(
+ self.project_id)
+ self.services = service_manager.Services(
+ project=fake.ProjectService())
+ self.project = self.services.project.TestAddProject(
+ 'proj', project_id=self.project_id)
+
+ def tearDown(self):
+ self.testbed.deactivate()
+ self.mox.UnsetStubs()
+ self.mox.ResetAll()
+
+ def testMakeBackendCallback(self):
+ called_with = []
+
+ def func(a, b):
+ called_with.append((a, b))
+
+ callback = frontendsearchpipeline._MakeBackendCallback(func, 10, 20)
+ callback()
+ self.assertEqual([(10, 20)], called_with)
+
+ def testParseUserQuery_CheckQuery(self):
+ warnings = []
+ msg = frontendsearchpipeline._CheckQuery(
+ 'cnxn', self.services, 'ok query', self.default_config,
+ [self.project_id], True, warnings=warnings)
+ self.assertIsNone(msg)
+ self.assertEqual([], warnings)
+
+ warnings = []
+ msg = frontendsearchpipeline._CheckQuery(
+ 'cnxn', self.services, 'modified:0-0-0', self.default_config,
+ [self.project_id], True, warnings=warnings)
+ self.assertEqual(
+ 'Could not parse date: 0-0-0',
+ msg)
+
+ warnings = []
+ msg = frontendsearchpipeline._CheckQuery(
+ 'cnxn', self.services, 'blocking:3.14', self.default_config,
+ [self.project_id], True, warnings=warnings)
+ self.assertEqual(
+ 'Could not parse issue reference: 3.14',
+ msg)
+ self.assertEqual([], warnings)
+
+ def testStartBackendSearch(self):
+ # TODO(jrobbins): write this test.
+ pass
+
+ def testFinishBackendSearch(self):
+ # TODO(jrobbins): write this test.
+ pass
+
+ def testGetProjectTimestamps_NoneSet(self):
+ project_shard_timestamps = frontendsearchpipeline._GetProjectTimestamps(
+ [], [])
+ self.assertEqual({}, project_shard_timestamps)
+
+ project_shard_timestamps = frontendsearchpipeline._GetProjectTimestamps(
+ [], [(0, (0, 'p:v')), (1, (1, 'p:v')), (2, (2, 'p:v'))])
+ self.assertEqual({}, project_shard_timestamps)
+
+ project_shard_timestamps = frontendsearchpipeline._GetProjectTimestamps(
+ [789], [(0, (0, 'p:v')), (1, (1, 'p:v')), (2, (2, 'p:v'))])
+ self.assertEqual({}, project_shard_timestamps)
+
+ def testGetProjectTimestamps_SpecificProjects(self):
+ memcache.set('789;0', NOW)
+ memcache.set('789;1', NOW - 1000)
+ memcache.set('789;2', NOW - 3000)
+ project_shard_timestamps = frontendsearchpipeline._GetProjectTimestamps(
+ [789], [(0, (0, 'p:v')), (1, (1, 'p:v')), (2, (2, 'p:v'))])
+ self.assertEqual(
+ { (789, 0): NOW,
+ (789, 1): NOW - 1000,
+ (789, 2): NOW - 3000,
+ },
+ project_shard_timestamps)
+
+ memcache.set('790;0', NOW)
+ memcache.set('790;1', NOW - 10000)
+ memcache.set('790;2', NOW - 30000)
+ project_shard_timestamps = frontendsearchpipeline._GetProjectTimestamps(
+ [789, 790], [(0, (0, 'p:v')), (1, (1, 'p:v')), (2, (2, 'p:v'))])
+ self.assertEqual(
+ { (789, 0): NOW,
+ (789, 1): NOW - 1000,
+ (789, 2): NOW - 3000,
+ (790, 0): NOW,
+ (790, 1): NOW - 10000,
+ (790, 2): NOW - 30000,
+ },
+ project_shard_timestamps)
+
+ def testGetProjectTimestamps_SiteWide(self):
+ memcache.set('all;0', NOW)
+ memcache.set('all;1', NOW - 10000)
+ memcache.set('all;2', NOW - 30000)
+ project_shard_timestamps = frontendsearchpipeline._GetProjectTimestamps(
+ [], [(0, (0, 'p:v')), (1, (1, 'p:v')), (2, (2, 'p:v'))])
+ self.assertEqual(
+ { ('all', 0): NOW,
+ ('all', 1): NOW - 10000,
+ ('all', 2): NOW - 30000,
+ },
+ project_shard_timestamps)
+
+ def testGetNonviewableIIDs_SearchMissSoNoOp(self):
+ """If search cache missed, don't bother looking up nonviewable IIDs."""
+ unfiltered_iids_dict = {} # No cached search results found.
+ rpc_tuples = [] # Nothing should accumulate here in this case.
+ nonviewable_iids = {} # Nothing should accumulate here in this case.
+ processed_invalidations_up_to = 12345
+ frontendsearchpipeline._GetNonviewableIIDs(
+ [789], 111, list(unfiltered_iids_dict.keys()), rpc_tuples,
+ nonviewable_iids, {}, processed_invalidations_up_to, True)
+ self.assertEqual([], rpc_tuples)
+ self.assertEqual({}, nonviewable_iids)
+
+ def testGetNonviewableIIDs_SearchHitThenNonviewableHit(self):
+ """If search cache hit, get nonviewable info from cache."""
+ unfiltered_iids_dict = {
+ 1: [10001, 10021],
+ 2: ['the search result issue_ids do not matter'],
+ }
+ rpc_tuples = [] # Nothing should accumulate here in this case.
+ nonviewable_iids = {} # Our mock results should end up here.
+ processed_invalidations_up_to = 12345
+ memcache.set('nonviewable:789;111;1',
+ ([10001, 10031], processed_invalidations_up_to - 10))
+ memcache.set('nonviewable:789;111;2',
+ ([10002, 10042], processed_invalidations_up_to - 30))
+
+ project_shard_timestamps = {
+ (789, 1): 0, # not stale
+ (789, 2): 0, # not stale
+ }
+ frontendsearchpipeline._GetNonviewableIIDs(
+ [789], 111, list(unfiltered_iids_dict.keys()), rpc_tuples,
+ nonviewable_iids, project_shard_timestamps,
+ processed_invalidations_up_to, True)
+ self.assertEqual([], rpc_tuples)
+ self.assertEqual({1: {10001, 10031}, 2: {10002, 10042}}, nonviewable_iids)
+
+ def testGetNonviewableIIDs_SearchHitNonviewableMissSoStartRPC(self):
+ """If search hit and n-v miss, create RPCs to get nonviewable info."""
+ self.mox.StubOutWithMock(
+ frontendsearchpipeline, '_StartBackendNonviewableCall')
+ unfiltered_iids_dict = {
+ 2: ['the search result issue_ids do not matter'],
+ }
+ rpc_tuples = [] # One RPC object should accumulate here.
+ nonviewable_iids = {} # This will stay empty until RPCs complete.
+ processed_invalidations_up_to = 12345
+ # Nothing is set in memcache for this case.
+ a_fake_rpc = testing_helpers.Blank(callback=None)
+ frontendsearchpipeline._StartBackendNonviewableCall(
+ 789, 111, 2, processed_invalidations_up_to).AndReturn(a_fake_rpc)
+ self.mox.ReplayAll()
+
+ frontendsearchpipeline._GetNonviewableIIDs(
+ [789], 111, list(unfiltered_iids_dict.keys()), rpc_tuples,
+ nonviewable_iids, {}, processed_invalidations_up_to, True)
+ self.mox.VerifyAll()
+ _, sid_0, rpc_0 = rpc_tuples[0]
+ self.assertEqual(2, sid_0)
+ self.assertEqual({}, nonviewable_iids)
+ self.assertEqual(a_fake_rpc, rpc_0)
+ self.assertIsNotNone(a_fake_rpc.callback)
+
+ def testAccumulateNonviewableIIDs_MemcacheHitForProject(self):
+ processed_invalidations_up_to = 12345
+ cached_dict = {
+ '789;111;2': ([10002, 10042], processed_invalidations_up_to - 10),
+ '789;111;3': ([10003, 10093], processed_invalidations_up_to - 30),
+ }
+ rpc_tuples = [] # Nothing should accumulate here.
+ nonviewable_iids = {1: {10001}} # This will gain the shard 2 values.
+ project_shard_timestamps = {
+ (789, 1): 0, # not stale
+ (789, 2): 0, # not stale
+ }
+ frontendsearchpipeline._AccumulateNonviewableIIDs(
+ 789, 111, 2, cached_dict, nonviewable_iids, project_shard_timestamps,
+ rpc_tuples, processed_invalidations_up_to)
+ self.assertEqual([], rpc_tuples)
+ self.assertEqual({1: {10001}, 2: {10002, 10042}}, nonviewable_iids)
+
+ def testAccumulateNonviewableIIDs_MemcacheStaleForProject(self):
+ self.mox.StubOutWithMock(
+ frontendsearchpipeline, '_StartBackendNonviewableCall')
+ processed_invalidations_up_to = 12345
+ cached_dict = {
+ '789;111;2': ([10002, 10042], processed_invalidations_up_to - 10),
+ '789;111;3': ([10003, 10093], processed_invalidations_up_to - 30),
+ }
+ rpc_tuples = [] # Nothing should accumulate here.
+ nonviewable_iids = {1: {10001}} # Nothing added here until RPC completes
+ project_shard_timestamps = {
+ (789, 1): 0, # not stale
+ (789, 2): processed_invalidations_up_to, # stale!
+ }
+ a_fake_rpc = testing_helpers.Blank(callback=None)
+ frontendsearchpipeline._StartBackendNonviewableCall(
+ 789, 111, 2, processed_invalidations_up_to).AndReturn(a_fake_rpc)
+ self.mox.ReplayAll()
+
+ frontendsearchpipeline._AccumulateNonviewableIIDs(
+ 789, 111, 2, cached_dict, nonviewable_iids, project_shard_timestamps,
+ rpc_tuples, processed_invalidations_up_to)
+ self.mox.VerifyAll()
+ _, sid_0, rpc_0 = rpc_tuples[0]
+ self.assertEqual(2, sid_0)
+ self.assertEqual(a_fake_rpc, rpc_0)
+ self.assertIsNotNone(a_fake_rpc.callback)
+ self.assertEqual({1: {10001}}, nonviewable_iids)
+
+ def testAccumulateNonviewableIIDs_MemcacheHitForWholeSite(self):
+ processed_invalidations_up_to = 12345
+ cached_dict = {
+ 'all;111;2': ([10002, 10042], processed_invalidations_up_to - 10),
+ 'all;111;3': ([10003, 10093], processed_invalidations_up_to - 30),
+ }
+ rpc_tuples = [] # Nothing should accumulate here.
+ nonviewable_iids = {1: {10001}} # This will gain the shard 2 values.
+ project_shard_timestamps = {
+ (None, 1): 0, # not stale
+ (None, 2): 0, # not stale
+ }
+ frontendsearchpipeline._AccumulateNonviewableIIDs(
+ None, 111, 2, cached_dict, nonviewable_iids, project_shard_timestamps,
+ rpc_tuples, processed_invalidations_up_to)
+ self.assertEqual([], rpc_tuples)
+ self.assertEqual({1: {10001}, 2: {10002, 10042}}, nonviewable_iids)
+
+ def testAccumulateNonviewableIIDs_MemcacheMissSoStartRPC(self):
+ self.mox.StubOutWithMock(
+ frontendsearchpipeline, '_StartBackendNonviewableCall')
+ cached_dict = {} # Nothing here, so it is an at-risk cache miss.
+ rpc_tuples = [] # One RPC should accumulate here.
+ nonviewable_iids = {1: {10001}} # Nothing added here until RPC completes.
+ processed_invalidations_up_to = 12345
+ a_fake_rpc = testing_helpers.Blank(callback=None)
+ frontendsearchpipeline._StartBackendNonviewableCall(
+ 789, 111, 2, processed_invalidations_up_to).AndReturn(a_fake_rpc)
+ self.mox.ReplayAll()
+
+ frontendsearchpipeline._AccumulateNonviewableIIDs(
+ 789, 111, 2, cached_dict, nonviewable_iids, {}, rpc_tuples,
+ processed_invalidations_up_to)
+ self.mox.VerifyAll()
+ _, sid_0, rpc_0 = rpc_tuples[0]
+ self.assertEqual(2, sid_0)
+ self.assertEqual(a_fake_rpc, rpc_0)
+ self.assertIsNotNone(a_fake_rpc.callback)
+ self.assertEqual({1: {10001}}, nonviewable_iids)
+
+ def testGetCachedSearchResults(self):
+ # TODO(jrobbins): Write this test.
+ pass
+
+ def testMakeBackendRequestHeaders(self):
+ headers = frontendsearchpipeline._MakeBackendRequestHeaders(False)
+ self.assertNotIn('X-AppEngine-FailFast', headers)
+ headers = frontendsearchpipeline._MakeBackendRequestHeaders(True)
+ self.assertEqual('Yes', headers['X-AppEngine-FailFast'])
+
+ def testStartBackendSearchCall(self):
+ self.mox.StubOutWithMock(urlfetch, 'create_rpc')
+ self.mox.StubOutWithMock(urlfetch, 'make_fetch_call')
+ self.mox.StubOutWithMock(modules, 'get_hostname')
+ a_fake_rpc = testing_helpers.Blank(callback=None)
+ urlfetch.create_rpc(deadline=settings.backend_deadline).AndReturn(
+ a_fake_rpc)
+ modules.get_hostname(module='besearch')
+ urlfetch.make_fetch_call(
+ a_fake_rpc, mox.StrContains(
+ urls.BACKEND_SEARCH + '?groupby=cc&invalidation_timestep=12345&'
+ +'logged_in_user_id=777&me_user_ids=555&'
+ +'num=201&projects=proj&q=priority%3Dhigh&shard_id=2&start=0'),
+ follow_redirects=False,
+ headers=mox.IsA(dict))
+ self.mox.ReplayAll()
+
+ processed_invalidations_up_to = 12345
+ me_user_ids = [555]
+ logged_in_user_id = 777
+ new_url_num = 201
+ frontendsearchpipeline._StartBackendSearchCall(
+ ['proj'], (2, 'priority=high'),
+ processed_invalidations_up_to,
+ me_user_ids,
+ logged_in_user_id,
+ new_url_num,
+ group_by_spec='cc')
+ self.mox.VerifyAll()
+
+ def testStartBackendSearchCall_SortAndGroup(self):
+ self.mox.StubOutWithMock(urlfetch, 'create_rpc')
+ self.mox.StubOutWithMock(urlfetch, 'make_fetch_call')
+ self.mox.StubOutWithMock(modules, 'get_hostname')
+ a_fake_rpc = testing_helpers.Blank(callback=None)
+ urlfetch.create_rpc(deadline=settings.backend_deadline).AndReturn(
+ a_fake_rpc)
+ modules.get_hostname(module='besearch')
+ urlfetch.make_fetch_call(
+ a_fake_rpc,
+ mox.StrContains(
+ urls.BACKEND_SEARCH + '?groupby=bar&' +
+ 'invalidation_timestep=12345&' +
+ 'logged_in_user_id=777&me_user_ids=555&num=201&projects=proj&' +
+ 'q=priority%3Dhigh&shard_id=2&sort=foo&start=0'),
+ follow_redirects=False,
+ headers=mox.IsA(dict))
+ self.mox.ReplayAll()
+
+ processed_invalidations_up_to = 12345
+ me_user_ids = [555]
+ logged_in_user_id = 777
+ new_url_num = 201
+ sort_spec = 'foo'
+ group_by_spec = 'bar'
+ frontendsearchpipeline._StartBackendSearchCall(
+ ['proj'], (2, 'priority=high'),
+ processed_invalidations_up_to,
+ me_user_ids,
+ logged_in_user_id,
+ new_url_num,
+ sort_spec=sort_spec,
+ group_by_spec=group_by_spec)
+ self.mox.VerifyAll()
+
+ def testStartBackendNonviewableCall(self):
+ self.mox.StubOutWithMock(urlfetch, 'create_rpc')
+ self.mox.StubOutWithMock(urlfetch, 'make_fetch_call')
+ self.mox.StubOutWithMock(modules, 'get_hostname')
+ a_fake_rpc = testing_helpers.Blank(callback=None)
+ urlfetch.create_rpc(deadline=settings.backend_deadline).AndReturn(
+ a_fake_rpc)
+ modules.get_hostname(module='besearch')
+ urlfetch.make_fetch_call(
+ a_fake_rpc, mox.StrContains(urls.BACKEND_NONVIEWABLE),
+ follow_redirects=False, headers=mox.IsA(dict))
+ self.mox.ReplayAll()
+
+ processed_invalidations_up_to = 12345
+ frontendsearchpipeline._StartBackendNonviewableCall(
+ 789, 111, 2, processed_invalidations_up_to)
+ self.mox.VerifyAll()
+
+ def testHandleBackendSearchResponse_500(self):
+ response_str = 'There was a problem processing the query.'
+ rpc = testing_helpers.Blank(
+ get_result=lambda: testing_helpers.Blank(
+ content=response_str, status_code=500))
+ rpc_tuple = (NOW, 2, rpc)
+ rpc_tuples = [] # Nothing should be added for this case.
+ filtered_iids = {} # Search results should accumlate here, per-shard.
+ search_limit_reached = {} # Booleans accumulate here, per-shard.
+ processed_invalidations_up_to = 12345
+
+ me_user_ids = [111]
+ logged_in_user_id = 0
+ new_url_num = 100
+ error_responses = set()
+
+ self.mox.StubOutWithMock(frontendsearchpipeline, '_StartBackendSearchCall')
+ frontendsearchpipeline._HandleBackendSearchResponse(
+ ['proj'], rpc_tuple, rpc_tuples, 0, filtered_iids, search_limit_reached,
+ processed_invalidations_up_to, error_responses, me_user_ids,
+ logged_in_user_id, new_url_num, 1, None, None)
+ self.assertEqual([], rpc_tuples)
+ self.assertIn(2, error_responses)
+
+ def testHandleBackendSearchResponse_Error(self):
+ response_str = (
+ '})]\'\n'
+ '{'
+ ' "unfiltered_iids": [],'
+ ' "search_limit_reached": false,'
+ ' "error": "Invalid query"'
+ '}'
+ )
+ rpc = testing_helpers.Blank(
+ get_result=lambda: testing_helpers.Blank(
+ content=response_str, status_code=200))
+ rpc_tuple = (NOW, 2, rpc)
+ rpc_tuples = [] # Nothing should be added for this case.
+ filtered_iids = {} # Search results should accumlate here, per-shard.
+ search_limit_reached = {} # Booleans accumulate here, per-shard.
+ processed_invalidations_up_to = 12345
+
+ me_user_ids = [111]
+ logged_in_user_id = 0
+ new_url_num = 100
+ error_responses = set()
+ frontendsearchpipeline._HandleBackendSearchResponse(
+ ['proj'], rpc_tuple, rpc_tuples, 2, filtered_iids, search_limit_reached,
+ processed_invalidations_up_to, error_responses, me_user_ids,
+ logged_in_user_id, new_url_num, 1, None, None)
+ self.assertEqual([], rpc_tuples)
+ self.assertEqual({2: []}, filtered_iids)
+ self.assertEqual({2: False}, search_limit_reached)
+ self.assertEqual({2}, error_responses)
+
+ def testHandleBackendSearchResponse_Normal(self):
+ response_str = (
+ '})]\'\n'
+ '{'
+ ' "unfiltered_iids": [10002, 10042],'
+ ' "search_limit_reached": false'
+ '}'
+ )
+ rpc = testing_helpers.Blank(
+ get_result=lambda: testing_helpers.Blank(
+ content=response_str, status_code=200))
+ rpc_tuple = (NOW, 2, rpc)
+ rpc_tuples = [] # Nothing should be added for this case.
+ filtered_iids = {} # Search results should accumlate here, per-shard.
+ search_limit_reached = {} # Booleans accumulate here, per-shard.
+ processed_invalidations_up_to = 12345
+
+ me_user_ids = [111]
+ logged_in_user_id = 0
+ new_url_num = 100
+ error_responses = set()
+ frontendsearchpipeline._HandleBackendSearchResponse(
+ ['proj'], rpc_tuple, rpc_tuples, 2, filtered_iids, search_limit_reached,
+ processed_invalidations_up_to, error_responses, me_user_ids,
+ logged_in_user_id, new_url_num, 1, None, None)
+ self.assertEqual([], rpc_tuples)
+ self.assertEqual({2: [10002, 10042]}, filtered_iids)
+ self.assertEqual({2: False}, search_limit_reached)
+
+ def testHandleBackendSearchResponse_TriggersRetry(self):
+ response_str = None
+ rpc = testing_helpers.Blank(
+ get_result=lambda: testing_helpers.Blank(content=response_str))
+ rpc_tuple = (NOW, 2, rpc)
+ rpc_tuples = [] # New RPC should be appended here
+ filtered_iids = {} # No change here until retry completes.
+ search_limit_reached = {} # No change here until retry completes.
+ processed_invalidations_up_to = 12345
+ error_responses = set()
+
+ me_user_ids = [111]
+ logged_in_user_id = 0
+ new_url_num = 100
+
+ self.mox.StubOutWithMock(frontendsearchpipeline, '_StartBackendSearchCall')
+ a_fake_rpc = testing_helpers.Blank(callback=None)
+ rpc = frontendsearchpipeline._StartBackendSearchCall(
+ ['proj'],
+ 2,
+ processed_invalidations_up_to,
+ me_user_ids,
+ logged_in_user_id,
+ new_url_num,
+ can=1,
+ group_by_spec=None,
+ sort_spec=None,
+ failfast=False).AndReturn(a_fake_rpc)
+ self.mox.ReplayAll()
+
+ frontendsearchpipeline._HandleBackendSearchResponse(
+ ['proj'], rpc_tuple, rpc_tuples, 2, filtered_iids, search_limit_reached,
+ processed_invalidations_up_to, error_responses, me_user_ids,
+ logged_in_user_id, new_url_num, 1, None, None)
+ self.mox.VerifyAll()
+ _, retry_shard_id, retry_rpc = rpc_tuples[0]
+ self.assertEqual(2, retry_shard_id)
+ self.assertEqual(a_fake_rpc, retry_rpc)
+ self.assertIsNotNone(retry_rpc.callback)
+ self.assertEqual({}, filtered_iids)
+ self.assertEqual({}, search_limit_reached)
+
+ def testHandleBackendNonviewableResponse_Error(self):
+ response_str = 'There was an error.'
+ rpc = testing_helpers.Blank(
+ get_result=lambda: testing_helpers.Blank(
+ content=response_str,
+ status_code=500
+ ))
+ rpc_tuple = (NOW, 2, rpc)
+ rpc_tuples = [] # Nothing should be added for this case.
+ nonviewable_iids = {} # At-risk issue IDs should accumlate here, per-shard.
+ processed_invalidations_up_to = 12345
+
+ self.mox.StubOutWithMock(
+ frontendsearchpipeline, '_StartBackendNonviewableCall')
+ frontendsearchpipeline._HandleBackendNonviewableResponse(
+ 789, 111, 2, rpc_tuple, rpc_tuples, 0, nonviewable_iids,
+ processed_invalidations_up_to)
+ self.assertEqual([], rpc_tuples)
+ self.assertNotEqual({2: {10002, 10042}}, nonviewable_iids)
+
+ def testHandleBackendNonviewableResponse_Normal(self):
+ response_str = (
+ '})]\'\n'
+ '{'
+ ' "nonviewable": [10002, 10042]'
+ '}'
+ )
+ rpc = testing_helpers.Blank(
+ get_result=lambda: testing_helpers.Blank(
+ content=response_str,
+ status_code=200
+ ))
+ rpc_tuple = (NOW, 2, rpc)
+ rpc_tuples = [] # Nothing should be added for this case.
+ nonviewable_iids = {} # At-risk issue IDs should accumlate here, per-shard.
+ processed_invalidations_up_to = 12345
+
+ frontendsearchpipeline._HandleBackendNonviewableResponse(
+ 789, 111, 2, rpc_tuple, rpc_tuples, 2, nonviewable_iids,
+ processed_invalidations_up_to)
+ self.assertEqual([], rpc_tuples)
+ self.assertEqual({2: {10002, 10042}}, nonviewable_iids)
+
+ def testHandleBackendAtRiskResponse_TriggersRetry(self):
+ response_str = None
+ rpc = testing_helpers.Blank(
+ get_result=lambda: testing_helpers.Blank(content=response_str))
+ rpc_tuple = (NOW, 2, rpc)
+ rpc_tuples = [] # New RPC should be appended here
+ nonviewable_iids = {} # No change here until retry completes.
+ processed_invalidations_up_to = 12345
+
+ self.mox.StubOutWithMock(
+ frontendsearchpipeline, '_StartBackendNonviewableCall')
+ a_fake_rpc = testing_helpers.Blank(callback=None)
+ rpc = frontendsearchpipeline._StartBackendNonviewableCall(
+ 789, 111, 2, processed_invalidations_up_to, failfast=False
+ ).AndReturn(a_fake_rpc)
+ self.mox.ReplayAll()
+
+ frontendsearchpipeline._HandleBackendNonviewableResponse(
+ 789, 111, 2, rpc_tuple, rpc_tuples, 2, nonviewable_iids,
+ processed_invalidations_up_to)
+ self.mox.VerifyAll()
+ _, retry_shard_id, retry_rpc = rpc_tuples[0]
+ self.assertEqual(2, retry_shard_id)
+ self.assertIsNotNone(retry_rpc.callback)
+ self.assertEqual(a_fake_rpc, retry_rpc)
+ self.assertEqual({}, nonviewable_iids)
+
+ def testSortIssues(self):
+ services = service_manager.Services(
+ cache_manager=fake.CacheManager())
+ sorting.InitializeArtValues(services)
+
+ issue_1 = fake.MakeTestIssue(
+ 789, 1, 'one', 'New', 111, labels=['Priority-High'])
+ issue_2 = fake.MakeTestIssue(
+ 789, 2, 'two', 'New', 111, labels=['Priority-Low'])
+ issue_3 = fake.MakeTestIssue(
+ 789, 3, 'three', 'New', 111, labels=['Priority-Medium'])
+ issues = [issue_1, issue_2, issue_3]
+ config = tracker_bizobj.MakeDefaultProjectIssueConfig(789)
+
+ sorted_issues = frontendsearchpipeline._SortIssues(
+ issues, config, {}, '', 'priority')
+
+ self.assertEqual(
+ [issue_1, issue_3, issue_2], # Order is high, medium, low.
+ sorted_issues)
+
+
+class FrontendSearchPipelineShardMethodsTest(unittest.TestCase):
+
+ def setUp(self):
+ self.sharded_iids = {
+ (0, 'p:v'): [10, 20, 30, 40, 50],
+ (1, 'p:v'): [21, 41, 61, 81],
+ (2, 'p:v'): [42, 52, 62, 72, 102],
+ (3, 'p:v'): [],
+ }
+
+ def testTotalLength_Empty(self):
+ """If there were no results, the length of the sharded list is zero."""
+ self.assertEqual(0, frontendsearchpipeline._TotalLength({}))
+
+ def testTotalLength_Normal(self):
+ """The length of the sharded list is the sum of the shard lengths."""
+ self.assertEqual(
+ 14, frontendsearchpipeline._TotalLength(self.sharded_iids))
+
+ def testReverseShards_Empty(self):
+ """Reversing an empty sharded list is still empty."""
+ empty_sharded_iids = {}
+ frontendsearchpipeline._ReverseShards(empty_sharded_iids)
+ self.assertEqual({}, empty_sharded_iids)
+
+ def testReverseShards_Normal(self):
+ """Reversing a sharded list reverses each shard."""
+ frontendsearchpipeline._ReverseShards(self.sharded_iids)
+ self.assertEqual(
+ {(0, 'p:v'): [50, 40, 30, 20, 10],
+ (1, 'p:v'): [81, 61, 41, 21],
+ (2, 'p:v'): [102, 72, 62, 52, 42],
+ (3, 'p:v'): [],
+ },
+ self.sharded_iids)
+
+ def testTrimShardedIIDs_Empty(self):
+ """If the sharded list is empty, trimming it makes no change."""
+ empty_sharded_iids = {}
+ frontendsearchpipeline._TrimEndShardedIIDs(empty_sharded_iids, [], 12)
+ self.assertEqual({}, empty_sharded_iids)
+
+ frontendsearchpipeline._TrimEndShardedIIDs(
+ empty_sharded_iids,
+ [(100, (0, 'p:v')), (88, (8, 'p:v')), (99, (9, 'p:v'))],
+ 12)
+ self.assertEqual({}, empty_sharded_iids)
+
+ def testTrimShardedIIDs_NoSamples(self):
+ """If there are no samples, we don't trim off any IIDs."""
+ orig_sharded_iids = {
+ shard_id: iids[:] for shard_id, iids in self.sharded_iids.items()}
+ num_trimmed = frontendsearchpipeline._TrimEndShardedIIDs(
+ self.sharded_iids, [], 12)
+ self.assertEqual(0, num_trimmed)
+ self.assertEqual(orig_sharded_iids, self.sharded_iids)
+
+ num_trimmed = frontendsearchpipeline._TrimEndShardedIIDs(
+ self.sharded_iids, [], 1)
+ self.assertEqual(0, num_trimmed)
+ self.assertEqual(orig_sharded_iids, self.sharded_iids)
+
+ def testTrimShardedIIDs_Normal(self):
+ """The first 3 samples contribute all needed IIDs, so trim off the rest."""
+ samples = [(30, (0, 'p:v')), (41, (1, 'p:v')), (62, (2, 'p:v')),
+ (40, (0, 'p:v')), (81, (1, 'p:v'))]
+ num_trimmed = frontendsearchpipeline._TrimEndShardedIIDs(
+ self.sharded_iids, samples, 5)
+ self.assertEqual(2 + 1 + 0 + 0, num_trimmed)
+ self.assertEqual(
+ { # shard_id: iids before lower-bound + iids before 1st excess sample.
+ (0, 'p:v'): [10, 20] + [30],
+ (1, 'p:v'): [21] + [41, 61],
+ (2, 'p:v'): [42, 52] + [62, 72, 102],
+ (3, 'p:v'): [] + []},
+ self.sharded_iids)
+
+ def testCalcSamplePositions_Empty(self):
+ sharded_iids = {0: []}
+ samples = []
+ self.assertEqual(
+ [], frontendsearchpipeline._CalcSamplePositions(sharded_iids, samples))
+
+ sharded_iids = {0: [10, 20, 30, 40]}
+ samples = []
+ self.assertEqual(
+ [], frontendsearchpipeline._CalcSamplePositions(sharded_iids, samples))
+
+ sharded_iids = {0: []}
+ # E.g., the IIDs 2 and 4 might have been trimmed out in the forward phase.
+ # But we still have them in the list for the backwards phase, and they
+ # should just not contribute anything to the result.
+ samples = [(2, (2, 'p:v')), (4, (4, 'p:v'))]
+ self.assertEqual(
+ [], frontendsearchpipeline._CalcSamplePositions(sharded_iids, samples))
+
+ def testCalcSamplePositions_Normal(self):
+ samples = [(30, (0, 'p:v')), (41, (1, 'p:v')), (62, (2, 'p:v')),
+ (40, (0, 'p:v')), (81, (1, 'p:v'))]
+ self.assertEqual(
+ [(30, (0, 'p:v'), 2),
+ (41, (1, 'p:v'), 1),
+ (62, (2, 'p:v'), 2),
+ (40, (0, 'p:v'), 3),
+ (81, (1, 'p:v'), 3)],
+ frontendsearchpipeline._CalcSamplePositions(self.sharded_iids, samples))