blob: d977dea250d9de0867bad49d23f5c770e5ae6c89 [file] [log] [blame]
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +01001# Copyright 2016 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
Copybara854996b2021-09-07 19:36:02 +00004
5"""Unit tests for tracker_fulltext module."""
6from __future__ import print_function
7from __future__ import division
8from __future__ import absolute_import
9
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +010010import six
Copybara854996b2021-09-07 19:36:02 +000011import unittest
12
Adrià Vilanova Martínez9f9ade52022-10-10 23:20:11 +020013try:
14 from mox3 import mox
15except ImportError:
16 import mox
Copybara854996b2021-09-07 19:36:02 +000017
18from google.appengine.api import search
19
20import settings
21from framework import framework_views
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +010022from mrproto import ast_pb2
23from mrproto import tracker_pb2
Copybara854996b2021-09-07 19:36:02 +000024from services import fulltext_helpers
25from services import tracker_fulltext
26from testing import fake
27from tracker import tracker_bizobj
28
29
30class TrackerFulltextTest(unittest.TestCase):
31
32 def setUp(self):
33 self.mox = mox.Mox()
34 self.mock_index = self.mox.CreateMockAnything()
35 self.mox.StubOutWithMock(search, 'Index')
36 self.docs = None
37 self.cnxn = 'fake connection'
38 self.user_service = fake.UserService()
39 self.user_service.TestAddUser('test@example.com', 111)
40 self.issue_service = fake.IssueService()
41 self.config_service = fake.ConfigService()
42
43 self.issue = fake.MakeTestIssue(
44 123, 1, 'test summary', 'New', 111)
45 self.issue_service.TestAddIssue(self.issue)
46 self.comment = tracker_pb2.IssueComment(
47 project_id=789, issue_id=self.issue.issue_id, user_id=111,
48 content='comment content',
49 attachments=[
50 tracker_pb2.Attachment(filename='hello.c'),
51 tracker_pb2.Attachment(filename='hello.h')])
52 self.issue_service.TestAddComment(self.comment, 1)
53 self.users_by_id = framework_views.MakeAllUserViews(
54 self.cnxn, self.user_service, [111])
55
56 def tearDown(self):
57 self.mox.UnsetStubs()
58 self.mox.ResetAll()
59
60 def RecordDocs(self, docs):
61 self.docs = docs
62
63 def SetUpIndexIssues(self):
64 search.Index(name=settings.search_index_name_format % 1).AndReturn(
65 self.mock_index)
66 self.mock_index.put(mox.IgnoreArg()).WithSideEffects(self.RecordDocs)
67
68 def testIndexIssues(self):
69 self.SetUpIndexIssues()
70 self.mox.ReplayAll()
71 tracker_fulltext.IndexIssues(
72 self.cnxn, [self.issue], self.user_service, self.issue_service,
73 self.config_service)
74 self.mox.VerifyAll()
75 self.assertEqual(1, len(self.docs))
76 issue_doc = self.docs[0]
77 self.assertEqual(123, issue_doc.fields[0].value)
78 self.assertEqual('test summary', issue_doc.fields[1].value)
79
80 def SetUpCreateIssueSearchDocuments(self):
81 self.mox.StubOutWithMock(tracker_fulltext, '_IndexDocsInShard')
82 tracker_fulltext._IndexDocsInShard(1, mox.IgnoreArg()).WithSideEffects(
83 lambda shard_id, docs: self.RecordDocs(docs))
84
85 def testCreateIssueSearchDocuments_Normal(self):
86 self.SetUpCreateIssueSearchDocuments()
87 self.mox.ReplayAll()
88 config_dict = {123: tracker_bizobj.MakeDefaultProjectIssueConfig(123)}
89 tracker_fulltext._CreateIssueSearchDocuments(
90 [self.issue], {self.issue.issue_id: [self.comment]}, self.users_by_id,
91 config_dict)
92 self.mox.VerifyAll()
93 self.assertEqual(1, len(self.docs))
94 issue_doc = self.docs[0]
95 self.assertEqual(5, len(issue_doc.fields))
96 self.assertEqual(123, issue_doc.fields[0].value)
97 self.assertEqual('test summary', issue_doc.fields[1].value)
98 self.assertEqual('test@example.com comment content hello.c hello.h',
99 issue_doc.fields[3].value)
100 self.assertEqual('', issue_doc.fields[4].value)
101
102 def testCreateIssueSearchDocuments_NoIndexableComments(self):
103 """Sometimes all comments on a issue are spam or deleted."""
104 self.SetUpCreateIssueSearchDocuments()
105 self.mox.ReplayAll()
106 config_dict = {123: tracker_bizobj.MakeDefaultProjectIssueConfig(123)}
107 self.comment.deleted_by = 111
108 tracker_fulltext._CreateIssueSearchDocuments(
109 [self.issue], {self.issue.issue_id: [self.comment]}, self.users_by_id,
110 config_dict)
111 self.mox.VerifyAll()
112 self.assertEqual(1, len(self.docs))
113 issue_doc = self.docs[0]
114 self.assertEqual(5, len(issue_doc.fields))
115 self.assertEqual(123, issue_doc.fields[0].value)
116 self.assertEqual('test summary', issue_doc.fields[1].value)
117 self.assertEqual('', issue_doc.fields[3].value)
118 self.assertEqual('', issue_doc.fields[4].value)
119
120 def testCreateIssueSearchDocuments_CustomFields(self):
121 self.SetUpCreateIssueSearchDocuments()
122 self.mox.ReplayAll()
123 config = tracker_bizobj.MakeDefaultProjectIssueConfig(123)
124 config_dict = {123: tracker_bizobj.MakeDefaultProjectIssueConfig(123)}
125 int_field = tracker_bizobj.MakeFieldDef(
126 1, 123, 'CustomInt', tracker_pb2.FieldTypes.INT_TYPE, None, False,
127 False, False, None, None, None, None, False, None, None, None,
128 'no_action', 'A custom int field', False)
129 int_field_value = tracker_bizobj.MakeFieldValue(
130 1, 42, None, None, False, None, None)
131 str_field = tracker_bizobj.MakeFieldDef(
132 2, 123, 'CustomStr', tracker_pb2.FieldTypes.STR_TYPE, None, False,
133 False, False, None, None, None, None, False, None, None, None,
134 'no_action', 'A custom string field', False)
135 str_field_value = tracker_bizobj.MakeFieldValue(
136 2, None, u'\xf0\x9f\x92\x96\xef\xb8\x8f', None, None, None, False)
137 # TODO(jrobbins): user-type field 3
138 date_field = tracker_bizobj.MakeFieldDef(
139 4, 123, 'CustomDate', tracker_pb2.FieldTypes.DATE_TYPE, None, False,
140 False, False, None, None, None, None, False, None, None, None,
141 'no_action', 'A custom date field', False)
142 date_field_value = tracker_bizobj.MakeFieldValue(
143 4, None, None, None, 1234567890, None, False)
144 config.field_defs.extend([int_field, str_field, date_field])
145 self.issue.field_values.extend([
146 int_field_value, str_field_value, date_field_value])
147
148 tracker_fulltext._CreateIssueSearchDocuments(
149 [self.issue], {self.issue.issue_id: [self.comment]}, self.users_by_id,
150 config_dict)
151 self.mox.VerifyAll()
152 self.assertEqual(1, len(self.docs))
153 issue_doc = self.docs[0]
154 metadata = issue_doc.fields[2]
155 self.assertEqual(
156 u'New test@example.com [] 42 \xf0\x9f\x92\x96\xef\xb8\x8f 2009-02-13 ',
157 metadata.value)
158
159 def testExtractCommentText(self):
160 extracted_text = tracker_fulltext._ExtractCommentText(
161 self.comment, self.users_by_id)
162 self.assertEqual(
163 'test@example.com comment content hello.c hello.h',
164 extracted_text)
165
166 def testIndexableComments_NumberOfComments(self):
167 """We consider at most 100 initial comments and 500 most recent comments."""
168 comments = [self.comment]
169 indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
170 self.assertEqual(1, len(indexable))
171
172 comments = [self.comment] * 100
173 indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
174 self.assertEqual(100, len(indexable))
175
176 comments = [self.comment] * 101
177 indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
178 self.assertEqual(101, len(indexable))
179
180 comments = [self.comment] * 600
181 indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
182 self.assertEqual(600, len(indexable))
183
184 comments = [self.comment] * 601
185 indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
186 self.assertEqual(600, len(indexable))
187 self.assertNotIn(100, indexable)
188
189 def testIndexableComments_NumberOfChars(self):
190 """We consider comments that can fit into the search index document."""
191 self.comment.content = 'x' * 1000
192 comments = [self.comment] * 100
193
194 indexable = tracker_fulltext._IndexableComments(
195 comments, self.users_by_id, remaining_chars=100000)
196 self.assertEqual(100, len(indexable))
197
198 indexable = tracker_fulltext._IndexableComments(
199 comments, self.users_by_id, remaining_chars=50000)
200 self.assertEqual(50, len(indexable))
201 indexable = tracker_fulltext._IndexableComments(
202 comments, self.users_by_id, remaining_chars=50999)
203 self.assertEqual(50, len(indexable))
204
205 indexable = tracker_fulltext._IndexableComments(
206 comments, self.users_by_id, remaining_chars=999)
207 self.assertEqual(0, len(indexable))
208
209 indexable = tracker_fulltext._IndexableComments(
210 comments, self.users_by_id, remaining_chars=0)
211 self.assertEqual(0, len(indexable))
212
213 indexable = tracker_fulltext._IndexableComments(
214 comments, self.users_by_id, remaining_chars=-1)
215 self.assertEqual(0, len(indexable))
216
217 def SetUpUnindexIssues(self):
218 search.Index(name=settings.search_index_name_format % 1).AndReturn(
219 self.mock_index)
220 self.mock_index.delete(['1'])
221
222 def testUnindexIssues(self):
223 self.SetUpUnindexIssues()
224 self.mox.ReplayAll()
225 tracker_fulltext.UnindexIssues([1])
226 self.mox.VerifyAll()
227
228 def SetUpSearchIssueFullText(self):
229 self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch')
230 fulltext_helpers.ComprehensiveSearch(
231 '(project_id:789) (summary:"test")',
232 settings.search_index_name_format % 1).AndReturn([123, 234])
233
234 def testSearchIssueFullText_Normal(self):
235 self.SetUpSearchIssueFullText()
236 self.mox.ReplayAll()
237 summary_fd = tracker_pb2.FieldDef(
238 field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
239 query_ast_conj = ast_pb2.Conjunction(conds=[
240 ast_pb2.Condition(
241 op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd],
242 str_values=['test'])])
243 issue_ids, capped = tracker_fulltext.SearchIssueFullText(
244 [789], query_ast_conj, 1)
245 self.mox.VerifyAll()
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +0100246 six.assertCountEqual(self, [123, 234], issue_ids)
Copybara854996b2021-09-07 19:36:02 +0000247 self.assertFalse(capped)
248
249 def testSearchIssueFullText_CrossProject(self):
250 self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch')
251 fulltext_helpers.ComprehensiveSearch(
252 '(project_id:789 OR project_id:678) (summary:"test")',
253 settings.search_index_name_format % 1).AndReturn([123, 234])
254 self.mox.ReplayAll()
255
256 summary_fd = tracker_pb2.FieldDef(
257 field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
258 query_ast_conj = ast_pb2.Conjunction(conds=[
259 ast_pb2.Condition(
260 op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd],
261 str_values=['test'])])
262 issue_ids, capped = tracker_fulltext.SearchIssueFullText(
263 [789, 678], query_ast_conj, 1)
264 self.mox.VerifyAll()
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +0100265 six.assertCountEqual(self, [123, 234], issue_ids)
Copybara854996b2021-09-07 19:36:02 +0000266 self.assertFalse(capped)
267
268 def testSearchIssueFullText_Capped(self):
269 try:
270 orig = settings.fulltext_limit_per_shard
271 settings.fulltext_limit_per_shard = 1
272 self.SetUpSearchIssueFullText()
273 self.mox.ReplayAll()
274 summary_fd = tracker_pb2.FieldDef(
275 field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
276 query_ast_conj = ast_pb2.Conjunction(conds=[
277 ast_pb2.Condition(
278 op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd],
279 str_values=['test'])])
280 issue_ids, capped = tracker_fulltext.SearchIssueFullText(
281 [789], query_ast_conj, 1)
282 self.mox.VerifyAll()
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +0100283 six.assertCountEqual(self, [123, 234], issue_ids)
Copybara854996b2021-09-07 19:36:02 +0000284 self.assertTrue(capped)
285 finally:
286 settings.fulltext_limit_per_shard = orig