blob: a4c935ec2688d0e7886c5b6a44c9d08d9eb46f89 [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Unit tests for tracker_fulltext module."""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import unittest
try:
from mox3 import mox
except ImportError:
import mox
from google.appengine.api import search
import settings
from framework import framework_views
from proto import ast_pb2
from proto import tracker_pb2
from services import fulltext_helpers
from services import tracker_fulltext
from testing import fake
from tracker import tracker_bizobj
class TrackerFulltextTest(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
self.mock_index = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(search, 'Index')
self.docs = None
self.cnxn = 'fake connection'
self.user_service = fake.UserService()
self.user_service.TestAddUser('test@example.com', 111)
self.issue_service = fake.IssueService()
self.config_service = fake.ConfigService()
self.issue = fake.MakeTestIssue(
123, 1, 'test summary', 'New', 111)
self.issue_service.TestAddIssue(self.issue)
self.comment = tracker_pb2.IssueComment(
project_id=789, issue_id=self.issue.issue_id, user_id=111,
content='comment content',
attachments=[
tracker_pb2.Attachment(filename='hello.c'),
tracker_pb2.Attachment(filename='hello.h')])
self.issue_service.TestAddComment(self.comment, 1)
self.users_by_id = framework_views.MakeAllUserViews(
self.cnxn, self.user_service, [111])
def tearDown(self):
self.mox.UnsetStubs()
self.mox.ResetAll()
def RecordDocs(self, docs):
self.docs = docs
def SetUpIndexIssues(self):
search.Index(name=settings.search_index_name_format % 1).AndReturn(
self.mock_index)
self.mock_index.put(mox.IgnoreArg()).WithSideEffects(self.RecordDocs)
def testIndexIssues(self):
self.SetUpIndexIssues()
self.mox.ReplayAll()
tracker_fulltext.IndexIssues(
self.cnxn, [self.issue], self.user_service, self.issue_service,
self.config_service)
self.mox.VerifyAll()
self.assertEqual(1, len(self.docs))
issue_doc = self.docs[0]
self.assertEqual(123, issue_doc.fields[0].value)
self.assertEqual('test summary', issue_doc.fields[1].value)
def SetUpCreateIssueSearchDocuments(self):
self.mox.StubOutWithMock(tracker_fulltext, '_IndexDocsInShard')
tracker_fulltext._IndexDocsInShard(1, mox.IgnoreArg()).WithSideEffects(
lambda shard_id, docs: self.RecordDocs(docs))
def testCreateIssueSearchDocuments_Normal(self):
self.SetUpCreateIssueSearchDocuments()
self.mox.ReplayAll()
config_dict = {123: tracker_bizobj.MakeDefaultProjectIssueConfig(123)}
tracker_fulltext._CreateIssueSearchDocuments(
[self.issue], {self.issue.issue_id: [self.comment]}, self.users_by_id,
config_dict)
self.mox.VerifyAll()
self.assertEqual(1, len(self.docs))
issue_doc = self.docs[0]
self.assertEqual(5, len(issue_doc.fields))
self.assertEqual(123, issue_doc.fields[0].value)
self.assertEqual('test summary', issue_doc.fields[1].value)
self.assertEqual('test@example.com comment content hello.c hello.h',
issue_doc.fields[3].value)
self.assertEqual('', issue_doc.fields[4].value)
def testCreateIssueSearchDocuments_NoIndexableComments(self):
"""Sometimes all comments on a issue are spam or deleted."""
self.SetUpCreateIssueSearchDocuments()
self.mox.ReplayAll()
config_dict = {123: tracker_bizobj.MakeDefaultProjectIssueConfig(123)}
self.comment.deleted_by = 111
tracker_fulltext._CreateIssueSearchDocuments(
[self.issue], {self.issue.issue_id: [self.comment]}, self.users_by_id,
config_dict)
self.mox.VerifyAll()
self.assertEqual(1, len(self.docs))
issue_doc = self.docs[0]
self.assertEqual(5, len(issue_doc.fields))
self.assertEqual(123, issue_doc.fields[0].value)
self.assertEqual('test summary', issue_doc.fields[1].value)
self.assertEqual('', issue_doc.fields[3].value)
self.assertEqual('', issue_doc.fields[4].value)
def testCreateIssueSearchDocuments_CustomFields(self):
self.SetUpCreateIssueSearchDocuments()
self.mox.ReplayAll()
config = tracker_bizobj.MakeDefaultProjectIssueConfig(123)
config_dict = {123: tracker_bizobj.MakeDefaultProjectIssueConfig(123)}
int_field = tracker_bizobj.MakeFieldDef(
1, 123, 'CustomInt', tracker_pb2.FieldTypes.INT_TYPE, None, False,
False, False, None, None, None, None, False, None, None, None,
'no_action', 'A custom int field', False)
int_field_value = tracker_bizobj.MakeFieldValue(
1, 42, None, None, False, None, None)
str_field = tracker_bizobj.MakeFieldDef(
2, 123, 'CustomStr', tracker_pb2.FieldTypes.STR_TYPE, None, False,
False, False, None, None, None, None, False, None, None, None,
'no_action', 'A custom string field', False)
str_field_value = tracker_bizobj.MakeFieldValue(
2, None, u'\xf0\x9f\x92\x96\xef\xb8\x8f', None, None, None, False)
# TODO(jrobbins): user-type field 3
date_field = tracker_bizobj.MakeFieldDef(
4, 123, 'CustomDate', tracker_pb2.FieldTypes.DATE_TYPE, None, False,
False, False, None, None, None, None, False, None, None, None,
'no_action', 'A custom date field', False)
date_field_value = tracker_bizobj.MakeFieldValue(
4, None, None, None, 1234567890, None, False)
config.field_defs.extend([int_field, str_field, date_field])
self.issue.field_values.extend([
int_field_value, str_field_value, date_field_value])
tracker_fulltext._CreateIssueSearchDocuments(
[self.issue], {self.issue.issue_id: [self.comment]}, self.users_by_id,
config_dict)
self.mox.VerifyAll()
self.assertEqual(1, len(self.docs))
issue_doc = self.docs[0]
metadata = issue_doc.fields[2]
self.assertEqual(
u'New test@example.com [] 42 \xf0\x9f\x92\x96\xef\xb8\x8f 2009-02-13 ',
metadata.value)
def testExtractCommentText(self):
extracted_text = tracker_fulltext._ExtractCommentText(
self.comment, self.users_by_id)
self.assertEqual(
'test@example.com comment content hello.c hello.h',
extracted_text)
def testIndexableComments_NumberOfComments(self):
"""We consider at most 100 initial comments and 500 most recent comments."""
comments = [self.comment]
indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
self.assertEqual(1, len(indexable))
comments = [self.comment] * 100
indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
self.assertEqual(100, len(indexable))
comments = [self.comment] * 101
indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
self.assertEqual(101, len(indexable))
comments = [self.comment] * 600
indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
self.assertEqual(600, len(indexable))
comments = [self.comment] * 601
indexable = tracker_fulltext._IndexableComments(comments, self.users_by_id)
self.assertEqual(600, len(indexable))
self.assertNotIn(100, indexable)
def testIndexableComments_NumberOfChars(self):
"""We consider comments that can fit into the search index document."""
self.comment.content = 'x' * 1000
comments = [self.comment] * 100
indexable = tracker_fulltext._IndexableComments(
comments, self.users_by_id, remaining_chars=100000)
self.assertEqual(100, len(indexable))
indexable = tracker_fulltext._IndexableComments(
comments, self.users_by_id, remaining_chars=50000)
self.assertEqual(50, len(indexable))
indexable = tracker_fulltext._IndexableComments(
comments, self.users_by_id, remaining_chars=50999)
self.assertEqual(50, len(indexable))
indexable = tracker_fulltext._IndexableComments(
comments, self.users_by_id, remaining_chars=999)
self.assertEqual(0, len(indexable))
indexable = tracker_fulltext._IndexableComments(
comments, self.users_by_id, remaining_chars=0)
self.assertEqual(0, len(indexable))
indexable = tracker_fulltext._IndexableComments(
comments, self.users_by_id, remaining_chars=-1)
self.assertEqual(0, len(indexable))
def SetUpUnindexIssues(self):
search.Index(name=settings.search_index_name_format % 1).AndReturn(
self.mock_index)
self.mock_index.delete(['1'])
def testUnindexIssues(self):
self.SetUpUnindexIssues()
self.mox.ReplayAll()
tracker_fulltext.UnindexIssues([1])
self.mox.VerifyAll()
def SetUpSearchIssueFullText(self):
self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch')
fulltext_helpers.ComprehensiveSearch(
'(project_id:789) (summary:"test")',
settings.search_index_name_format % 1).AndReturn([123, 234])
def testSearchIssueFullText_Normal(self):
self.SetUpSearchIssueFullText()
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(
op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd],
str_values=['test'])])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertFalse(capped)
def testSearchIssueFullText_CrossProject(self):
self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch')
fulltext_helpers.ComprehensiveSearch(
'(project_id:789 OR project_id:678) (summary:"test")',
settings.search_index_name_format % 1).AndReturn([123, 234])
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(
op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd],
str_values=['test'])])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789, 678], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertFalse(capped)
def testSearchIssueFullText_Capped(self):
try:
orig = settings.fulltext_limit_per_shard
settings.fulltext_limit_per_shard = 1
self.SetUpSearchIssueFullText()
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(
op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd],
str_values=['test'])])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertTrue(capped)
finally:
settings.fulltext_limit_per_shard = orig