blob: 23f793c50260043874a2797926b1aa5d593ece07 [file] [log] [blame]
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +01001# Copyright 2016 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
Copybara854996b2021-09-07 19:36:02 +00004
5"""Tests for the cache classes."""
6from __future__ import print_function
7from __future__ import division
8from __future__ import absolute_import
9
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +010010import six
Copybara854996b2021-09-07 19:36:02 +000011import unittest
12
13from google.appengine.api import memcache
14from google.appengine.ext import testbed
15
Copybara854996b2021-09-07 19:36:02 +000016from services import caches
17from testing import fake
18
19
20class RamCacheTest(unittest.TestCase):
21
22 def setUp(self):
23 self.cnxn = 'fake connection'
24 self.cache_manager = fake.CacheManager()
25 self.ram_cache = caches.RamCache(self.cache_manager, 'issue', max_size=3)
26
27 def testInit(self):
28 self.assertEqual('issue', self.ram_cache.kind)
29 self.assertEqual(3, self.ram_cache.max_size)
30 self.assertEqual(
31 [self.ram_cache],
32 self.cache_manager.cache_registry['issue'])
33
34 def testCacheItem(self):
35 self.ram_cache.CacheItem(123, 'foo')
36 self.assertEqual('foo', self.ram_cache.cache[123])
37
38 def testCacheItem_DropsOldItems(self):
39 self.ram_cache.CacheItem(123, 'foo')
40 self.ram_cache.CacheItem(234, 'foo')
41 self.ram_cache.CacheItem(345, 'foo')
42 self.ram_cache.CacheItem(456, 'foo')
43 # The cache does not get bigger than its limit.
44 self.assertEqual(3, len(self.ram_cache.cache))
45 # An old value is dropped, not the newly added one.
46 self.assertIn(456, self.ram_cache.cache)
47
48 def testCacheAll(self):
49 self.ram_cache.CacheAll({123: 'foo'})
50 self.assertEqual('foo', self.ram_cache.cache[123])
51
52 def testCacheAll_DropsOldItems(self):
53 self.ram_cache.CacheAll({1: 'a', 2: 'b', 3: 'c'})
54 self.ram_cache.CacheAll({4: 'x', 5: 'y'})
55 # The cache does not get bigger than its limit.
56 self.assertEqual(3, len(self.ram_cache.cache))
57 # An old value is dropped, not the newly added one.
58 self.assertIn(4, self.ram_cache.cache)
59 self.assertIn(5, self.ram_cache.cache)
60 self.assertEqual('y', self.ram_cache.cache[5])
61
62 def testHasItem(self):
63 self.ram_cache.CacheItem(123, 'foo')
64 self.assertTrue(self.ram_cache.HasItem(123))
65 self.assertFalse(self.ram_cache.HasItem(999))
66
67 def testGetItem(self):
68 self.ram_cache.CacheItem(123, 'foo')
69 self.assertEqual('foo', self.ram_cache.GetItem(123))
70 self.assertEqual(None, self.ram_cache.GetItem(456))
71
72 def testGetAll(self):
73 self.ram_cache.CacheItem(123, 'foo')
74 self.ram_cache.CacheItem(124, 'bar')
75 hits, misses = self.ram_cache.GetAll([123, 124, 999])
76 self.assertEqual({123: 'foo', 124: 'bar'}, hits)
77 self.assertEqual([999], misses)
78
79 def testLocalInvalidate(self):
80 self.ram_cache.CacheAll({123: 'a', 124: 'b', 125: 'c'})
81 self.ram_cache.LocalInvalidate(124)
82 self.assertEqual(2, len(self.ram_cache.cache))
83 self.assertNotIn(124, self.ram_cache.cache)
84
85 self.ram_cache.LocalInvalidate(999)
86 self.assertEqual(2, len(self.ram_cache.cache))
87
88 def testInvalidate(self):
89 self.ram_cache.CacheAll({123: 'a', 124: 'b', 125: 'c'})
90 self.ram_cache.Invalidate(self.cnxn, 124)
91 self.assertEqual(2, len(self.ram_cache.cache))
92 self.assertNotIn(124, self.ram_cache.cache)
93 self.assertEqual(self.cache_manager.last_call,
94 ('StoreInvalidateRows', self.cnxn, 'issue', [124]))
95
96 def testInvalidateKeys(self):
97 self.ram_cache.CacheAll({123: 'a', 124: 'b', 125: 'c'})
98 self.ram_cache.InvalidateKeys(self.cnxn, [124])
99 self.assertEqual(2, len(self.ram_cache.cache))
100 self.assertNotIn(124, self.ram_cache.cache)
101 self.assertEqual(self.cache_manager.last_call,
102 ('StoreInvalidateRows', self.cnxn, 'issue', [124]))
103
104 def testLocalInvalidateAll(self):
105 self.ram_cache.CacheAll({123: 'a', 124: 'b', 125: 'c'})
106 self.ram_cache.LocalInvalidateAll()
107 self.assertEqual(0, len(self.ram_cache.cache))
108
109 def testInvalidateAll(self):
110 self.ram_cache.CacheAll({123: 'a', 124: 'b', 125: 'c'})
111 self.ram_cache.InvalidateAll(self.cnxn)
112 self.assertEqual(0, len(self.ram_cache.cache))
113 self.assertEqual(self.cache_manager.last_call,
114 ('StoreInvalidateAll', self.cnxn, 'issue'))
115
116
117class ShardedRamCacheTest(unittest.TestCase):
118
119 def setUp(self):
120 self.cnxn = 'fake connection'
121 self.cache_manager = fake.CacheManager()
122 self.sharded_ram_cache = caches.ShardedRamCache(
123 self.cache_manager, 'issue', max_size=3, num_shards=3)
124
125 def testLocalInvalidate(self):
126 self.sharded_ram_cache.CacheAll({
127 (123, 0): 'a',
128 (123, 1): 'aa',
129 (123, 2): 'aaa',
130 (124, 0): 'b',
131 (124, 1): 'bb',
132 (124, 2): 'bbb',
133 })
134 self.sharded_ram_cache.LocalInvalidate(124)
135 self.assertEqual(3, len(self.sharded_ram_cache.cache))
136 self.assertNotIn((124, 0), self.sharded_ram_cache.cache)
137 self.assertNotIn((124, 1), self.sharded_ram_cache.cache)
138 self.assertNotIn((124, 2), self.sharded_ram_cache.cache)
139
140 self.sharded_ram_cache.LocalInvalidate(999)
141 self.assertEqual(3, len(self.sharded_ram_cache.cache))
142
143
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +0100144class _TestableTwoLevelCache(caches.AbstractTwoLevelCache):
Copybara854996b2021-09-07 19:36:02 +0000145
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +0200146 def __init__(self, cache_manager, kind, max_size=None):
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +0100147 super(_TestableTwoLevelCache, self).__init__(
Adrià Vilanova Martínezde942802022-07-15 14:06:55 +0200148 cache_manager, kind, 'testable:', None, max_size=max_size)
Copybara854996b2021-09-07 19:36:02 +0000149
150 # pylint: disable=unused-argument
151 def FetchItems(self, cnxn, keys, **kwargs):
152 """On RAM and memcache miss, hit the database."""
153 return {key: key for key in keys if key < 900}
154
155
156class AbstractTwoLevelCacheTest_Memcache(unittest.TestCase):
157
158 def setUp(self):
159 self.testbed = testbed.Testbed()
160 self.testbed.activate()
161 self.testbed.init_memcache_stub()
162
163 self.cnxn = 'fake connection'
164 self.cache_manager = fake.CacheManager()
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +0100165 self.testable_2lc = _TestableTwoLevelCache(self.cache_manager, 'issue')
Copybara854996b2021-09-07 19:36:02 +0000166
167 def tearDown(self):
168 self.testbed.deactivate()
169
170 def testCacheItem(self):
171 self.testable_2lc.CacheItem(123, 12300)
172 self.assertEqual(12300, self.testable_2lc.cache.cache[123])
173
174 def testHasItem(self):
175 self.testable_2lc.CacheItem(123, 12300)
176 self.assertTrue(self.testable_2lc.HasItem(123))
177 self.assertFalse(self.testable_2lc.HasItem(444))
178 self.assertFalse(self.testable_2lc.HasItem(999))
179
180 def testWriteToMemcache_Normal(self):
181 retrieved_dict = {123: 12300, 124: 12400}
182 self.testable_2lc._WriteToMemcache(retrieved_dict)
183 actual_123, _ = self.testable_2lc._ReadFromMemcache([123])
184 self.assertEqual(12300, actual_123[123])
185 actual_124, _ = self.testable_2lc._ReadFromMemcache([124])
186 self.assertEqual(12400, actual_124[124])
187
188 def testWriteToMemcache_String(self):
189 retrieved_dict = {123: 'foo', 124: 'bar'}
190 self.testable_2lc._WriteToMemcache(retrieved_dict)
191 actual_123, _ = self.testable_2lc._ReadFromMemcache([123])
192 self.assertEqual('foo', actual_123[123])
193 actual_124, _ = self.testable_2lc._ReadFromMemcache([124])
194 self.assertEqual('bar', actual_124[124])
195
196 def testWriteToMemcache_ProtobufInt(self):
197 self.testable_2lc.pb_class = int
198 retrieved_dict = {123: 12300, 124: 12400}
199 self.testable_2lc._WriteToMemcache(retrieved_dict)
200 actual_123, _ = self.testable_2lc._ReadFromMemcache([123])
201 self.assertEqual(12300, actual_123[123])
202 actual_124, _ = self.testable_2lc._ReadFromMemcache([124])
203 self.assertEqual(12400, actual_124[124])
204
205 def testWriteToMemcache_List(self):
206 retrieved_dict = {123: [1, 2, 3], 124: [1, 2, 4]}
207 self.testable_2lc._WriteToMemcache(retrieved_dict)
208 actual_123, _ = self.testable_2lc._ReadFromMemcache([123])
209 self.assertEqual([1, 2, 3], actual_123[123])
210 actual_124, _ = self.testable_2lc._ReadFromMemcache([124])
211 self.assertEqual([1, 2, 4], actual_124[124])
212
213 def testWriteToMemcache_Dict(self):
214 retrieved_dict = {123: {'ham': 2, 'spam': 3}, 124: {'eggs': 2, 'bean': 4}}
215 self.testable_2lc._WriteToMemcache(retrieved_dict)
216 actual_123, _ = self.testable_2lc._ReadFromMemcache([123])
217 self.assertEqual({'ham': 2, 'spam': 3}, actual_123[123])
218 actual_124, _ = self.testable_2lc._ReadFromMemcache([124])
219 self.assertEqual({'eggs': 2, 'bean': 4}, actual_124[124])
220
221 def testWriteToMemcache_HugeValue(self):
222 """If memcache refuses to store a huge value, we don't store any."""
223 self.testable_2lc._WriteToMemcache({124: 124999}) # Gets deleted.
224 huge_str = 'huge' * 260000
225 retrieved_dict = {123: huge_str, 124: 12400}
226 self.testable_2lc._WriteToMemcache(retrieved_dict)
227 actual_123 = memcache.get('testable:123')
228 self.assertEqual(None, actual_123)
229 actual_124 = memcache.get('testable:124')
230 self.assertEqual(None, actual_124)
231
232 def testGetAll_FetchGetsIt(self):
233 self.testable_2lc.CacheItem(123, 12300)
234 self.testable_2lc.CacheItem(124, 12400)
235 # Clear the RAM cache so that we find items in memcache.
236 self.testable_2lc.cache.LocalInvalidateAll()
237 self.testable_2lc.CacheItem(125, 12500)
238 hits, misses = self.testable_2lc.GetAll(self.cnxn, [123, 124, 333, 444])
239 self.assertEqual({123: 12300, 124: 12400, 333: 333, 444: 444}, hits)
240 self.assertEqual([], misses)
241 # The RAM cache now has items found in memcache and DB.
Adrià Vilanova Martínezf19ea432024-01-23 20:20:52 +0100242 six.assertCountEqual(
243 self, [123, 124, 125, 333, 444],
244 list(self.testable_2lc.cache.cache.keys()))
Copybara854996b2021-09-07 19:36:02 +0000245
246 def testGetAll_FetchGetsItFromDB(self):
247 self.testable_2lc.CacheItem(123, 12300)
248 self.testable_2lc.CacheItem(124, 12400)
249 hits, misses = self.testable_2lc.GetAll(self.cnxn, [123, 124, 333, 444])
250 self.assertEqual({123: 12300, 124: 12400, 333: 333, 444: 444}, hits)
251 self.assertEqual([], misses)
252
253 def testGetAll_FetchDoesNotFindIt(self):
254 self.testable_2lc.CacheItem(123, 12300)
255 self.testable_2lc.CacheItem(124, 12400)
256 hits, misses = self.testable_2lc.GetAll(self.cnxn, [123, 124, 999])
257 self.assertEqual({123: 12300, 124: 12400}, hits)
258 self.assertEqual([999], misses)
259
260 def testInvalidateKeys(self):
261 self.testable_2lc.CacheItem(123, 12300)
262 self.testable_2lc.CacheItem(124, 12400)
263 self.testable_2lc.CacheItem(125, 12500)
264 self.testable_2lc.InvalidateKeys(self.cnxn, [124])
265 self.assertEqual(2, len(self.testable_2lc.cache.cache))
266 self.assertNotIn(124, self.testable_2lc.cache.cache)
267 self.assertEqual(
268 self.cache_manager.last_call,
269 ('StoreInvalidateRows', self.cnxn, 'issue', [124]))
270
271 def testGetAllAlreadyInRam(self):
272 self.testable_2lc.CacheItem(123, 12300)
273 self.testable_2lc.CacheItem(124, 12400)
274 hits, misses = self.testable_2lc.GetAllAlreadyInRam(
275 [123, 124, 333, 444, 999])
276 self.assertEqual({123: 12300, 124: 12400}, hits)
277 self.assertEqual([333, 444, 999], misses)
278
279 def testInvalidateAllRamEntries(self):
280 self.testable_2lc.CacheItem(123, 12300)
281 self.testable_2lc.CacheItem(124, 12400)
282 self.testable_2lc.InvalidateAllRamEntries(self.cnxn)
283 self.assertFalse(self.testable_2lc.HasItem(123))
284 self.assertFalse(self.testable_2lc.HasItem(124))