Cache thread avatars
This change adds the AvatarsDB class, which lets the threadListAvatars
feature interact with a cache of thread avatars.
This is an implementation of points 1 and 2 in the "Idea" section of the
following doc: go/eu7T9m (public link available in the linked bug). The
doc includes a rationale for this change and what it does.
Bug: 2
Change-Id: Ida9fcd909e3bd4a552361317b9013cb8734272a6
diff --git a/package-lock.json b/package-lock.json
index 03df080..21767d8 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,6 +8,10 @@
"name": "twpowertools",
"version": "0.0.0",
"license": "MIT",
+ "dependencies": {
+ "idb": "^6.1.2",
+ "poll-until-promise": "^3.6.1"
+ },
"devDependencies": {
"copy-webpack-plugin": "^9.0.1",
"json5": "^2.2.0",
@@ -769,6 +773,11 @@
"node": ">=10.17.0"
}
},
+ "node_modules/idb": {
+ "version": "6.1.2",
+ "resolved": "https://registry.npmjs.org/idb/-/idb-6.1.2.tgz",
+ "integrity": "sha512-1DNDVu3yDhAZkFDlJf0t7r+GLZ248F5pTAtA7V0oVG3yjmV125qZOx3g0XpAEkGZVYQiFDAsSOnGet2bhugc3w=="
+ },
"node_modules/ignore": {
"version": "5.1.8",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz",
@@ -1188,6 +1197,11 @@
"node": ">=8"
}
},
+ "node_modules/poll-until-promise": {
+ "version": "3.6.1",
+ "resolved": "https://registry.npmjs.org/poll-until-promise/-/poll-until-promise-3.6.1.tgz",
+ "integrity": "sha512-m9awH+xxzFJ+SI3McCO+eQl2qoTYqE9Ql50Mf0oxHzdkrhzd4XdleRsLgPRqbZMqTkAip7XGia026wbZ00y59w=="
+ },
"node_modules/process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
@@ -2343,6 +2357,11 @@
"integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
"dev": true
},
+ "idb": {
+ "version": "6.1.2",
+ "resolved": "https://registry.npmjs.org/idb/-/idb-6.1.2.tgz",
+ "integrity": "sha512-1DNDVu3yDhAZkFDlJf0t7r+GLZ248F5pTAtA7V0oVG3yjmV125qZOx3g0XpAEkGZVYQiFDAsSOnGet2bhugc3w=="
+ },
"ignore": {
"version": "5.1.8",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz",
@@ -2650,6 +2669,11 @@
"find-up": "^4.0.0"
}
},
+ "poll-until-promise": {
+ "version": "3.6.1",
+ "resolved": "https://registry.npmjs.org/poll-until-promise/-/poll-until-promise-3.6.1.tgz",
+ "integrity": "sha512-m9awH+xxzFJ+SI3McCO+eQl2qoTYqE9Ql50Mf0oxHzdkrhzd4XdleRsLgPRqbZMqTkAip7XGia026wbZ00y59w=="
+ },
"process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
diff --git a/package.json b/package.json
index 0044d04..33e929e 100644
--- a/package.json
+++ b/package.json
@@ -32,5 +32,9 @@
"webpack-cli": "^4.7.2",
"webpack-shell-plugin-next": "^2.2.2"
},
- "private": true
+ "private": true,
+ "dependencies": {
+ "idb": "^6.1.2",
+ "poll-until-promise": "^3.6.1"
+ }
}
diff --git a/src/common/xhrInterceptors.json5 b/src/common/xhrInterceptors.json5
index 158edb9..cc0699f 100644
--- a/src/common/xhrInterceptors.json5
+++ b/src/common/xhrInterceptors.json5
@@ -1,9 +1,9 @@
{
interceptors: [
- /*{
- eventName: "ViewThreadResponse",
- urlRegex: "api/ViewThread",
+ {
+ eventName: "ViewForumResponse",
+ urlRegex: "api/ViewForum",
intercepts: "response",
- },*/ // Example
+ },
],
}
diff --git a/src/contentScripts/communityConsole/autoRefresh.js b/src/contentScripts/communityConsole/autoRefresh.js
index c39970f..6a1a491 100644
--- a/src/contentScripts/communityConsole/autoRefresh.js
+++ b/src/contentScripts/communityConsole/autoRefresh.js
@@ -1,4 +1,4 @@
-import {createExtBadge} from './utils.js';
+import {createExtBadge} from './utils/common.js';
import {getAuthUser} from '../../common/communityConsoleUtils.js';
var authuser = getAuthUser();
diff --git a/src/contentScripts/communityConsole/avatars.js b/src/contentScripts/communityConsole/avatars.js
index 253fe8f..3b27333 100644
--- a/src/contentScripts/communityConsole/avatars.js
+++ b/src/contentScripts/communityConsole/avatars.js
@@ -1,9 +1,16 @@
+import {waitFor} from 'poll-until-promise';
+
import {CCApi} from '../../common/api.js';
import {parseUrl} from '../../common/commonUtils.js';
-export var avatars = {
- isFilterSetUp: false,
- privateForums: [],
+import AvatarsDB from './utils/AvatarsDB.js'
+
+export default class AvatarsHandler {
+ constructor() {
+ this.isFilterSetUp = false;
+ this.privateForums = [];
+ this.db = new AvatarsDB();
+ }
// Gets a list of private forums. If it is already cached, the cached list is
// returned; otherwise it is also computed and cached.
@@ -41,7 +48,7 @@
this.isFilterSetUp = true;
return resolve(this.privateForums);
});
- },
+ }
// Some threads belong to private forums, and this feature will not be able to
// get its avatars since it makes an anonymomus call to get the contents of
@@ -53,10 +60,11 @@
return this.getPrivateForums().then(privateForums => {
return !privateForums.includes(thread.forum);
});
- },
+ }
- // Get an object with the author of the thread and an array of the first |num|
- // replies from the thread |thread|.
+ // Get an object with the author of the thread, an array of the first |num|
+ // replies from the thread |thread|, and additional information about the
+ // thread.
getFirstMessages(thread, num = 15) {
return CCApi(
'ViewThread', {
@@ -103,9 +111,82 @@
return {
messages,
author,
+
+ // The following fields are useful for the cache and can be
+ // undefined, but this is checked before adding an entry to the
+ // cache.
+ updatedTimestamp: data?.['1']?.['2']?.['1']?.['4'],
+ lastMessageId: data?.['1']?.['2']?.['10'],
};
});
- },
+ }
+
+ // Get a list of at most |num| avatars for thread |thread| by calling the API
+ getVisibleAvatarsFromServer(thread, num) {
+ return this.getFirstMessages(thread).then(result => {
+ var messages = result.messages;
+ var author = result.author;
+ var updatedTimestamp =
+ Math.floor(Number.parseInt(result.updatedTimestamp) / 1000000);
+ var lastMessageId = result.lastMessageId;
+
+ var avatarUrls = [];
+
+ var authorUrl = author?.['1']?.['2'];
+ if (authorUrl !== undefined) avatarUrls.push(authorUrl);
+
+ for (var m of messages) {
+ var url = m?.['3']?.['1']?.['2'];
+
+ if (url === undefined) continue;
+ if (!avatarUrls.includes(url)) avatarUrls.push(url);
+ if (avatarUrls.length == 3) break;
+ }
+
+ // Add entry to cache if all the extra metadata could be retrieved.
+ if (updatedTimestamp !== undefined && lastMessageId !== undefined)
+ this.db.putCacheEntry({
+ threadId: thread.thread,
+ updatedTimestamp,
+ lastMessageId,
+ avatarUrls,
+ num,
+ lastUsedTimestamp: Math.floor(Date.now() / 1000),
+ });
+
+ return avatarUrls;
+ });
+ }
+
+ // Returns an object with a cache entry that matches the request if found (via
+ // the |entry| property). The property |found| indicates whether the cache
+ // entry was found.
+ getVisibleAvatarsFromCache(thread, num) {
+ return waitFor(
+ () => this.db.getCacheEntry(thread.thread).then(entry => {
+ if (entry === undefined || entry.num < num)
+ return {
+ found: false,
+ };
+
+ // Only use cache entry if lastUsedTimestamp is within the last 30
+ // seconds (which means it has been checked for invalidations):
+ var now = Math.floor(Date.now() / 1000);
+ var diff = now - entry.lastUsedTimestamp;
+ if (diff > 30)
+ throw new Error(
+ 'lastUsedTimestamp isn\'t within the last 30 seconds (the difference is:',
+ diff, ').');
+ return {
+ found: true,
+ entry,
+ };
+ }),
+ {
+ interval: 400,
+ timeout: 10 * 1000,
+ });
+ }
// Get a list of at most |num| avatars for thread |thread|
getVisibleAvatars(thread, num = 3) {
@@ -115,27 +196,19 @@
return [];
}
- return this.getFirstMessages(thread).then(result => {
- var messages = result.messages;
- var author = result.author;
+ return this.getVisibleAvatarsFromCache(thread, num)
+ .then(res => {
+ if (res.found) return res.entry.avatarUrls;
- var avatarUrls = [];
-
- var authorUrl = author?.['1']?.['2'];
- if (authorUrl !== undefined) avatarUrls.push(authorUrl);
-
- for (var m of messages) {
- var url = m?.['3']?.['1']?.['2'];
-
- if (url === undefined) continue;
- if (!avatarUrls.includes(url)) avatarUrls.push(url);
- if (avatarUrls.length == 3) break;
- }
-
- return avatarUrls;
- });
+ return this.getVisibleAvatarsFromServer(thread, num);
+ })
+ .catch(err => {
+ console.error(
+ '[threadListAvatars] Error while retrieving avatars:', err);
+ return this.getVisibleAvatarsFromServer(thread, num);
+ });
});
- },
+ }
// Inject avatars for thread summary (thread item) |node| in a thread list.
inject(node) {
@@ -174,5 +247,5 @@
'[threadListAvatars] Could not retrieve avatars for thread',
thread, err);
});
- },
+ }
};
diff --git a/src/contentScripts/communityConsole/batchLock.js b/src/contentScripts/communityConsole/batchLock.js
index 5bc0361..c8aca09 100644
--- a/src/contentScripts/communityConsole/batchLock.js
+++ b/src/contentScripts/communityConsole/batchLock.js
@@ -1,4 +1,4 @@
-import {removeChildNodes, createExtBadge} from './utils.js';
+import {removeChildNodes, createExtBadge} from './utils/common.js';
export function nodeIsReadToggleBtn(node) {
return ('tagName' in node) && node.tagName == 'MATERIAL-BUTTON' &&
diff --git a/src/contentScripts/communityConsole/darkMode.js b/src/contentScripts/communityConsole/darkMode.js
index d10a77c..d357bad 100644
--- a/src/contentScripts/communityConsole/darkMode.js
+++ b/src/contentScripts/communityConsole/darkMode.js
@@ -1,4 +1,4 @@
-import {createExtBadge} from './utils.js';
+import {createExtBadge} from './utils/common.js';
export function injectDarkModeButton(rightControl, previousDarkModeOption) {
var darkThemeSwitch = document.createElement('material-button');
diff --git a/src/contentScripts/communityConsole/main.js b/src/contentScripts/communityConsole/main.js
index dd11851..69b3bb7 100644
--- a/src/contentScripts/communityConsole/main.js
+++ b/src/contentScripts/communityConsole/main.js
@@ -1,7 +1,7 @@
import {injectScript, injectStyles, injectStylesheet} from '../../common/contentScriptsUtils.js';
import {autoRefresh} from './autoRefresh.js';
-import {avatars} from './avatars.js';
+import AvatarsHandler from './avatars.js';
import {addBatchLockBtn, nodeIsReadToggleBtn} from './batchLock.js';
import {injectDarkModeButton, isDarkThemeOn} from './darkMode.js';
import {applyDragAndDropFix} from './dragAndDropFix.js';
@@ -9,7 +9,7 @@
import {injectPreviousPostsLinks} from './profileHistoryLink.js';
import {unifiedProfilesFix} from './unifiedProfiles.js';
-var mutationObserver, intersectionObserver, intersectionOptions, options;
+var mutationObserver, intersectionObserver, intersectionOptions, options, avatars;
const watchedNodesSelectors = [
// App container (used to set up the intersection observer and inject the dark
@@ -177,6 +177,10 @@
chrome.storage.sync.get(null, function(items) {
options = items;
+ // Initialize classes needed by the mutation observer
+ if (options.threadlistavatars)
+ avatars = new AvatarsHandler();
+
// Before starting the mutation Observer, check whether we missed any
// mutations by manually checking whether some watched nodes already
// exist.
@@ -208,8 +212,7 @@
}
if (options.repositionexpandthread) {
- injectStylesheet(
- chrome.runtime.getURL('css/reposition_expand_thread.css'));
+ injectStylesheet(chrome.runtime.getURL('css/reposition_expand_thread.css'));
}
if (options.ccforcehidedrawer) {
@@ -225,8 +228,7 @@
}
if (options.threadlistavatars) {
- injectStylesheet(
- chrome.runtime.getURL('css/thread_list_avatars.css'));
+ injectStylesheet(chrome.runtime.getURL('css/thread_list_avatars.css'));
}
if (options.autorefreshlist) {
diff --git a/src/contentScripts/communityConsole/profileHistoryLink.js b/src/contentScripts/communityConsole/profileHistoryLink.js
index d53f9ee..751ddd4 100644
--- a/src/contentScripts/communityConsole/profileHistoryLink.js
+++ b/src/contentScripts/communityConsole/profileHistoryLink.js
@@ -1,4 +1,4 @@
-import {getNParent, createExtBadge} from './utils.js';
+import {getNParent, createExtBadge} from './utils/common.js';
import {escapeUsername, getAuthUser} from '../../common/communityConsoleUtils.js';
var authuser = getAuthUser();
diff --git a/src/contentScripts/communityConsole/utils/AvatarsDB.js b/src/contentScripts/communityConsole/utils/AvatarsDB.js
new file mode 100644
index 0000000..b59460d
--- /dev/null
+++ b/src/contentScripts/communityConsole/utils/AvatarsDB.js
@@ -0,0 +1,130 @@
+import {openDB} from 'idb';
+
+const dbName = 'TWPTAvatarsDB';
+const threadListLoadEvent = 'TWPT_ViewForumResponse';
+// Time after the last use when a cache entry should be deleted (in s):
+const expirationTime = 4 * 24 * 60 * 60; // 4 days
+// Probability of running the piece of code to remove unused cache entries after
+// loading the thread list.
+const probRemoveUnusedCacheEntries = 0.10; // 10%
+
+export default class AvatarsDB {
+ constructor() {
+ this.dbPromise = undefined;
+ this.openDB();
+ this.setUpInvalidationsHandler();
+ }
+
+ openDB() {
+ if (this.dbPromise === undefined)
+ this.dbPromise = openDB(dbName, 1, {
+ upgrade: (udb, oldVersion, newVersion, transaction) => {
+ switch (oldVersion) {
+ case 0:
+ var cache = udb.createObjectStore('avatarsCache', {
+ keyPath: 'threadId',
+ });
+ cache.createIndex(
+ 'lastUsedTimestamp', 'lastUsedTimestamp', {unique: false});
+
+ var unauthedForums = udb.createObjectStore('unauthorizedForums', {
+ keyPath: 'forumId',
+ });
+ unauthedForums.createIndex(
+ 'expirationTimestamp', 'expirationTimestamp',
+ {unique: false});
+ }
+ },
+ });
+ }
+
+ getCacheEntry(threadId) {
+ return this.dbPromise.then(db => db.get('avatarsCache', threadId));
+ }
+
+ putCacheEntry(entry) {
+ return this.dbPromise.then(db => db.put('avatarsCache', entry));
+ }
+
+ invalidateCacheEntryIfExists(threadId) {
+ return this.dbPromise.then(db => db.delete('avatarsCache', threadId));
+ }
+
+ removeUnusedCacheEntries() {
+ console.debug('[threadListAvatars] Removing unused cache entries...');
+ return this.dbPromise
+ .then(db => {
+ var upperBoundTimestamp =
+ Math.floor(Date.now() / 1000) - expirationTime;
+ var range = IDBKeyRange.upperBound(upperBoundTimestamp);
+
+ var tx = db.transaction('avatarsCache', 'readwrite');
+ var index = tx.store.index('lastUsedTimestamp');
+ return index.openCursor(range);
+ })
+ .then(function iterateCursor(cursor) {
+ if (!cursor) return;
+ cursor.delete();
+ return cursor.continue().then(iterateCursor);
+ });
+ }
+
+ setUpInvalidationsHandler() {
+ window.addEventListener(
+ threadListLoadEvent, e => this.handleInvalidations(e));
+ }
+
+ handleInvalidations(e) {
+ var response = e?.detail?.body;
+ var threads = response?.['1']?.['2'];
+ if (threads === undefined) {
+ console.warn(
+ '[threadListAvatars] The thread list doesn\'t contain any threads.');
+ return;
+ }
+
+ var promises = [];
+ threads.forEach(t => {
+ var id = t?.['2']?.['1']?.['1'];
+ var currentUpdatedTimestamp =
+ Math.floor(Number.parseInt(t?.['2']?.['1']?.['4']) / 1000000);
+ var currentLastMessageId = t?.['2']?.['10'];
+
+ if (id === undefined || currentUpdatedTimestamp === undefined ||
+ currentLastMessageId === undefined)
+ return;
+
+ promises.push(this.getCacheEntry(id).then(entry => {
+ if (entry === undefined) return;
+
+ // If the cache entry is still valid.
+ if (currentLastMessageId == entry.lastMessageId ||
+ currentUpdatedTimestamp <= entry.updatedTimestamp) {
+ entry.lastUsedTimestamp = Math.floor(Date.now() / 1000);
+ return this.putCacheEntry(entry).catch(err => {
+ console.error(
+ '[threadListAvatars] Error while updating lastUsedTimestamp from thread in cache:',
+ err);
+ });
+ }
+
+ console.debug(
+ '[threadListAvatars] Invalidating thread', entry.threadId);
+ return this.invalidateCacheEntryIfExists(entry.threadId).catch(err => {
+ console.error(
+ '[threadListAvatars] Error while invalidating thread from cache:',
+ err);
+ });
+ }));
+ });
+
+ Promise.allSettled(promises).then(() => {
+ if (Math.random() < probRemoveUnusedCacheEntries)
+ this.removeUnusedCacheEntries().catch(err => {
+ console.error(
+ '[threadListAvatars] Error while removing unused cache entries:',
+ err);
+ });
+ });
+ }
+};
diff --git a/src/contentScripts/communityConsole/utils.js b/src/contentScripts/communityConsole/utils/common.js
similarity index 100%
rename from src/contentScripts/communityConsole/utils.js
rename to src/contentScripts/communityConsole/utils/common.js