diff --git a/src/api/util/utility/EmbedHandlers.ts b/src/api/util/utility/EmbedHandlers.ts
index 77d0c8f0..da6a7429 100644
--- a/src/api/util/utility/EmbedHandlers.ts
+++ b/src/api/util/utility/EmbedHandlers.ts
@@ -16,12 +16,13 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
-import { Config } from "@spacebar/util";
+import { arrayDistinctBy, arrayGroupBy, arrayRemove, Config, EmbedCache, emitEvent, Message, MessageUpdateEvent, normalizeUrl } from "@spacebar/util";
import { Embed, EmbedImage, EmbedType } from "@spacebar/schemas";
import * as cheerio from "cheerio";
import crypto from "crypto";
import { yellow } from "picocolors";
import probe from "probe-image-size";
+import { FindOptionsWhere, In } from "typeorm";
export const DEFAULT_FETCH_OPTIONS: RequestInit = {
redirect: "follow",
@@ -517,3 +518,161 @@ export const EmbedHandlers: {
};
},
};
+
+const LINK_REGEX = /<?https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)>?/g;
+
+export function getMessageContentUrls(message: Message) {
+ const content = message.content?.replace(/ *`[^)]*` */g, ""); // remove markdown
+
+ return content?.match(LINK_REGEX) ?? [];
+}
+
+export async function dropDuplicateCacheEntries(entries: EmbedCache[]): Promise<EmbedCache[]> {
+ const grouped = Array.from(arrayGroupBy(entries, (e) => e.url).values()).map((g) =>
+ g.toSorted((e1, e2) => {
+ let diff = e2.createdAt.getTime() - e1.createdAt.getTime();
+ if (diff == 0) diff = Number(BigInt(e2.id) - BigInt(e1.id));
+ return diff;
+ }),
+ );
+
+ const fullToDeleteIds: string[] = [];
+ for (const group of grouped) {
+ if (group.length <= 1) continue;
+ // console.log("[EmbedCache] Removing all but first from cache:", group);
+ // this might be backwards, sort always confuses me lol
+ const toDelete = group.slice(1);
+ const toDeleteIds = toDelete.map((x) => x.id);
+ fullToDeleteIds.push(...toDeleteIds);
+ console.warn("[EmbedCache] Removing duplicate IDs for", toDelete[0].url, " - ", toDeleteIds);
+ }
+
+ await EmbedCache.delete({ id: In(fullToDeleteIds) } as FindOptionsWhere<EmbedCache>);
+
+ // console.log("[EmbedCache] Cached embeds:", Array.from(grouped.map((x) => x[0].url)));
+ return Array.from(grouped.map((x) => x[0]));
+}
+
+async function sleep(ms: number) {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// hack to make nodejs not die
+function getSlowdownFactor(off: number) {
+ if (off < 10) return off;
+ if (off < 25) return 100 + off * 2;
+ if (off < 50) return 200 + off * 10;
+ if (off < 100) return 500 + off * 15;
+ if (off < 250) return 750 + off * 20;
+ return 1000 + off * 150;
+}
+
+export async function getOrUpdateEmbedCache(urls: string[], cb?: (url: string, embeds: Embed[]) => Promise<void>): Promise<EmbedCache[]> {
+ urls = arrayDistinctBy(urls, (x) => x);
+ const embeds: EmbedCache[] = [];
+
+ const cachedEmbeds = await dropDuplicateCacheEntries(
+ await EmbedCache.find({
+ where: {
+ url: In(urls.map(normalizeUrl)),
+ },
+ }),
+ );
+ embeds.push(...cachedEmbeds);
+ cb?.(
+ "cached",
+ cachedEmbeds.map((e) => e.embed),
+ );
+
+ const urlsToGenerate = urls.filter((url) => {
+ return !cachedEmbeds.some((e) => e.url == normalizeUrl(url));
+ });
+
+ if (urlsToGenerate.length > 0) console.log("[Embeds] Need to generate embeds for urls:", urlsToGenerate);
+ if (cachedEmbeds.length > 0)
+ console.log(
+ "[Embeds] Already had embeds for urls:",
+ cachedEmbeds.map((e) => e.url),
+ );
+
+ let off = 0;
+ const generatedEmbeds = await Promise.all(
+ urlsToGenerate.map(async (link) => {
+ await sleep(getSlowdownFactor(off++)); // ...or nodejs gets overwhelmed and times out
+ return await getOrUpdateEmbedCacheSingle(link, cb);
+ }),
+ );
+
+ embeds.push(...generatedEmbeds.filter((e): e is EmbedCache[] => e !== null).flat());
+
+ return embeds;
+}
+
+async function getOrUpdateEmbedCacheSingle(link: string, cb?: (url: string, embeds: Embed[]) => Promise<void>): Promise<EmbedCache[] | null> {
+ const url = new URL(link);
+ const handler = url.hostname === new URL(Config.get().cdn.endpointPublic!).hostname ? EmbedHandlers["self"] : (EmbedHandlers[url.hostname] ?? EmbedHandlers["default"]);
+ const results: EmbedCache[] = [];
+ try {
+ let res = await handler(url);
+ if (!res) return null;
+ if (!Array.isArray(res)) res = [res];
+
+ for (const embed of res) {
+ // Cache with normalized URL
+ const cache = await EmbedCache.create({
+ url: normalizeUrl(url.href),
+ embed: embed,
+ createdAt: new Date(),
+ }).save();
+ results.push(cache);
+ console.log("[Embeds] Generated embed for", link);
+ }
+ await cb?.(link, res);
+ } catch (e) {
+ console.error(`[Embeds] Error while generating embed for ${link}`, e);
+ }
+ return results.length == 0 ? null : results;
+}
+
+export async function fillMessageUrlEmbeds(message: Message) {
+ const linkMatches = getMessageContentUrls(message).filter((l) => !l.startsWith("<") && !l.endsWith(">"));
+
+ // Filter out embeds that could be links, start from scratch
+ message.embeds = message.embeds.filter((embed) => embed.type === "rich");
+
+ if (linkMatches.length == 0) return message;
+
+ const uniqueLinks: string[] = arrayDistinctBy(linkMatches, normalizeUrl);
+
+ if (uniqueLinks.length === 0) {
+ // No valid unique links found, update message to remove old embeds
+ message.embeds = message.embeds?.filter((embed) => embed.type === "rich");
+ await saveAndEmitMessageUpdate(message);
+ return message;
+ }
+
+ // avoid a race condition updating the same row
+ let messageUpdateLock = saveAndEmitMessageUpdate(message);
+ await getOrUpdateEmbedCache(uniqueLinks, async (_, embeds) => {
+ if (message.embeds.length + embeds.length > Config.get().limits.message.maxEmbeds) return;
+ message.embeds.push(...embeds);
+ try {
+ await messageUpdateLock;
+ } catch {
+ /* empty */
+ }
+ messageUpdateLock = saveAndEmitMessageUpdate(message);
+ });
+
+ await saveAndEmitMessageUpdate(message);
+ return message;
+}
+
+async function saveAndEmitMessageUpdate(message: Message) {
+ await Message.update({ id: message.id, channel_id: message.channel_id }, { embeds: message.embeds });
+ await emitEvent({
+ event: "MESSAGE_UPDATE",
+ channel_id: message.channel_id,
+ data: message.toJSON(),
+ } satisfies MessageUpdateEvent);
+}
|
